├── testdata ├── data │ └── foo └── README.md ├── .gitignore ├── .github └── workflows │ └── gotest.yml ├── entity ├── chunk.go ├── stat.go ├── file.go └── index.go ├── util ├── disk_mobile.go ├── disk.go └── hash.go ├── ref_test.go ├── sync_test.go ├── log_test.go ├── store_test.go ├── README_zh_CN.md ├── README.md ├── go.mod ├── diff.go ├── ref.go ├── log.go ├── sync_lock.go ├── repo_test.go ├── backup.go ├── sync_manual.go ├── cloud ├── cloud.go ├── local.go ├── webdav.go ├── s3.go └── siyuan.go ├── store.go ├── go.sum └── LICENSE /testdata/data/foo: -------------------------------------------------------------------------------- 1 | Hello -------------------------------------------------------------------------------- /testdata/README.md: -------------------------------------------------------------------------------- 1 | Test data. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | *.log 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # IDE 16 | .idea/ 17 | 18 | # Test 19 | !testdata/data/foo 20 | !testdata/data/bar 21 | testdata/data/ 22 | testdata/data-checkout/ 23 | testdata/repo/ 24 | covprofile 25 | -------------------------------------------------------------------------------- /.github/workflows/gotest.yml: -------------------------------------------------------------------------------- 1 | name: Go Test 2 | on: [ push, pull_request ] 3 | jobs: 4 | test: 5 | name: Test with Coverage 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Set up Go 9 | uses: actions/setup-go@v2 10 | with: 11 | go-version: '1.21' 12 | - name: Check out code 13 | uses: actions/checkout@v2 14 | - name: Install dependencies 15 | run: | 16 | go mod download 17 | - name: Run Unit tests 18 | run: | 19 | go test -coverprofile=covprofile -coverpkg="github.com/siyuan-note/dejavu" ./... 20 | - name: Install goveralls 21 | run: go install github.com/mattn/goveralls@latest 22 | - name: Send coverage 23 | env: 24 | COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | run: goveralls -coverprofile=covprofile -service=github -------------------------------------------------------------------------------- /entity/chunk.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package entity 18 | 19 | // Chunk 描述了文件分块 20 | type Chunk struct { 21 | ID string `json:"id"` 22 | Data []byte `json:"data"` // 实际的数据 23 | } 24 | -------------------------------------------------------------------------------- /util/disk_mobile.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | //go:build ios || android 18 | 19 | package util 20 | 21 | import "math" 22 | 23 | func GetFreeDiskSpace(p string) (free int64) { 24 | return math.MaxInt64 25 | } 26 | -------------------------------------------------------------------------------- /entity/stat.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package entity 18 | 19 | type ObjectInfo struct { 20 | Path string 21 | Size int64 22 | } 23 | 24 | type PurgeStat struct { 25 | Objects int 26 | Indexes int 27 | Size int64 28 | } 29 | -------------------------------------------------------------------------------- /util/disk.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | //go:build !ios && !android 18 | 19 | package util 20 | 21 | import ( 22 | "math" 23 | 24 | "github.com/shirou/gopsutil/v4/disk" 25 | ) 26 | 27 | func GetFreeDiskSpace(p string) (free int64) { 28 | usage, err := disk.Usage(p) 29 | if err != nil { 30 | return math.MaxInt64 31 | } 32 | return int64(usage.Free) 33 | } 34 | -------------------------------------------------------------------------------- /util/hash.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package util 18 | 19 | import ( 20 | "crypto/rand" 21 | "crypto/sha1" 22 | "fmt" 23 | 24 | "github.com/88250/gulu" 25 | "github.com/siyuan-note/logging" 26 | ) 27 | 28 | func Hash(data []byte) string { 29 | return fmt.Sprintf("%x", sha1.Sum(data)) 30 | } 31 | 32 | func RandHash() string { 33 | b := make([]byte, 32) 34 | _, err := rand.Read(b) 35 | if nil != err { 36 | logging.LogErrorf("read rand failed: %s", err) 37 | return Hash([]byte(gulu.Rand.String(512))) 38 | } 39 | return Hash(b) 40 | } 41 | -------------------------------------------------------------------------------- /ref_test.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestTag(t *testing.T) { 24 | clearTestdata(t) 25 | 26 | repo, index := initIndex(t) 27 | err := repo.AddTag(index.ID, "v1.0.0") 28 | if nil != err { 29 | t.Fatalf("add tag failed: %s", err) 30 | return 31 | } 32 | 33 | v100, err := repo.GetTag("v1.0.0") 34 | if v100 != index.ID { 35 | t.Fatalf("get tag failed: %s", err) 36 | return 37 | } 38 | 39 | err = repo.AddTag(index.ID, "v1.0.1") 40 | if nil != err { 41 | t.Fatalf("add tag failed: %s", err) 42 | return 43 | } 44 | 45 | v101, err := repo.GetTag("v1.0.1") 46 | if v101 != v100 { 47 | t.Fatalf("get tag failed: %s", err) 48 | return 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /sync_test.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/siyuan-note/dejavu/cloud" 23 | ) 24 | 25 | func TestSync(t *testing.T) { 26 | repo, _ := initIndex(t) 27 | 28 | userId := "0" 29 | token := "" 30 | 31 | return // 注释掉不跑 32 | 33 | repo.cloud = &cloud.SiYuan{BaseCloud: &cloud.BaseCloud{Conf: &cloud.Conf{ 34 | Dir: "test", 35 | UserID: userId, 36 | AvailableSize: 1024 * 1024 * 1024 * 8, 37 | Token: token, 38 | Server: "http://127.0.0.1:64388", 39 | }}} 40 | 41 | mergeResult, trafficStat, err := repo.Sync(nil) 42 | if nil != err { 43 | t.Fatalf("sync failed: %s", err) 44 | return 45 | } 46 | _ = mergeResult 47 | _ = trafficStat 48 | } 49 | -------------------------------------------------------------------------------- /entity/file.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package entity 18 | 19 | import ( 20 | "bytes" 21 | "strconv" 22 | 23 | "github.com/siyuan-note/dejavu/util" 24 | ) 25 | 26 | // File 描述了文件。 27 | type File struct { 28 | ID string `json:"id"` // Hash 29 | Path string `json:"path"` // 文件路径 30 | Size int64 `json:"size"` // 文件大小 31 | Updated int64 `json:"updated"` // 最后更新时间 32 | Chunks []string `json:"chunks"` // 文件分块列表 33 | } 34 | 35 | func NewFile(path string, size int64, updated int64) (ret *File) { 36 | ret = &File{ 37 | Path: path, 38 | Size: size, 39 | Updated: updated, 40 | } 41 | buf := bytes.Buffer{} 42 | buf.WriteString(ret.Path) 43 | buf.WriteString(strconv.FormatInt(ret.Updated/1000, 10)) 44 | ret.ID = util.Hash(buf.Bytes()) 45 | return 46 | } 47 | 48 | func (f *File) SecUpdated() int64 { 49 | return f.Updated / 1000 50 | } 51 | -------------------------------------------------------------------------------- /log_test.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestGetIndexLogs(t *testing.T) { 24 | clearTestdata(t) 25 | 26 | repo, _ := initIndex(t) 27 | 28 | logs, pageCount, totalCount, err := repo.GetIndexLogs(1, 10) 29 | if nil != err { 30 | t.Fatalf("get index logs failed: %s", err) 31 | return 32 | } 33 | if 1 > len(logs) { 34 | t.Fatalf("logs length not match: %d", len(logs)) 35 | return 36 | } 37 | 38 | t.Logf("page count [%d], total count [%d]", pageCount, totalCount) 39 | for _, log := range logs { 40 | t.Logf("%+v", log) 41 | } 42 | } 43 | 44 | func TestGetTagLogs(t *testing.T) { 45 | clearTestdata(t) 46 | 47 | repo, index := initIndex(t) 48 | err := repo.AddTag(index.ID, "v1.0.0") 49 | if nil != err { 50 | t.Fatalf("add tag failed: %s", err) 51 | return 52 | } 53 | 54 | err = repo.AddTag(index.ID, "v1.0.1") 55 | if nil != err { 56 | t.Fatalf("add tag failed: %s", err) 57 | return 58 | } 59 | 60 | logs, err := repo.GetTagLogs() 61 | if nil != err { 62 | t.Fatalf("get tag logs failed: %s", err) 63 | return 64 | } 65 | if 2 != len(logs) { 66 | t.Fatalf("logs length not match: %d", len(logs)) 67 | return 68 | } 69 | 70 | for _, log := range logs { 71 | t.Logf("%+v", log) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /store_test.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "bytes" 21 | "testing" 22 | 23 | "github.com/siyuan-note/dejavu/entity" 24 | "github.com/siyuan-note/dejavu/util" 25 | "github.com/siyuan-note/encryption" 26 | ) 27 | 28 | func TestPutGet(t *testing.T) { 29 | clearTestdata(t) 30 | 31 | aesKey, err := encryption.KDF(testRepoPassword, testRepoPasswordSalt) 32 | if nil != err { 33 | t.Fatalf("kdf failed: %s", err) 34 | return 35 | } 36 | 37 | store, err := NewStore(testRepoPath, aesKey) 38 | if nil != err { 39 | t.Fatalf("new store failed: %s", err) 40 | return 41 | } 42 | 43 | data := []byte("Hello!") 44 | chunk := &entity.Chunk{ID: util.Hash(data), Data: data} 45 | err = store.PutChunk(chunk) 46 | if nil != err { 47 | t.Fatalf("put failed: %s", err) 48 | return 49 | } 50 | 51 | chunk, err = store.GetChunk(chunk.ID) 52 | if nil != err { 53 | t.Fatalf("get failed: %s", err) 54 | return 55 | } 56 | if 0 != bytes.Compare(chunk.Data, data) { 57 | t.Fatalf("data not match") 58 | return 59 | } 60 | 61 | err = store.Remove(chunk.ID) 62 | if nil != err { 63 | t.Fatalf("remove failed: %s", err) 64 | return 65 | } 66 | 67 | chunk, err = store.GetChunk(chunk.ID) 68 | if nil != chunk { 69 | t.Fatalf("get should be failed") 70 | return 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /entity/index.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package entity 18 | 19 | import ( 20 | "fmt" 21 | "time" 22 | 23 | "github.com/88250/go-humanize" 24 | ) 25 | 26 | // Index 描述了快照索引。 27 | type Index struct { 28 | ID string `json:"id"` // Hash 29 | Memo string `json:"memo"` // 索引备注 30 | Created int64 `json:"created"` // 索引时间 31 | Files []string `json:"files"` // 文件列表 32 | Count int `json:"count"` // 文件总数 33 | Size int64 `json:"size"` // 文件总大小 34 | SystemID string `json:"systemID"` // 系统 ID 35 | SystemName string `json:"systemName"` // 系统名称 36 | SystemOS string `json:"systemOS"` // 系统操作系统 37 | CheckIndexID string `json:"checkIndexID"` // Check Index ID 38 | } 39 | 40 | func (index *Index) String() string { 41 | return fmt.Sprintf("device=%s/%s, id=%s, files=%d, size=%s, created=%s", 42 | index.SystemID, index.SystemOS, index.ID, len(index.Files), humanize.BytesCustomCeil(uint64(index.Size), 2), time.UnixMilli(index.Created).Format("2006-01-02 15:04:05")) 43 | } 44 | 45 | // CheckIndex 描述了一个 Index 对应的数据 ID，包括 File ID 和 Chunk ID。 46 | // 47 | // 该结构体在数据同步云端时根据本地 Latest Index 生成，在云端服务上用于校验数据完整性。 48 | // 49 | // 该数据结构仅保存 ID，因此不会影响端到端加密的安全性，不会对数据安全造成任何影响。 50 | // 51 | // 存放路径：repo/check/indexes/{id}。 52 | type CheckIndex struct { 53 | ID string `json:"id"` // Hash 54 | IndexID string `json:"indexID"` // Index ID 55 | Files []*CheckIndexFile `json:"files"` // File IDs 56 | } 57 | 58 | type CheckIndexFile struct { 59 | ID string `json:"id"` // File ID 60 | Chunks []string `json:"chunks"` // Chunk IDs 61 | } 62 | 63 | type CheckReport struct { 64 | CheckTime int64 `json:"checkTime"` 65 | CheckCount int `json:"checkCount"` 66 | FixCount int `json:"fixCount"` 67 | MissingObjects []string `json:"missingObjects"` 68 | } 69 | -------------------------------------------------------------------------------- /README_zh_CN.md: -------------------------------------------------------------------------------- 1 | # DejaVu 2 | 3 | [English](README.md) 4 | 5 | ## 💡 简介 6 | 7 | [DejaVu](https://github.com/siyuan-note/dejavu) 是思源笔记的数据快照和同步组件。 8 | 9 | ## ✨ 特性 10 | 11 | * 类似 Git 的版本控制 12 | * 文件分块去重 13 | * 数据压缩 14 | * AES 加密 15 | * 云端同步和备份 16 | 17 | ⚠️ 注意 18 | 19 | * 不支持文件夹 20 | * 不支持权限属性 21 | * 不支持符号链接 22 | 23 | ## 🎨 设计 24 | 25 | 设计参考自 [ArtiVC](https://github.com/InfuseAI/ArtiVC)。 26 | 27 | ### 实体 28 | 29 | * `ID` 每个实体都通过 SHA-1 标识 30 | * `Index` 文件列表，每次索引操作都生成一个新的索引 31 | * `memo` 索引备注 32 | * `created` 索引时间 33 | * `files` 文件列表 34 | * `count` 文件总数 35 | * `size` 文件列表总大小 36 | * `File` 文件，实际的数据文件路径或者内容发生变动时生成一个新的文件 37 | * `path` 文件路径 38 | * `size` 文件大小 39 | * `updated` 最后更新时间 40 | * `chunks` 文件分块列表 41 | * `Chunk` 文件块 42 | * `data` 实际的数据 43 | * `Ref` 引用指向索引 44 | * `latest` 内置引用，自动指向最新的索引 45 | * `tag` 标签引用，手动指向指定的索引 46 | * `Repo` 仓库 47 | 48 | ### 仓库 49 | 50 | * `DataPath` 数据文件夹路径，实际的数据文件所在文件夹 51 | * `Path` 仓库文件夹路径，仓库不保存在数据文件夹中，需要单独指定仓库文件夹路径 52 | 53 | 仓库文件夹结构如下： 54 | 55 | ```text 56 | ├─indexes 57 | │ 0531732dca85404e716abd6bb896319a41fa372b 58 | │ 19fc2c2e5317b86f9e048f8d8da2e4ed8300d8af 59 | │ 5f32d78d69e314beee36ad7de302b984da47ddd2 60 | │ cbd254ca246498978d4f47e535bac87ad7640fe6 61 | │ 62 | ├─objects 63 | │ ├─1e 64 | │ │ 0ac5f319f5f24b3fe5bf63639e8dbc31a52e3b 65 | │ │ 66 | │ ├─56 67 | │ │ 322ccdb61feab7f2f76f5eb82006bd51da7348 68 | │ │ 69 | │ ├─7e 70 | │ │ dccca8340ebe149b10660a079f34a20f35c4d4 71 | │ │ 72 | │ ├─83 73 | │ │ a7d72fe9a071b696fc81a3dc041cf36cbde802 74 | │ │ 75 | │ ├─85 76 | │ │ 26b9a7efde615b67b4666ae509f9fbc91d370b 77 | │ │ 78 | │ ├─87 79 | │ │ 1355acd062116d1713e8f7f55969dbb507a040 80 | │ │ 81 | │ ├─96 82 | │ │ 46ba13a4e8eabeca4f5259bfd7da41d368a1a6 83 | │ │ 84 | │ ├─a5 85 | │ │ 5b8e6b9ccad3fc9b792d3d453a0793f8635b9f 86 | │ │ b28787922f4e2a477b4f027e132aa7e35253d4 87 | │ │ 88 | │ ├─be 89 | │ │ c7a729d1b5f021f8eca0dd8b6ef689ad753567 90 | │ │ 91 | │ ├─d1 92 | │ │ 324c714bde18442b5629a84a361b5e7528b14a 93 | │ │ 94 | │ ├─f1 95 | │ │ d7229171f4fa1c5eacb411995b16938a04f7f6 96 | │ │ 97 | │ └─f7 98 | │ ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0 99 | │ 100 | └─refs 101 | │ latest 102 | │ 103 | └─tags 104 | v1.0.0 105 | v1.0.1 106 | ``` 107 | 108 | ## 📄 授权 109 | 110 | DejaVu 使用 [GNU Affero 通用公共许可证, 版本 3](https://www.gnu.org/licenses/agpl-3.0.txt) 开源协议。 111 | 112 | ## 🙏 鸣谢 113 | 114 | * [https://github.com/dustin/go-humanize](https://github.com/dustin/go-humanize) `MIT license` 115 | * [https://github.com/klauspost/compress](https://github.com/klauspost/compress) `BSD-3-Clause license` 116 | * [https://github.com/panjf2000/ants](https://github.com/panjf2000/ants) `MIT license` 117 | * [https://github.com/InfuseAI/ArtiVC](https://github.com/InfuseAI/ArtiVC) `Apache-2.0 license` 118 | * [https://github.com/restic/restic](https://github.com/restic/restic) `BSD-2-Clause license` 119 | * [https://github.com/sabhiram/go-gitignore](https://github.com/sabhiram/go-gitignore) `MIT license` 120 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DejaVu 2 | 3 | [中文](README_zh_CN.md) 4 | 5 | ## 💡 Introduction 6 | 7 | [DejaVu](https://github.com/siyuan-note/dejavu) is the component of data snapshot and sync for SiYuan. 8 | 9 | ## ✨ Features 10 | 11 | * Git-like version control 12 | * File deduplication in chunks 13 | * Data compression 14 | * AES Encrypted 15 | * Cloud sync and backup 16 | 17 | ⚠️ Attention 18 | 19 | * Folders are not supported 20 | * Permission attributes are not supported 21 | * Symbolic links are not supported 22 | 23 | ## 🎨 Design 24 | 25 | Design reference from [ArtiVC](https://github.com/InfuseAI/ArtiVC). 26 | 27 | ### Entity 28 | 29 | * `ID` Each entity is identified by SHA-1 30 | * `Index` file list, each index operation generates a new index 31 | * `memo` index memo 32 | * `created` index time 33 | * `files` file list 34 | * `count` count of total files 35 | * `size` size of total files 36 | * `File` file, a new file is generated when the actual data file path or content changes 37 | * `path` file path 38 | * `size` file size 39 | * `updated` last update time 40 | * `chunks` file chunk list 41 | * `Chunk` file chunk 42 | * `data` actual data 43 | * `Ref` refers to the index 44 | * `latest` built-in reference, automatically points to the latest index 45 | * `tag` tag reference, manually point to the specified index 46 | * `Repo` repository 47 | 48 | ### Repo 49 | 50 | * `DataPath` data folder path, the folder where the actual data file is located 51 | * `Path` repo folder path, the repo is not stored in the data folder, we need to specify the repo folder path separately 52 | 53 | The repo folder layout is as follows: 54 | 55 | ```text 56 | ├─indexes 57 | │ 0531732dca85404e716abd6bb896319a41fa372b 58 | │ 19fc2c2e5317b86f9e048f8d8da2e4ed8300d8af 59 | │ 5f32d78d69e314beee36ad7de302b984da47ddd2 60 | │ cbd254ca246498978d4f47e535bac87ad7640fe6 61 | │ 62 | ├─objects 63 | │ ├─1e 64 | │ │ 0ac5f319f5f24b3fe5bf63639e8dbc31a52e3b 65 | │ │ 66 | │ ├─56 67 | │ │ 322ccdb61feab7f2f76f5eb82006bd51da7348 68 | │ │ 69 | │ ├─7e 70 | │ │ dccca8340ebe149b10660a079f34a20f35c4d4 71 | │ │ 72 | │ ├─83 73 | │ │ a7d72fe9a071b696fc81a3dc041cf36cbde802 74 | │ │ 75 | │ ├─85 76 | │ │ 26b9a7efde615b67b4666ae509f9fbc91d370b 77 | │ │ 78 | │ ├─87 79 | │ │ 1355acd062116d1713e8f7f55969dbb507a040 80 | │ │ 81 | │ ├─96 82 | │ │ 46ba13a4e8eabeca4f5259bfd7da41d368a1a6 83 | │ │ 84 | │ ├─a5 85 | │ │ 5b8e6b9ccad3fc9b792d3d453a0793f8635b9f 86 | │ │ b28787922f4e2a477b4f027e132aa7e35253d4 87 | │ │ 88 | │ ├─be 89 | │ │ c7a729d1b5f021f8eca0dd8b6ef689ad753567 90 | │ │ 91 | │ ├─d1 92 | │ │ 324c714bde18442b5629a84a361b5e7528b14a 93 | │ │ 94 | │ ├─f1 95 | │ │ d7229171f4fa1c5eacb411995b16938a04f7f6 96 | │ │ 97 | │ └─f7 98 | │ ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0 99 | │ 100 | └─refs 101 | │ latest 102 | │ 103 | └─tags 104 | v1.0.0 105 | v1.0.1 106 | ``` 107 | 108 | ## 📄 License 109 | 110 | DejaVu uses the [GNU AFFERO GENERAL PUBLIC LICENSE, Version 3](https://www.gnu.org/licenses/agpl-3.0.txt) open source license. 111 | 112 | ## 🙏 Acknowledgement 113 | 114 | * [https://github.com/dustin/go-humanize](https://github.com/dustin/go-humanize) `MIT license` 115 | * [https://github.com/klauspost/compress](https://github.com/klauspost/compress) `BSD-3-Clause license` 116 | * [https://github.com/panjf2000/ants](https://github.com/panjf2000/ants) `MIT license` 117 | * [https://github.com/InfuseAI/ArtiVC](https://github.com/InfuseAI/ArtiVC) `Apache-2.0 license` 118 | * [https://github.com/restic/restic](https://github.com/restic/restic) `BSD-2-Clause license` 119 | * [https://github.com/sabhiram/go-gitignore](https://github.com/sabhiram/go-gitignore) `MIT license` 120 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/siyuan-note/dejavu 2 | 3 | go 1.24.4 4 | 5 | require ( 6 | github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7 7 | github.com/88250/gulu v1.2.3-0.20251119142510-7b1583ab4aa0 8 | github.com/88250/lute v1.7.7-0.20250801084148-32f2ef961381 9 | github.com/aws/aws-sdk-go-v2 v1.41.0 10 | github.com/aws/aws-sdk-go-v2/config v1.32.6 11 | github.com/aws/aws-sdk-go-v2/credentials v1.19.6 12 | github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 13 | github.com/aws/smithy-go v1.24.0 14 | github.com/dgraph-io/ristretto v0.2.0 15 | github.com/klauspost/compress v1.18.2 16 | github.com/panjf2000/ants/v2 v2.11.3 17 | github.com/qiniu/go-sdk/v7 v7.25.5 18 | github.com/restic/chunker v0.4.0 19 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 20 | github.com/shirou/gopsutil/v4 v4.25.11 21 | github.com/siyuan-note/dataparser v0.0.0-20251203120213-59c16535cb56 22 | github.com/siyuan-note/encryption v0.0.0-20251120032857-3ddc3c2cc49f 23 | github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 24 | github.com/siyuan-note/filelock v0.0.0-20251212095217-08318833e008 25 | github.com/siyuan-note/httpclient v0.0.0-20251217011734-7f49de93158a 26 | github.com/siyuan-note/logging v0.0.0-20251209020516-52f1a2f65ec5 27 | github.com/studio-b12/gowebdav v0.11.0 28 | github.com/vmihailenco/msgpack/v5 v5.4.1 29 | ) 30 | 31 | require ( 32 | github.com/BurntSushi/toml v1.5.0 // indirect 33 | github.com/alecthomas/chroma v0.10.0 // indirect 34 | github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect 35 | github.com/andybalholm/brotli v1.2.0 // indirect 36 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect 37 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect 38 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect 39 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect 40 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect 41 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect 42 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect 43 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect 44 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect 46 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect 47 | github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect 48 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect 49 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect 50 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect 51 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 52 | github.com/dlclark/regexp2 v1.11.5 // indirect 53 | github.com/dustin/go-humanize v1.0.1 // indirect 54 | github.com/ebitengine/purego v0.9.1 // indirect 55 | github.com/gammazero/toposort v0.1.1 // indirect 56 | github.com/go-ole/go-ole v1.3.0 // indirect 57 | github.com/goccy/go-json v0.10.5 // indirect 58 | github.com/gofrs/flock v0.13.0 // indirect 59 | github.com/google/go-querystring v1.1.0 // indirect 60 | github.com/gopherjs/gopherjs v1.17.2 // indirect 61 | github.com/icholy/digest v1.1.0 // indirect 62 | github.com/imroc/req/v3 v3.57.0 // indirect 63 | github.com/pkg/errors v0.9.1 // indirect 64 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect 65 | github.com/quic-go/qpack v0.6.0 // indirect 66 | github.com/quic-go/quic-go v0.57.1 // indirect 67 | github.com/refraction-networking/utls v1.8.1 // indirect 68 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 69 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 70 | golang.org/x/crypto v0.46.0 // indirect 71 | golang.org/x/net v0.48.0 // indirect 72 | golang.org/x/sync v0.19.0 // indirect 73 | golang.org/x/sys v0.39.0 // indirect 74 | golang.org/x/text v0.32.0 // indirect 75 | modernc.org/fileutil v1.3.40 // indirect 76 | ) 77 | 78 | //replace github.com/siyuan-note/filelock => D:\88250\filelock 79 | -------------------------------------------------------------------------------- /diff.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/siyuan-note/dejavu/entity" 23 | "github.com/siyuan-note/logging" 24 | ) 25 | 26 | // diffUpsertRemove 比较 left 多于/变动 right 的文件以及 left 少于 right 的文件。 27 | func (repo *Repo) diffUpsertRemove(left, right []*entity.File, log bool) (upserts, removes []*entity.File) { 28 | l := map[string]*entity.File{} 29 | r := map[string]*entity.File{} 30 | for _, f := range left { 31 | l[f.Path] = f 32 | } 33 | for _, f := range right { 34 | r[f.Path] = f 35 | } 36 | 37 | for lPath, lFile := range l { 38 | rFile := r[lPath] 39 | if nil == rFile { 40 | upserts = append(upserts, l[lPath]) 41 | if log { 42 | logging.LogInfof("upsert [%s, %s, %s]", l[lPath].ID, l[lPath].Path, time.UnixMilli(l[lPath].Updated).Format("2006-01-02 15:04:05")) 43 | } 44 | 45 | continue 46 | } 47 | if !equalFile(lFile, rFile) { 48 | if log { 49 | logging.LogInfof("upsert [lID=%s, lPath=%s, lUpdated=%s, rID=%s, rPath=%s, rUpdated=%s]", 50 | l[lPath].ID, l[lPath].Path, time.UnixMilli(l[lPath].Updated).Format("2006-01-02 15:04:05"), 51 | rFile.ID, rFile.Path, time.UnixMilli(rFile.Updated).Format("2006-01-02 15:04:05")) 52 | } 53 | upserts = append(upserts, l[lPath]) 54 | continue 55 | } 56 | } 57 | 58 | for rPath := range r { 59 | lFile := l[rPath] 60 | if nil == lFile { 61 | removes = append(removes, r[rPath]) 62 | if log { 63 | logging.LogInfof("remove [%s, %s, %s]", r[rPath].ID, r[rPath].Path, time.UnixMilli(r[rPath].Updated).Format("2006-01-02 15:04:05")) 64 | } 65 | continue 66 | } 67 | } 68 | return 69 | } 70 | 71 | type LeftRightDiff struct { 72 | LeftIndex *entity.Index 73 | RightIndex *entity.Index 74 | AddsLeft []*entity.File 75 | UpdatesLeft []*entity.File 76 | UpdatesRight []*entity.File 77 | RemovesRight []*entity.File 78 | } 79 | 80 | // DiffIndex 返回索引 left 比索引 right 新增、更新和删除的文件列表。 81 | func (repo *Repo) DiffIndex(leftIndexID, rightIndexID string) (ret *LeftRightDiff, err error) { 82 | leftIndex, err := repo.GetIndex(leftIndexID) 83 | if nil != err { 84 | return 85 | } 86 | rightIndex, err := repo.GetIndex(rightIndexID) 87 | if nil != err { 88 | return 89 | } 90 | 91 | ret, err = repo.diffIndex(leftIndex, rightIndex) 92 | return 93 | } 94 | 95 | func (repo *Repo) diffIndex(leftIndex, rightIndex *entity.Index) (ret *LeftRightDiff, err error) { 96 | leftFiles, err := repo.getFiles(leftIndex.Files) 97 | if nil != err { 98 | return 99 | } 100 | rightFiles, err := repo.getFiles(rightIndex.Files) 101 | if nil != err { 102 | return 103 | } 104 | 105 | l := map[string]*entity.File{} 106 | r := map[string]*entity.File{} 107 | for _, f := range leftFiles { 108 | l[f.Path] = f 109 | } 110 | for _, f := range rightFiles { 111 | r[f.Path] = f 112 | } 113 | 114 | ret = &LeftRightDiff{ 115 | LeftIndex: leftIndex, 116 | RightIndex: rightIndex, 117 | } 118 | 119 | for lPath, lFile := range l { 120 | rFile := r[lPath] 121 | if nil == rFile { 122 | ret.AddsLeft = append(ret.AddsLeft, l[lPath]) 123 | continue 124 | } 125 | if !equalFile(lFile, rFile) { 126 | ret.UpdatesLeft = append(ret.UpdatesLeft, l[lPath]) 127 | ret.UpdatesRight = append(ret.UpdatesRight, r[lPath]) 128 | continue 129 | } 130 | } 131 | 132 | for rPath := range r { 133 | lFile := l[rPath] 134 | if nil == lFile { 135 | ret.RemovesRight = append(ret.RemovesRight, r[rPath]) 136 | continue 137 | } 138 | } 139 | return 140 | } 141 | 142 | func equalFile(left, right *entity.File) bool { 143 | if left.Path != right.Path { 144 | return false 145 | } 146 | if left.Updated/1000 != right.Updated/1000 { // Improve data sync file timestamp comparison https://github.com/siyuan-note/siyuan/issues/8573 147 | return false 148 | } 149 | return true 150 | } 151 | -------------------------------------------------------------------------------- /ref.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "errors" 21 | "os" 22 | "path/filepath" 23 | "time" 24 | 25 | "github.com/88250/go-humanize" 26 | "github.com/88250/gulu" 27 | "github.com/siyuan-note/dejavu/entity" 28 | "github.com/siyuan-note/filelock" 29 | "github.com/siyuan-note/logging" 30 | "github.com/vmihailenco/msgpack/v5" 31 | ) 32 | 33 | var ErrNotFoundIndex = errors.New("not found index") 34 | 35 | func (repo *Repo) Latest() (ret *entity.Index, err error) { 36 | latest := filepath.Join(repo.Path, "refs", "latest") 37 | if !filelock.IsExist(latest) { 38 | err = ErrNotFoundIndex 39 | return 40 | } 41 | 42 | data, err := os.ReadFile(latest) 43 | if nil != err { 44 | logging.LogErrorf("read latest index [%s] failed: %s", latest, err) 45 | return 46 | } 47 | hash := string(data) 48 | ret, err = repo.store.GetIndex(hash) 49 | if nil != err { 50 | logging.LogErrorf("get latest index [%s] failed: %s", hash, err) 51 | return 52 | } 53 | //logging.LogInfof("got local latest [%s]", ret.String()) 54 | return 55 | } 56 | 57 | // FullIndex 描述了完整的索引结构。 58 | type FullIndex struct { 59 | ID string `json:"id"` 60 | Files []*entity.File `json:"files"` 61 | Spec int `json:"spec"` 62 | } 63 | 64 | func (repo *Repo) UpdateLatest(index *entity.Index) (err error) { 65 | start := time.Now() 66 | 67 | refs := filepath.Join(repo.Path, "refs") 68 | err = os.MkdirAll(refs, 0755) 69 | if nil != err { 70 | return 71 | } 72 | err = gulu.File.WriteFileSafer(filepath.Join(refs, "latest"), []byte(index.ID), 0644) 73 | if nil != err { 74 | return 75 | } 76 | 77 | fullLatestPath := filepath.Join(repo.Path, "full-latest.json") 78 | files, err := repo.GetFiles(index) 79 | if nil != err { 80 | return 81 | } 82 | 83 | fullIndex := &FullIndex{ID: index.ID, Files: files, Spec: 0} 84 | data, err := msgpack.Marshal(fullIndex) 85 | if nil != err { 86 | return 87 | } 88 | err = gulu.File.WriteFileSafer(fullLatestPath, data, 0644) 89 | if nil != err { 90 | return 91 | } 92 | 93 | logging.LogInfof("updated local latest to [%s], full latest [size=%s], cost [%s]", index.String(), humanize.Bytes(uint64(len(data))), time.Since(start)) 94 | return 95 | } 96 | 97 | func (repo *Repo) getFullLatest(latest *entity.Index) (ret *FullIndex) { 98 | start := time.Now() 99 | 100 | fullLatestPath := filepath.Join(repo.Path, "full-latest.json") 101 | if !gulu.File.IsExist(fullLatestPath) { 102 | return 103 | } 104 | 105 | data, err := os.ReadFile(fullLatestPath) 106 | if nil != err { 107 | logging.LogErrorf("read full latest failed: %s", err) 108 | return 109 | } 110 | 111 | ret = &FullIndex{} 112 | if err = msgpack.Unmarshal(data, ret); nil != err { 113 | logging.LogErrorf("unmarshal full latest [%s] failed: %s", fullLatestPath, err) 114 | ret = nil 115 | if err = os.RemoveAll(fullLatestPath); nil != err { 116 | logging.LogErrorf("remove full latest [%s] failed: %s", fullLatestPath, err) 117 | } 118 | return 119 | } 120 | 121 | if ret.ID != latest.ID { 122 | logging.LogErrorf("full latest ID [%s] not match latest ID [%s]", ret.ID, latest.ID) 123 | ret = nil 124 | if err = os.RemoveAll(fullLatestPath); nil != err { 125 | logging.LogErrorf("remove full latest [%s] failed: %s", fullLatestPath, err) 126 | } 127 | return 128 | } 129 | 130 | for _, f := range ret.Files { 131 | repo.store.cacheFile(f) 132 | } 133 | 134 | logging.LogInfof("got local full latest [files=%d, size=%s], cost [%s]", len(ret.Files), humanize.Bytes(uint64(len(data))), time.Since(start)) 135 | return 136 | } 137 | 138 | func (repo *Repo) GetTag(tag string) (id string, err error) { 139 | if !gulu.File.IsValidFilename(tag) { 140 | err = errors.New("invalid tag name") 141 | } 142 | tag = filepath.Join(repo.Path, "refs", "tags", tag) 143 | if !filelock.IsExist(tag) { 144 | err = errors.New("tag not found") 145 | } 146 | data, err := filelock.ReadFile(tag) 147 | if nil != err { 148 | return 149 | } 150 | id = string(data) 151 | return 152 | } 153 | 154 | func (repo *Repo) AddTag(id, tag string) (err error) { 155 | if !gulu.File.IsValidFilename(tag) { 156 | return errors.New("invalid tag name") 157 | } 158 | 159 | _, err = repo.store.GetIndex(id) 160 | if nil != err { 161 | return 162 | } 163 | 164 | tags := filepath.Join(repo.Path, "refs", "tags") 165 | if err = os.MkdirAll(tags, 0755); nil != err { 166 | return 167 | } 168 | tag = filepath.Join(tags, tag) 169 | err = gulu.File.WriteFileSafer(tag, []byte(id), 0644) 170 | return 171 | } 172 | 173 | func (repo *Repo) RemoveTag(tag string) (err error) { 174 | tag = filepath.Join(repo.Path, "refs", "tags", tag) 175 | if !gulu.File.IsExist(tag) { 176 | return 177 | } 178 | 179 | err = os.Remove(tag) 180 | return 181 | } 182 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "os" 21 | "path/filepath" 22 | "sort" 23 | "time" 24 | 25 | "github.com/88250/go-humanize" 26 | "github.com/88250/gulu" 27 | "github.com/siyuan-note/dejavu/entity" 28 | "github.com/siyuan-note/filelock" 29 | ) 30 | 31 | type Log struct { 32 | ID string `json:"id"` // 索引 ID 33 | Memo string `json:"memo"` // 索引备注 34 | Created int64 `json:"created"` // 索引时间 35 | HCreated string `json:"hCreated"` // 索引时间 "2006-01-02 15:04:05" 36 | Files []*entity.File `json:"files"` // 文件列表 37 | Count int `json:"count"` // 文件总数 38 | Size int64 `json:"size"` // 文件总大小 39 | HSize string `json:"hSize"` // 格式化好的文件总大小 "10.00 MB" 40 | SystemID string `json:"systemID"` // 设备 ID 41 | SystemName string `json:"systemName"` // 设备名称 42 | SystemOS string `json:"systemOS"` // 设备操作系统 43 | Tag string `json:"tag"` // 索引标记名称 44 | HTagUpdated string `json:"hTagUpdated"` // 标记时间 "2006-01-02 15:04:05" 45 | } 46 | 47 | func (log *Log) String() string { 48 | data, err := gulu.JSON.MarshalJSON(log) 49 | if nil != err { 50 | return "print log [" + log.ID + "] failed" 51 | } 52 | return string(data) 53 | } 54 | 55 | func (repo *Repo) GetCloudRepoLogs(page int) (ret []*Log, pageCount, totalCount int, err error) { 56 | cloudIndexes, pageCount, totalCount, err := repo.cloud.GetIndexes(page) 57 | if nil != err { 58 | return 59 | } 60 | 61 | for _, index := range cloudIndexes { 62 | var log *Log 63 | log, err = repo.getLog(index, true) 64 | if nil != err { 65 | return 66 | } 67 | ret = append(ret, log) 68 | } 69 | return 70 | } 71 | 72 | func (repo *Repo) GetCloudRepoTagLogs(context map[string]interface{}) (ret []*Log, err error) { 73 | cloudTags, err := repo.cloud.GetTags() 74 | if nil != err { 75 | return 76 | } 77 | for _, tag := range cloudTags { 78 | index, _ := repo.store.GetIndex(tag.ID) 79 | if nil == index { 80 | _, index, err = repo.downloadCloudIndex(tag.ID, context) 81 | if nil != err { 82 | return 83 | } 84 | } 85 | 86 | var log *Log 87 | log, err = repo.getLog(index, false) 88 | if nil != err { 89 | return 90 | } 91 | log.Tag = tag.Name 92 | log.HTagUpdated = tag.Updated 93 | ret = append(ret, log) 94 | } 95 | sort.Slice(ret, func(i, j int) bool { return ret[i].Created > ret[j].Created }) 96 | return 97 | } 98 | 99 | func (repo *Repo) GetTagLogs() (ret []*Log, err error) { 100 | tags := filepath.Join(repo.Path, "refs", "tags") 101 | if !gulu.File.IsExist(tags) { 102 | return 103 | } 104 | 105 | entries, err := os.ReadDir(tags) 106 | if nil != err { 107 | return 108 | } 109 | for _, entry := range entries { 110 | if entry.IsDir() { 111 | continue 112 | } 113 | var data []byte 114 | name := entry.Name() 115 | data, err = filelock.ReadFile(filepath.Join(tags, name)) 116 | if nil != err { 117 | return 118 | } 119 | info, _ := os.Stat(filepath.Join(tags, name)) 120 | updated := info.ModTime().Format("2006-01-02 15:04:05") 121 | id := string(data) 122 | if 40 != len(id) { 123 | continue 124 | } 125 | var index *entity.Index 126 | index, err = repo.store.GetIndex(id) 127 | if nil != err { 128 | return 129 | } 130 | 131 | var log *Log 132 | log, err = repo.getLog(index, true) 133 | if nil != err { 134 | return 135 | } 136 | log.Tag = name 137 | log.HTagUpdated = updated 138 | ret = append(ret, log) 139 | } 140 | sort.Slice(ret, func(i, j int) bool { return ret[i].Created > ret[j].Created }) 141 | return 142 | } 143 | 144 | func (repo *Repo) GetIndexLogs(page, pageSize int) (ret []*Log, pageCount, totalCount int, err error) { 145 | indexes, totalCount, pageCount, err := repo.GetIndexes(page, pageSize) 146 | if nil != err { 147 | return 148 | } 149 | 150 | for _, index := range indexes { 151 | var log *Log 152 | log, err = repo.getLog(index, true) 153 | if nil != err { 154 | return 155 | } 156 | ret = append(ret, log) 157 | } 158 | return 159 | } 160 | 161 | func (repo *Repo) getLog(index *entity.Index, fetchFiles bool) (ret *Log, err error) { 162 | var files []*entity.File 163 | if fetchFiles { 164 | files, _ = repo.getFiles(index.Files) 165 | } 166 | ret = &Log{ 167 | ID: index.ID, 168 | Memo: index.Memo, 169 | Created: index.Created, 170 | HCreated: time.UnixMilli(index.Created).Format("2006-01-02 15:04:05"), 171 | Files: files, 172 | Count: index.Count, 173 | Size: index.Size, 174 | HSize: humanize.BytesCustomCeil(uint64(index.Size), 2), 175 | SystemID: index.SystemID, 176 | SystemName: index.SystemName, 177 | SystemOS: index.SystemOS, 178 | } 179 | return 180 | } 181 | -------------------------------------------------------------------------------- /sync_lock.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "errors" 21 | "os" 22 | "path/filepath" 23 | "strings" 24 | "time" 25 | 26 | "github.com/88250/gulu" 27 | "github.com/siyuan-note/dejavu/cloud" 28 | "github.com/siyuan-note/eventbus" 29 | "github.com/siyuan-note/logging" 30 | ) 31 | 32 | var ( 33 | ErrLockCloudFailed = errors.New("lock cloud repo failed") 34 | ErrCloudLocked = errors.New("cloud repo is locked") 35 | ) 36 | 37 | const ( 38 | lockSyncKey = "lock-sync" 39 | ) 40 | 41 | func (repo *Repo) unlockCloud(context map[string]interface{}) { 42 | endRefreshLock <- true 43 | var err error 44 | for i := 0; i < 3; i++ { 45 | eventbus.Publish(eventbus.EvtCloudUnlock, context) 46 | err = repo.cloud.RemoveObject(lockSyncKey) 47 | if nil == err { 48 | return 49 | } 50 | } 51 | 52 | if errors.Is(err, cloud.ErrCloudAuthFailed) { 53 | return 54 | } 55 | 56 | logging.LogErrorf("unlock cloud repo failed: %s", err) 57 | return 58 | } 59 | 60 | var endRefreshLock = make(chan bool) 61 | 62 | func (repo *Repo) tryLockCloud(currentDeviceID string, context map[string]interface{}) (err error) { 63 | for i := 0; i < 3; i++ { 64 | err = repo.lockCloud(currentDeviceID, context) 65 | if nil != err { 66 | if errors.Is(err, ErrCloudLocked) { 67 | logging.LogInfof("cloud repo is locked, retry after 5s") 68 | time.Sleep(5 * time.Second) 69 | continue 70 | } 71 | return 72 | } 73 | 74 | // 锁定成功，定时刷新锁 75 | go func() { 76 | ticker := time.NewTicker(30 * time.Second) 77 | defer ticker.Stop() 78 | for { 79 | select { 80 | case <-endRefreshLock: 81 | return 82 | case <-ticker.C: 83 | if refershErr := repo.lockCloud0(currentDeviceID); nil != refershErr { 84 | logging.LogErrorf("refresh cloud repo lock failed: %s", refershErr) 85 | } 86 | } 87 | } 88 | }() 89 | 90 | return 91 | } 92 | return 93 | } 94 | 95 | // lockCloud 锁定云端仓库，不要单独调用，应该调用 tryLockCloud，否则解锁时 endRefreshLock 会阻塞。 96 | func (repo *Repo) lockCloud(currentDeviceID string, context map[string]interface{}) (err error) { 97 | eventbus.Publish(eventbus.EvtCloudLock, context) 98 | data, err := repo.cloud.DownloadObject(lockSyncKey) 99 | if errors.Is(err, cloud.ErrCloudObjectNotFound) { 100 | err = repo.lockCloud0(currentDeviceID) 101 | return 102 | } 103 | 104 | content := map[string]interface{}{} 105 | err = gulu.JSON.UnmarshalJSON(data, &content) 106 | if nil != err { 107 | logging.LogErrorf("unmarshal lock sync failed: %s", err) 108 | err = repo.cloud.RemoveObject(lockSyncKey) 109 | if nil != err { 110 | logging.LogErrorf("remove unmarshalled lock sync failed: %s", err) 111 | } else { 112 | err = repo.lockCloud0(currentDeviceID) 113 | } 114 | 115 | if ok, retErr := parseErr(err); ok { 116 | return retErr 117 | } 118 | return 119 | } 120 | 121 | deviceID := content["deviceID"].(string) 122 | t := int64(content["time"].(float64)) 123 | now := time.Now() 124 | lockTime := time.UnixMilli(t) 125 | if now.After(lockTime.Add(65*time.Second)) || deviceID == currentDeviceID { 126 | // 云端锁超时过期或者就是当前设备锁的，那么当前设备可以继续直接锁 127 | err = repo.lockCloud0(currentDeviceID) 128 | return 129 | } 130 | 131 | logging.LogWarnf("cloud repo is locked by device [%s] at [%s], will retry after 30s", content["deviceID"].(string), lockTime.Format("2006-01-02 15:04:05")) 132 | err = ErrCloudLocked 133 | return 134 | } 135 | 136 | func (repo *Repo) lockCloud0(currentDeviceID string) (err error) { 137 | if !gulu.File.IsDir(repo.Path) { 138 | if err = os.MkdirAll(repo.Path, 0755); nil != err { 139 | logging.LogErrorf("create repo dir failed: %s", err) 140 | return 141 | } 142 | } 143 | 144 | lockSyncPath := filepath.Join(repo.Path, lockSyncKey) 145 | content := map[string]interface{}{ 146 | "deviceID": currentDeviceID, 147 | "time": time.Now().UnixMilli(), 148 | } 149 | data, err := gulu.JSON.MarshalJSON(content) 150 | if nil != err { 151 | logging.LogErrorf("marshal lock sync failed: %s", err) 152 | err = ErrLockCloudFailed 153 | return 154 | } 155 | err = gulu.File.WriteFileSafer(lockSyncPath, data, 0644) 156 | if nil != err { 157 | logging.LogErrorf("write lock sync failed: %s", err) 158 | err = ErrCloudLocked 159 | return 160 | } 161 | 162 | _, err = repo.cloud.UploadObject(lockSyncKey, true) 163 | if nil != err { 164 | if errors.Is(err, cloud.ErrSystemTimeIncorrect) || errors.Is(err, cloud.ErrCloudAuthFailed) || errors.Is(err, cloud.ErrDeprecatedVersion) || 165 | errors.Is(err, cloud.ErrCloudCheckFailed) { 166 | return 167 | } 168 | 169 | logging.LogErrorf("upload lock sync failed: %s", err) 170 | if ok, retErr := parseErr(err); ok { 171 | return retErr 172 | } 173 | 174 | err = ErrLockCloudFailed 175 | return 176 | } 177 | return 178 | } 179 | 180 | func parseErr(err error) (bool, error) { 181 | if nil == err { 182 | return true, nil 183 | } 184 | 185 | msg := strings.ToLower(err.Error()) 186 | if strings.Contains(msg, "requesttimetooskewed") || strings.Contains(msg, "request time and the current time is too large") { 187 | return true, cloud.ErrSystemTimeIncorrect 188 | } else if strings.Contains(msg, "500") || strings.Contains(msg, "internal server error") || strings.Contains(msg, "503") || strings.Contains(msg, "unavailable") { 189 | return true, cloud.ErrCloudServiceUnavailable 190 | } else if strings.Contains(msg, "401") || strings.Contains(msg, "unauthorized") || 191 | strings.Contains(msg, "403") || strings.Contains(msg, "forbidden") { 192 | return true, cloud.ErrCloudForbidden 193 | } else if strings.Contains(msg, "429") || strings.Contains(msg, "too many requests") { 194 | return true, cloud.ErrCloudTooManyRequests 195 | } 196 | return false, err 197 | } 198 | -------------------------------------------------------------------------------- /repo_test.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "errors" 21 | "os" 22 | "path/filepath" 23 | "runtime" 24 | "testing" 25 | 26 | "github.com/88250/gulu" 27 | "github.com/siyuan-note/dejavu/entity" 28 | "github.com/siyuan-note/encryption" 29 | "github.com/siyuan-note/eventbus" 30 | ) 31 | 32 | const ( 33 | testRepoPassword = "pass" 34 | testRepoPasswordSalt = "salt" 35 | testRepoPath = "testdata/repo" 36 | testHistoryPath = "testdata/history" 37 | testTempPath = "testdata/temp" 38 | testDataPath = "testdata/data" 39 | testDataCheckoutPath = "testdata/data-checkout" 40 | ) 41 | 42 | var ( 43 | deviceID = "device-id-0" 44 | deviceName, _ = os.Hostname() 45 | deviceOS = runtime.GOOS 46 | ) 47 | 48 | func TestIndexEmpty(t *testing.T) { 49 | clearTestdata(t) 50 | subscribeEvents(t) 51 | 52 | aesKey, err := encryption.KDF(testRepoPassword, testRepoPasswordSalt) 53 | if nil != err { 54 | return 55 | } 56 | 57 | testEmptyDataPath := "testdata/empty-data" 58 | if err = os.MkdirAll(testEmptyDataPath, 0755); nil != err { 59 | t.Fatalf("mkdir failed: %s", err) 60 | return 61 | } 62 | repo, err := NewRepo(testEmptyDataPath, testRepoPath, testHistoryPath, testTempPath, deviceID, deviceName, deviceOS, aesKey, ignoreLines(), nil) 63 | if nil != err { 64 | t.Fatalf("new repo failed: %s", err) 65 | return 66 | } 67 | _, err = repo.Index("Index 1", true, map[string]interface{}{}) 68 | if !errors.Is(err, ErrEmptyIndex) { 69 | t.Fatalf("should be empty index") 70 | return 71 | } 72 | } 73 | 74 | func TestPurge(t *testing.T) { 75 | clearTestdata(t) 76 | subscribeEvents(t) 77 | 78 | repo, _ := initIndex(t) 79 | stat, err := repo.Purge() 80 | if nil != err { 81 | t.Fatalf("purge failed: %s", err) 82 | return 83 | } 84 | 85 | t.Logf("purge stat: %#v", stat) 86 | } 87 | 88 | func TestIndexCheckout(t *testing.T) { 89 | clearTestdata(t) 90 | subscribeEvents(t) 91 | 92 | repo, index := initIndex(t) 93 | index2, err := repo.Index("Index 2", true, map[string]interface{}{}) 94 | if nil != err { 95 | t.Fatalf("index failed: %s", err) 96 | return 97 | } 98 | if index.ID != index2.ID { 99 | t.Fatalf("index id not match") 100 | return 101 | } 102 | 103 | aesKey := repo.store.AesKey 104 | repo, err = NewRepo(testDataCheckoutPath, testRepoPath, testHistoryPath, testTempPath, deviceID, deviceName, deviceOS, aesKey, ignoreLines(), nil) 105 | if nil != err { 106 | t.Fatalf("new repo failed: %s", err) 107 | return 108 | } 109 | _, _, err = repo.Checkout(index.ID, map[string]interface{}{}) 110 | if nil != err { 111 | t.Fatalf("checkout failed: %s", err) 112 | return 113 | } 114 | 115 | if !gulu.File.IsExist(filepath.Join(testDataCheckoutPath, "foo")) { 116 | t.Fatalf("checkout failed") 117 | return 118 | } 119 | } 120 | 121 | func clearTestdata(t *testing.T) { 122 | err := os.RemoveAll(testRepoPath) 123 | if nil != err { 124 | t.Fatalf("remove failed: %s", err) 125 | return 126 | } 127 | 128 | err = os.RemoveAll(testDataCheckoutPath) 129 | if nil != err { 130 | t.Fatalf("remove failed: %s", err) 131 | return 132 | } 133 | } 134 | 135 | func subscribeEvents(t *testing.T) { 136 | eventbus.Subscribe(eventbus.EvtIndexBeforeWalkData, func(context map[string]interface{}, path string) { 137 | t.Logf("[%s]: [%s]", eventbus.EvtIndexBeforeWalkData, path) 138 | }) 139 | eventbus.Subscribe(eventbus.EvtIndexWalkData, func(context map[string]interface{}, path string) { 140 | t.Logf("[%s]: [%s]", eventbus.EvtIndexWalkData, path) 141 | }) 142 | eventbus.Subscribe(eventbus.EvtIndexBeforeGetLatestFiles, func(context map[string]interface{}, total int) { 143 | t.Logf("[%s]: [%v/%v]", eventbus.EvtIndexBeforeGetLatestFiles, 0, total) 144 | }) 145 | eventbus.Subscribe(eventbus.EvtIndexGetLatestFile, func(context map[string]interface{}, count int, total int) { 146 | t.Logf("[%s]: [%v/%v]", eventbus.EvtIndexGetLatestFile, count, total) 147 | }) 148 | eventbus.Subscribe(eventbus.EvtIndexUpsertFiles, func(context map[string]interface{}, total int) { 149 | t.Logf("[%s]: [%v/%v]", eventbus.EvtIndexUpsertFiles, 0, total) 150 | }) 151 | eventbus.Subscribe(eventbus.EvtIndexUpsertFile, func(context map[string]interface{}, count int, total int) { 152 | t.Logf("[%s]: [%v/%v]", eventbus.EvtIndexUpsertFile, count, total) 153 | }) 154 | 155 | eventbus.Subscribe(eventbus.EvtCheckoutBeforeWalkData, func(context map[string]interface{}, path string) { 156 | t.Logf("[%s]: [%s]", eventbus.EvtCheckoutBeforeWalkData, path) 157 | }) 158 | eventbus.Subscribe(eventbus.EvtCheckoutWalkData, func(context map[string]interface{}, path string) { 159 | t.Logf("[%s]: [%s]", eventbus.EvtCheckoutWalkData, path) 160 | }) 161 | eventbus.Subscribe(eventbus.EvtCheckoutUpsertFiles, func(context map[string]interface{}, total int) { 162 | t.Logf("[%s]: [%d/%d]", eventbus.EvtCheckoutUpsertFiles, 0, total) 163 | }) 164 | eventbus.Subscribe(eventbus.EvtCheckoutUpsertFile, func(context map[string]interface{}, count, total int) { 165 | t.Logf("[%s]: [%d/%d]", eventbus.EvtCheckoutUpsertFile, count, total) 166 | }) 167 | eventbus.Subscribe(eventbus.EvtCheckoutRemoveFiles, func(context map[string]interface{}, total int) { 168 | t.Logf("[%s]: [%d/%d]", eventbus.EvtCheckoutRemoveFiles, 0, total) 169 | }) 170 | eventbus.Subscribe(eventbus.EvtCheckoutRemoveFile, func(context map[string]interface{}, count, total int) { 171 | t.Logf("[%s]: [%d/%d]", eventbus.EvtCheckoutRemoveFile, count, total) 172 | }) 173 | } 174 | 175 | func initIndex(t *testing.T) (repo *Repo, index *entity.Index) { 176 | aesKey, err := encryption.KDF(testRepoPassword, testRepoPasswordSalt) 177 | if nil != err { 178 | return 179 | } 180 | 181 | repo, err = NewRepo(testDataPath, testRepoPath, testHistoryPath, testTempPath, deviceID, deviceName, deviceOS, aesKey, ignoreLines(), nil) 182 | if nil != err { 183 | t.Fatalf("new repo failed: %s", err) 184 | return 185 | } 186 | index, err = repo.Index("Index 1", true, map[string]interface{}{}) 187 | if nil != err { 188 | t.Fatalf("index failed: %s", err) 189 | return 190 | } 191 | return 192 | } 193 | 194 | func ignoreLines() []string { 195 | return []string{"bar"} 196 | } 197 | -------------------------------------------------------------------------------- /backup.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "github.com/siyuan-note/dejavu/cloud" 21 | "os" 22 | "path" 23 | "strings" 24 | 25 | "github.com/88250/gulu" 26 | "github.com/siyuan-note/dejavu/entity" 27 | "github.com/siyuan-note/logging" 28 | ) 29 | 30 | func (repo *Repo) DownloadIndex(id string, context map[string]interface{}) (downloadFileCount, downloadChunkCount int, downloadBytes int64, err error) { 31 | lock.Lock() 32 | defer lock.Unlock() 33 | 34 | downloadFileCount, downloadChunkCount, downloadBytes, err = repo.downloadIndex(id, context) 35 | return 36 | } 37 | 38 | func (repo *Repo) DownloadTagIndex(tag, id string, context map[string]interface{}) (downloadFileCount, downloadChunkCount int, downloadBytes int64, err error) { 39 | lock.Lock() 40 | defer lock.Unlock() 41 | 42 | downloadFileCount, downloadChunkCount, downloadBytes, err = repo.downloadIndex(id, context) 43 | 44 | // 更新本地标签 45 | err = repo.AddTag(id, tag) 46 | if nil != err { 47 | logging.LogErrorf("add tag failed: %s", err) 48 | return 49 | } 50 | 51 | return 52 | } 53 | 54 | func (repo *Repo) downloadIndex(id string, context map[string]interface{}) (downloadFileCount, downloadChunkCount int, downloadBytes int64, err error) { 55 | // 从云端下载标签指向的索引 56 | length, index, err := repo.downloadCloudIndex(id, context) 57 | if nil != err { 58 | logging.LogErrorf("download cloud index failed: %s", err) 59 | return 60 | } 61 | downloadFileCount++ 62 | downloadBytes += length 63 | apiGet := 1 64 | 65 | // 计算本地缺失的文件 66 | fetchFileIDs, err := repo.localNotFoundFiles(index.Files) 67 | if nil != err { 68 | logging.LogErrorf("get local not found files failed: %s", err) 69 | return 70 | } 71 | 72 | // 从云端下载缺失文件并入库 73 | length, fetchedFiles, err := repo.downloadCloudFilesPut(fetchFileIDs, context) 74 | if nil != err { 75 | logging.LogErrorf("download cloud files put failed: %s", err) 76 | return 77 | } 78 | downloadBytes += length 79 | downloadFileCount = len(fetchFileIDs) 80 | apiGet += downloadFileCount 81 | 82 | // 从文件列表中得到去重后的分块列表 83 | cloudChunkIDs := repo.getChunks(fetchedFiles) 84 | 85 | // 计算本地缺失的分块 86 | fetchChunkIDs, err := repo.localNotFoundChunks(cloudChunkIDs) 87 | if nil != err { 88 | logging.LogErrorf("get local not found chunks failed: %s", err) 89 | return 90 | } 91 | 92 | // 从云端获取分块并入库 93 | length, err = repo.downloadCloudChunksPut(fetchChunkIDs, context) 94 | downloadBytes += length 95 | downloadChunkCount = len(fetchChunkIDs) 96 | apiGet += downloadChunkCount 97 | 98 | // 更新本地索引 99 | err = repo.store.PutIndex(index) 100 | if nil != err { 101 | logging.LogErrorf("put index failed: %s", err) 102 | return 103 | } 104 | 105 | // 统计流量 106 | go repo.cloud.AddTraffic(&cloud.Traffic{DownloadBytes: downloadBytes, APIGet: apiGet}) 107 | return 108 | } 109 | 110 | func (repo *Repo) UploadTagIndex(tag, id string, context map[string]interface{}) (uploadFileCount, uploadChunkCount int, uploadBytes int64, err error) { 111 | lock.Lock() 112 | defer lock.Unlock() 113 | 114 | uploadFileCount, uploadChunkCount, uploadBytes, err = repo.uploadTagIndex(tag, id, context) 115 | if e, ok := err.(*os.PathError); ok && os.IsNotExist(err) { 116 | p := e.Path 117 | if !strings.Contains(p, "objects") { 118 | return 119 | } 120 | 121 | // 索引时正常，但是上传时可能因为外部变更导致对象（文件或者分块）不存在，此时需要告知用户数据仓库已经损坏，需要重置数据仓库 122 | logging.LogErrorf("upload tag index failed: %s", err) 123 | err = ErrRepoFatal 124 | } 125 | return 126 | } 127 | 128 | func (repo *Repo) uploadTagIndex(tag, id string, context map[string]interface{}) (uploadFileCount, uploadChunkCount int, uploadBytes int64, err error) { 129 | index, err := repo.store.GetIndex(id) 130 | if nil != err { 131 | logging.LogErrorf("get index failed: %s", err) 132 | return 133 | } 134 | 135 | availableSize := repo.cloud.GetAvailableSize() 136 | if availableSize <= index.Size { 137 | err = ErrCloudStorageSizeExceeded 138 | return 139 | } 140 | 141 | // 获取云端数据仓库统计信息 142 | cloudRepoSize, cloudBackupCount, err := repo.getCloudRepoStat() 143 | if nil != err { 144 | logging.LogErrorf("get cloud repo stat failed: %s", err) 145 | return 146 | } 147 | if 12 <= cloudBackupCount { 148 | err = ErrCloudBackupCountExceeded 149 | return 150 | } 151 | 152 | if availableSize <= cloudRepoSize+index.Size { 153 | err = ErrCloudStorageSizeExceeded 154 | return 155 | } 156 | 157 | // 从云端获取文件列表 158 | cloudFileIDs, refs, err := repo.cloud.GetRefsFiles() 159 | if nil != err { 160 | logging.LogErrorf("get cloud repo refs files failed: %s", err) 161 | return 162 | } 163 | apiGet := len(refs) + 1 164 | 165 | // 计算云端缺失的文件 166 | var uploadFiles []*entity.File 167 | for _, localFileID := range index.Files { 168 | if !gulu.Str.Contains(localFileID, cloudFileIDs) { 169 | var uploadFile *entity.File 170 | uploadFile, err = repo.store.GetFile(localFileID) 171 | if nil != err { 172 | logging.LogErrorf("get file failed: %s", err) 173 | return 174 | } 175 | uploadFiles = append(uploadFiles, uploadFile) 176 | } 177 | } 178 | 179 | // 从文件列表中得到去重后的分块列表 180 | uploadChunkIDs := repo.getChunks(uploadFiles) 181 | 182 | // 计算云端缺失的分块 183 | uploadChunkIDs, err = repo.cloud.GetChunks(uploadChunkIDs) 184 | if nil != err { 185 | logging.LogErrorf("get cloud repo upload chunks failed: %s", err) 186 | return 187 | } 188 | apiGet += len(uploadChunkIDs) 189 | 190 | // 上传分块 191 | length, err := repo.uploadChunks(uploadChunkIDs, context) 192 | if nil != err { 193 | logging.LogErrorf("upload chunks failed: %s", err) 194 | return 195 | } 196 | uploadChunkCount = len(uploadChunkIDs) 197 | uploadBytes += length 198 | apiPut := uploadChunkCount 199 | 200 | // 上传文件 201 | length, err = repo.uploadFiles(uploadFiles, context) 202 | if nil != err { 203 | logging.LogErrorf("upload files failed: %s", err) 204 | return 205 | } 206 | uploadFileCount = len(uploadFiles) 207 | uploadBytes += length 208 | apiPut += uploadFileCount 209 | 210 | // 上传索引 211 | length, err = repo.uploadIndex(index, context) 212 | uploadFileCount++ 213 | uploadBytes += length 214 | apiPut++ 215 | 216 | // 上传标签 217 | length, err = repo.updateCloudRef("refs/tags/"+tag, context) 218 | uploadFileCount++ 219 | uploadBytes += length 220 | apiPut++ 221 | 222 | // 统计流量 223 | go repo.cloud.AddTraffic(&cloud.Traffic{UploadBytes: uploadBytes, APIGet: apiGet, APIPut: apiPut}) 224 | return 225 | } 226 | 227 | func (repo *Repo) getCloudRepoStat() (repoSize int64, backupCount int, err error) { 228 | repoStat, err := repo.cloud.GetStat() 229 | if nil != err { 230 | return 231 | } 232 | 233 | repoSize = repoStat.Sync.Size + repoStat.Backup.Size 234 | backupCount = repoStat.Backup.Count 235 | return 236 | } 237 | 238 | func (repo *Repo) RemoveCloudRepoTag(tag string) (err error) { 239 | key := path.Join("refs", "tags", tag) 240 | return repo.cloud.RemoveObject(key) 241 | } 242 | -------------------------------------------------------------------------------- /sync_manual.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "errors" 21 | "path/filepath" 22 | "sync" 23 | "time" 24 | 25 | "github.com/88250/gulu" 26 | "github.com/siyuan-note/dejavu/cloud" 27 | "github.com/siyuan-note/dejavu/entity" 28 | "github.com/siyuan-note/logging" 29 | ) 30 | 31 | func (repo *Repo) SyncDownload(context map[string]interface{}) (mergeResult *MergeResult, trafficStat *TrafficStat, err error) { 32 | lock.Lock() 33 | defer lock.Unlock() 34 | 35 | // 锁定云端，防止其他设备并发上传数据 36 | err = repo.tryLockCloud(repo.DeviceID, context) 37 | if nil != err { 38 | return 39 | } 40 | defer repo.unlockCloud(context) 41 | 42 | mergeResult = &MergeResult{Time: time.Now()} 43 | trafficStat = &TrafficStat{m: &sync.Mutex{}} 44 | 45 | // 获取本地最新索引 46 | latest, err := repo.Latest() 47 | if nil != err { 48 | logging.LogErrorf("get latest failed: %s", err) 49 | return 50 | } 51 | 52 | // 从云端获取最新索引 53 | length, cloudLatest, err := repo.downloadCloudLatest(context) 54 | if nil != err { 55 | if !errors.Is(err, cloud.ErrCloudObjectNotFound) { 56 | logging.LogErrorf("download cloud latest failed: %s", err) 57 | return 58 | } 59 | } 60 | trafficStat.DownloadFileCount++ 61 | trafficStat.DownloadBytes += length 62 | trafficStat.APIGet++ 63 | 64 | if cloudLatest.ID == latest.ID || "" == cloudLatest.ID { 65 | // 数据一致或者云端为空，直接返回 66 | return 67 | } 68 | 69 | // 计算本地缺失的文件 70 | fetchFileIDs, err := repo.localNotFoundFiles(cloudLatest.Files) 71 | if nil != err { 72 | logging.LogErrorf("get local not found files failed: %s", err) 73 | return 74 | } 75 | 76 | // 从云端下载缺失文件并入库 77 | length, fetchedFiles, err := repo.downloadCloudFilesPut(fetchFileIDs, context) 78 | if nil != err { 79 | logging.LogErrorf("download cloud files put failed: %s", err) 80 | return 81 | } 82 | trafficStat.DownloadFileCount += len(fetchFileIDs) 83 | trafficStat.DownloadBytes += length 84 | trafficStat.APIGet += trafficStat.DownloadFileCount 85 | 86 | // 组装还原云端最新文件列表 87 | cloudLatestFiles, err := repo.getFiles(cloudLatest.Files) 88 | if nil != err { 89 | logging.LogErrorf("get cloud latest files failed: %s", err) 90 | return 91 | } 92 | 93 | // 从文件列表中得到去重后的分块列表 94 | cloudChunkIDs := repo.getChunks(cloudLatestFiles) 95 | 96 | // 计算本地缺失的分块 97 | fetchChunkIDs, err := repo.localNotFoundChunks(cloudChunkIDs) 98 | if nil != err { 99 | logging.LogErrorf("get local not found chunks failed: %s", err) 100 | return 101 | } 102 | 103 | // 从云端下载缺失分块并入库 104 | length, err = repo.downloadCloudChunksPut(fetchChunkIDs, context) 105 | trafficStat.DownloadBytes += length 106 | trafficStat.DownloadChunkCount += len(fetchChunkIDs) 107 | trafficStat.APIGet += trafficStat.DownloadChunkCount 108 | 109 | // 计算本地相比上一个同步点的 upsert 和 remove 差异 110 | latestFiles, err := repo.getFiles(latest.Files) 111 | if nil != err { 112 | logging.LogErrorf("get latest files failed: %s", err) 113 | return 114 | } 115 | latestSync := repo.latestSync() 116 | latestSyncFiles, err := repo.getFiles(latestSync.Files) 117 | if nil != err { 118 | logging.LogErrorf("get latest sync files failed: %s", err) 119 | return 120 | } 121 | localUpserts, localRemoves := repo.diffUpsertRemove(latestFiles, latestSyncFiles, false) 122 | localChanged := 0 < len(localUpserts) || 0 < len(localRemoves) 123 | 124 | // 计算云端最新相比本地最新的 upsert 和 remove 差异 125 | // 在单向同步的情况下该结果可直接作为合并结果 126 | mergeResult.Upserts, mergeResult.Removes = repo.diffUpsertRemove(cloudLatestFiles, latestFiles, false) 127 | 128 | var fetchedFileIDs []string 129 | for _, fetchedFile := range fetchedFiles { 130 | fetchedFileIDs = append(fetchedFileIDs, fetchedFile.ID) 131 | } 132 | 133 | // 计算冲突的 upsert 134 | // 冲突的文件以云端 upsert 和 remove 为准 135 | for _, localUpsert := range localUpserts { 136 | if nil != repo.getFile(mergeResult.Upserts, localUpsert) || nil != repo.getFile(mergeResult.Removes, localUpsert) { 137 | mergeResult.Conflicts = append(mergeResult.Conflicts, localUpsert) 138 | logging.LogInfof("sync download conflict [%s, %s, %s]", localUpsert.ID, localUpsert.Path, time.UnixMilli(localUpsert.Updated).Format("2006-01-02 15:04:05")) 139 | } 140 | } 141 | 142 | // 冲突文件复制到数据历史文件夹 143 | if 0 < len(mergeResult.Conflicts) { 144 | now := mergeResult.Time.Format("2006-01-02-150405") 145 | temp := filepath.Join(repo.TempPath, "repo", "sync", "conflicts", now) 146 | for i, file := range mergeResult.Conflicts { 147 | var checkoutTmp *entity.File 148 | checkoutTmp, err = repo.store.GetFile(file.ID) 149 | if nil != err { 150 | logging.LogErrorf("get file failed: %s", err) 151 | return 152 | } 153 | 154 | err = repo.checkoutFile(checkoutTmp, temp, i+1, len(mergeResult.Conflicts), context) 155 | if nil != err { 156 | logging.LogErrorf("checkout file failed: %s", err) 157 | return 158 | } 159 | 160 | absPath := filepath.Join(temp, checkoutTmp.Path) 161 | err = repo.genSyncHistory(now, file.Path, absPath) 162 | if nil != err { 163 | logging.LogErrorf("generate sync history failed: %s", err) 164 | err = ErrCloudGenerateConflictHistory 165 | return 166 | } 167 | } 168 | } 169 | 170 | // 数据变更后还原文件 171 | err = repo.restoreFiles(mergeResult, context) 172 | if nil != err { 173 | logging.LogErrorf("restore files failed: %s", err) 174 | return 175 | } 176 | 177 | // 处理合并 178 | err = repo.mergeSync(mergeResult, localChanged, false, latest, cloudLatest, cloudChunkIDs, trafficStat, context) 179 | if nil != err { 180 | logging.LogErrorf("merge sync failed: %s", err) 181 | return 182 | } 183 | 184 | // 统计流量 185 | go repo.cloud.AddTraffic(&cloud.Traffic{ 186 | DownloadBytes: trafficStat.DownloadBytes, 187 | APIGet: trafficStat.APIGet, 188 | }) 189 | 190 | // 移除空目录 191 | gulu.File.RemoveEmptyDirs(repo.DataPath, removeEmptyDirExcludes...) 192 | return 193 | } 194 | 195 | func (repo *Repo) SyncUpload(context map[string]interface{}) (trafficStat *TrafficStat, err error) { 196 | lock.Lock() 197 | defer lock.Unlock() 198 | 199 | // 锁定云端，防止其他设备并发上传数据 200 | err = repo.tryLockCloud(repo.DeviceID, context) 201 | if nil != err { 202 | return 203 | } 204 | defer repo.unlockCloud(context) 205 | 206 | trafficStat = &TrafficStat{m: &sync.Mutex{}} 207 | 208 | latest, err := repo.Latest() 209 | if nil != err { 210 | logging.LogErrorf("get latest failed: %s", err) 211 | return 212 | } 213 | 214 | // 从云端获取最新索引 215 | length, cloudLatest, err := repo.downloadCloudLatest(context) 216 | if nil != err { 217 | if !errors.Is(err, cloud.ErrCloudObjectNotFound) { 218 | logging.LogErrorf("download cloud latest failed: %s", err) 219 | return 220 | } 221 | } 222 | trafficStat.DownloadFileCount++ 223 | trafficStat.DownloadBytes += length 224 | trafficStat.APIPut++ 225 | 226 | if cloudLatest.ID == latest.ID { 227 | // 数据一致，直接返回 228 | return 229 | } 230 | 231 | availableSize := repo.cloud.GetAvailableSize() 232 | if availableSize <= cloudLatest.Size || availableSize <= latest.Size { 233 | err = ErrCloudStorageSizeExceeded 234 | return 235 | } 236 | 237 | // 计算云端缺失的文件 238 | var uploadFiles []*entity.File 239 | for _, localFileID := range latest.Files { 240 | if !gulu.Str.Contains(localFileID, cloudLatest.Files) { 241 | var uploadFile *entity.File 242 | uploadFile, err = repo.store.GetFile(localFileID) 243 | if nil != err { 244 | logging.LogErrorf("get file failed: %s", err) 245 | return 246 | } 247 | uploadFiles = append(uploadFiles, uploadFile) 248 | } 249 | } 250 | 251 | // 从文件列表中得到去重后的分块列表 252 | uploadChunkIDs := repo.getChunks(uploadFiles) 253 | 254 | // 这里暂时不计算云端缺失的分块了，因为目前计数云端缺失分块的代价太大 255 | //uploadChunkIDs, err = repo.cloud.GetChunks(uploadChunkIDs) 256 | //if nil != err { 257 | // logging.LogErrorf("get cloud repo upload chunks failed: %s", err) 258 | // return 259 | //} 260 | 261 | // 上传分块 262 | length, err = repo.uploadChunks(uploadChunkIDs, context) 263 | if nil != err { 264 | logging.LogErrorf("upload chunks failed: %s", err) 265 | return 266 | } 267 | trafficStat.UploadChunkCount += len(uploadChunkIDs) 268 | trafficStat.UploadBytes += length 269 | trafficStat.APIPut += trafficStat.UploadChunkCount 270 | 271 | // 上传文件 272 | length, err = repo.uploadFiles(uploadFiles, context) 273 | if nil != err { 274 | logging.LogErrorf("upload files failed: %s", err) 275 | return 276 | } 277 | trafficStat.UploadChunkCount += len(uploadFiles) 278 | trafficStat.UploadBytes += length 279 | trafficStat.APIPut += trafficStat.UploadChunkCount 280 | 281 | // 更新云端索引信息 282 | err = repo.updateCloudIndexes(latest, trafficStat, context) 283 | if nil != err { 284 | logging.LogErrorf("update cloud indexes failed: %s", err) 285 | return 286 | } 287 | 288 | // 更新本地同步点 289 | err = repo.UpdateLatestSync(latest) 290 | if nil != err { 291 | logging.LogErrorf("update latest sync failed: %s", err) 292 | return 293 | } 294 | 295 | // 统计流量 296 | go repo.cloud.AddTraffic(&cloud.Traffic{ 297 | UploadBytes: trafficStat.UploadBytes, 298 | APIPut: trafficStat.APIPut, 299 | }) 300 | return 301 | } 302 | -------------------------------------------------------------------------------- /cloud/cloud.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package cloud 18 | 19 | import ( 20 | "errors" 21 | "strings" 22 | 23 | "github.com/dgraph-io/ristretto" 24 | "github.com/klauspost/compress/zstd" 25 | "github.com/siyuan-note/dejavu/entity" 26 | ) 27 | 28 | // Conf 用于描述云端存储服务配置信息。 29 | type Conf struct { 30 | Dir string // 存储目录，第三方存储不使用 Dir 区别多租户 31 | UserID string // 用户 ID，没有的话请传入一个定值比如 "0" 32 | RepoPath string // 本地仓库的绝对路径，如：F:\\SiYuan\\repo\\ 33 | Endpoint string // 服务端点 34 | Extras map[string]interface{} // 一些可能需要的附加信息 35 | 36 | // S3 对象存储协议所需配置 37 | S3 *ConfS3 38 | 39 | // WebDAV 协议所需配置 40 | WebDAV *ConfWebDAV 41 | 42 | // 本地存储服务配置 43 | Local *ConfLocal 44 | 45 | // 以下值非官方存储服务不必传入 46 | Token string // 云端接口鉴权令牌 47 | AvailableSize int64 // 云端存储可用空间字节数 48 | Server string // 云端接口端点 49 | } 50 | 51 | // ConfS3 用于描述 S3 对象存储协议所需配置。 52 | type ConfS3 struct { 53 | Endpoint string // 服务端点 54 | AccessKey string // Access Key 55 | SecretKey string // Secret Key 56 | Region string // 存储区域 57 | Bucket string // 存储空间 58 | PathStyle bool // 是否使用路径风格寻址 59 | SkipTlsVerify bool // 是否跳过 TLS 验证 60 | Timeout int // 超时时间，单位：秒 61 | ConcurrentReqs int // 并发请求数 62 | } 63 | 64 | // ConfWebDAV 用于描述 WebDAV 协议所需配置。 65 | type ConfWebDAV struct { 66 | Endpoint string // 服务端点 67 | Username string // 用户名 68 | Password string // 密码 69 | SkipTlsVerify bool // 是否跳过 TLS 验证 70 | Timeout int // 超时时间，单位：秒 71 | ConcurrentReqs int // 并发请求数 72 | } 73 | 74 | // ConfLocal 用于描述本地存储服务配置信息。 75 | type ConfLocal struct { 76 | // 服务端点 (本地文件系统目录) 77 | // 78 | // "D:/path/to/repos/directory" // Windows 79 | // "/path/to/repos/directory" // Unix 80 | Endpoint string 81 | Timeout int // 超时时间，单位：秒 82 | ConcurrentReqs int // 并发请求数 83 | } 84 | 85 | // Cloud 描述了云端存储服务，接入云端存储服务时需要实现该接口。 86 | type Cloud interface { 87 | 88 | // CreateRepo 用于创建名称为 name 的云端仓库。 89 | CreateRepo(name string) (err error) 90 | 91 | // RemoveRepo 用于删除云端仓库。 92 | RemoveRepo(name string) (err error) 93 | 94 | // GetRepos 用于获取云端仓库列表 repos，size 为仓库总大小字节数。 95 | GetRepos() (repos []*Repo, size int64, err error) 96 | 97 | // UploadObject 用于上传对象，overwrite 参数用于指示是否覆盖已有对象。 98 | UploadObject(filePath string, overwrite bool) (length int64, err error) 99 | 100 | // UploadBytes 用于上传对象数据 data，overwrite 参数用于指示是否覆盖已有对象。 101 | UploadBytes(filePath string, data []byte, overwrite bool) (length int64, err error) 102 | 103 | // DownloadObject 用于下载对象数据 data。 104 | DownloadObject(filePath string) (data []byte, err error) 105 | 106 | // RemoveObject 用于删除对象。 107 | RemoveObject(filePath string) (err error) 108 | 109 | // GetTags 用于获取快照标记列表。 110 | GetTags() (tags []*Ref, err error) 111 | 112 | // GetIndexes 用于获取索引列表。 113 | GetIndexes(page int) (indexes []*entity.Index, pageCount, totalCount int, err error) 114 | 115 | // GetRefsFiles 用于获取所有引用索引中的文件 ID 列表 fileIDs。 116 | GetRefsFiles() (fileIDs []string, refs []*Ref, err error) 117 | 118 | // GetChunks 用于获取 checkChunkIDs 中不存在的分块 ID 列表 chunkIDs。 119 | GetChunks(checkChunkIDs []string) (chunkIDs []string, err error) 120 | 121 | // GetStat 用于获取统计信息 stat。 122 | GetStat() (stat *Stat, err error) 123 | 124 | // GetConf 用于获取配置信息。 125 | GetConf() *Conf 126 | 127 | // GetAvailableSize 用于获取云端存储可用空间字节数。 128 | GetAvailableSize() (size int64) 129 | 130 | // AddTraffic 用于统计流量。 131 | AddTraffic(traffic *Traffic) 132 | 133 | // ListObjects 用于列出指定前缀的对象。 134 | ListObjects(pathPrefix string) (objInfos map[string]*entity.ObjectInfo, err error) 135 | 136 | // GetIndex 用于获取索引。 137 | GetIndex(id string) (index *entity.Index, err error) 138 | 139 | // GetConcurrentReqs 用于获取配置的并发请求数。 140 | GetConcurrentReqs() int 141 | } 142 | 143 | // Traffic 描述了流量信息。 144 | type Traffic struct { 145 | UploadBytes int64 // 上传字节数 146 | DownloadBytes int64 // 下载字节数 147 | APIGet int // API GET 请求次数 148 | APIPut int // API PUT 请求次数 149 | } 150 | 151 | // Stat 描述了统计信息。 152 | type Stat struct { 153 | Sync *StatSync `json:"sync"` // 同步统计 154 | Backup *StatBackup `json:"backup"` // 备份统计 155 | AssetSize int64 `json:"assetSize"` // 资源文件大小字节数 156 | RepoCount int `json:"repoCount"` // 仓库数量 157 | } 158 | 159 | // Repo 描述了云端仓库。 160 | type Repo struct { 161 | Name string `json:"name"` 162 | Size int64 `json:"size"` 163 | Updated string `json:"updated"` 164 | } 165 | 166 | // Ref 描述了快照引用。 167 | type Ref struct { 168 | Name string `json:"name"` // 引用文件名称，比如 latest、tag1 169 | ID string `json:"id"` // 引用 ID 170 | Updated string `json:"updated"` // 最近更新时间 171 | } 172 | 173 | // StatSync 描述了同步统计信息。 174 | type StatSync struct { 175 | Size int64 `json:"size"` // 总大小字节数 176 | FileCount int `json:"fileCount"` // 总文件数 177 | Updated string `json:"updated"` // 最近更新时间 178 | } 179 | 180 | // StatBackup 描述了备份统计信息。 181 | type StatBackup struct { 182 | Count int `json:"count"` // 已标记的快照数量 183 | Size int64 `json:"size"` // 总大小字节数 184 | FileCount int `json:"fileCount"` // 总文件数 185 | Updated string `json:"updated"` // 最近更新时间 186 | } 187 | 188 | // Indexes 描述了云端索引列表。 189 | type Indexes struct { 190 | Indexes []*Index `json:"indexes"` 191 | } 192 | 193 | // Index 描述了云端索引。 194 | type Index struct { 195 | ID string `json:"id"` 196 | SystemID string `json:"systemID"` 197 | SystemName string `json:"systemName"` 198 | SystemOS string `json:"systemOS"` 199 | } 200 | 201 | // BaseCloud 描述了云端存储服务的基础实现。 202 | type BaseCloud struct { 203 | *Conf 204 | Cloud 205 | } 206 | 207 | func (baseCloud *BaseCloud) CreateRepo(name string) (err error) { 208 | err = ErrUnsupported 209 | return 210 | } 211 | 212 | func (baseCloud *BaseCloud) RemoveRepo(name string) (err error) { 213 | err = ErrUnsupported 214 | return 215 | } 216 | 217 | func (baseCloud *BaseCloud) GetRepos() (repos []*Repo, size int64, err error) { 218 | err = ErrUnsupported 219 | return 220 | } 221 | 222 | func (baseCloud *BaseCloud) UploadObject(filePath string, overwrite bool) (length int64, err error) { 223 | err = ErrUnsupported 224 | return 225 | } 226 | 227 | func (baseCloud *BaseCloud) UploadBytes(filePath string, data []byte, overwrite bool) (length int64, err error) { 228 | err = ErrUnsupported 229 | return 230 | } 231 | 232 | func (baseCloud *BaseCloud) DownloadObject(filePath string) (data []byte, err error) { 233 | err = ErrUnsupported 234 | return 235 | } 236 | 237 | func (baseCloud *BaseCloud) RemoveObject(filePath string) (err error) { 238 | err = ErrUnsupported 239 | return 240 | } 241 | 242 | func (baseCloud *BaseCloud) GetTags() (tags []*Ref, err error) { 243 | err = ErrUnsupported 244 | return 245 | } 246 | 247 | func (baseCloud *BaseCloud) GetIndexes(page int) (indexes []*entity.Index, pageCount, totalCount int, err error) { 248 | err = ErrUnsupported 249 | return 250 | } 251 | 252 | func (baseCloud *BaseCloud) GetRefsFiles() (fileIDs []string, refs []*Ref, err error) { 253 | err = ErrUnsupported 254 | return 255 | } 256 | 257 | func (baseCloud *BaseCloud) GetChunks(checkChunkIDs []string) (chunkIDs []string, err error) { 258 | err = ErrUnsupported 259 | return 260 | } 261 | 262 | func (baseCloud *BaseCloud) GetStat() (stat *Stat, err error) { 263 | stat = &Stat{ 264 | Sync: &StatSync{}, 265 | Backup: &StatBackup{}, 266 | } 267 | return 268 | } 269 | 270 | func (baseCloud *BaseCloud) ListObjects(pathPrefix string) (objInfos map[string]*entity.ObjectInfo, err error) { 271 | err = ErrUnsupported 272 | return 273 | } 274 | 275 | func (baseCloud *BaseCloud) GetIndex(id string) (index *entity.Index, err error) { 276 | err = ErrUnsupported 277 | return 278 | } 279 | 280 | func (baseCloud *BaseCloud) GetConcurrentReqs() int { 281 | return 8 282 | } 283 | 284 | func (baseCloud *BaseCloud) GetConf() *Conf { 285 | return baseCloud.Conf 286 | } 287 | 288 | func (baseCloud *BaseCloud) GetAvailableSize() int64 { 289 | return baseCloud.Conf.AvailableSize 290 | } 291 | 292 | func (baseCloud *BaseCloud) AddTraffic(*Traffic) { 293 | return 294 | } 295 | 296 | var ( 297 | ErrUnsupported = errors.New("not supported yet") // ErrUnsupported 描述了尚未支持的操作 298 | ErrCloudObjectNotFound = errors.New("cloud object not found") // ErrCloudObjectNotFound 描述了云端存储服务中的对象不存在的错误 299 | ErrCloudAuthFailed = errors.New("cloud account auth failed") // ErrCloudAuthFailed 描述了云端存储服务鉴权失败的错误 300 | ErrCloudServiceUnavailable = errors.New("cloud service unavailable") // ErrCloudServiceUnavailable 描述了云端存储服务不可用的错误 301 | ErrSystemTimeIncorrect = errors.New("system time incorrect") // ErrSystemTimeIncorrect 描述了系统时间不正确的错误 302 | ErrDeprecatedVersion = errors.New("deprecated version") // ErrDeprecatedVersion 描述了版本过低的错误 303 | ErrCloudCheckFailed = errors.New("cloud check failed") // ErrCloudCheckFailed 描述了云端存储服务检查失败的错误 304 | ErrCloudForbidden = errors.New("cloud forbidden") // ErrCloudForbidden 描述了云端存储服务禁止访问的错误 305 | ErrCloudTooManyRequests = errors.New("cloud too many requests") // ErrCloudTooManyRequests 描述了云端存储服务请求过多的错误 306 | ) 307 | 308 | func IsValidCloudDirName(cloudDirName string) bool { 309 | if 63 < len(cloudDirName) || 1 > len(cloudDirName) { 310 | return false 311 | } 312 | 313 | chars := []byte{'~', '`', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '+', '=', 314 | '[', ']', '{', '}', '\\', '|', ';', ':', '\'', '"', '<', ',', '>', '.', '?', '/', ' '} 315 | var charsStr string 316 | for _, char := range chars { 317 | charsStr += string(char) 318 | } 319 | 320 | if strings.ContainsAny(cloudDirName, charsStr) { 321 | return false 322 | } 323 | 324 | tmp := stripCtlFromUTF8(cloudDirName) 325 | return tmp == cloudDirName 326 | } 327 | 328 | func stripCtlFromUTF8(str string) string { 329 | return strings.Map(func(r rune) rune { 330 | if r >= 32 && r != 127 { 331 | return r 332 | } 333 | return -1 334 | }, str) 335 | } 336 | 337 | var ( 338 | compressDecoder *zstd.Decoder 339 | cache *ristretto.Cache 340 | ) 341 | 342 | func init() { 343 | var err error 344 | compressDecoder, err = zstd.NewReader(nil, zstd.WithDecoderMaxMemory(16*1024*1024*1024)) 345 | if nil != err { 346 | panic(err) 347 | } 348 | 349 | cache, err = ristretto.NewCache(&ristretto.Config{ 350 | NumCounters: 200000, 351 | MaxCost: 1000 * 1000 * 32, 352 | BufferItems: 64, 353 | }) 354 | if nil != err { 355 | panic(err) 356 | } 357 | } 358 | 359 | // objectInfo 描述了对象信息，用于内部处理。 360 | type objectInfo struct { 361 | Key string `json:"key"` 362 | Size int64 `json:"size"` 363 | Updated string `json:"updated"` 364 | } 365 | -------------------------------------------------------------------------------- /cloud/local.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package cloud 18 | 19 | import ( 20 | "math" 21 | "os" 22 | "path" 23 | "path/filepath" 24 | "sort" 25 | "strings" 26 | 27 | "github.com/88250/gulu" 28 | "github.com/siyuan-note/dejavu/entity" 29 | "github.com/siyuan-note/dejavu/util" 30 | "github.com/siyuan-note/logging" 31 | ) 32 | 33 | // Local 描述了本地文件系统服务实现。 34 | type Local struct { 35 | *BaseCloud 36 | } 37 | 38 | func NewLocal(baseCloud *BaseCloud) (local *Local) { 39 | local = &Local{ 40 | BaseCloud: baseCloud, 41 | } 42 | return 43 | } 44 | 45 | func (local *Local) CreateRepo(name string) (err error) { 46 | repoPath := path.Join(local.Local.Endpoint, name) 47 | err = os.MkdirAll(repoPath, 0755) 48 | return 49 | } 50 | 51 | func (local *Local) RemoveRepo(name string) (err error) { 52 | repoPath := path.Join(local.Local.Endpoint, name) 53 | err = os.RemoveAll(repoPath) 54 | return 55 | } 56 | 57 | func (local *Local) GetRepos() (repos []*Repo, size int64, err error) { 58 | repos, err = local.listRepos() 59 | if err != nil { 60 | return 61 | } 62 | 63 | for _, repo := range repos { 64 | size += repo.Size 65 | } 66 | return 67 | } 68 | 69 | func (local *Local) UploadObject(filePath string, overwrite bool) (length int64, err error) { 70 | absFilePath := filepath.Join(local.Conf.RepoPath, filePath) 71 | data, err := os.ReadFile(absFilePath) 72 | if err != nil { 73 | return 74 | } 75 | 76 | length, err = local.UploadBytes(filePath, data, overwrite) 77 | return 78 | } 79 | 80 | func (local *Local) UploadBytes(filePath string, data []byte, overwrite bool) (length int64, err error) { 81 | key := path.Join(local.getCurrentRepoDirPath(), filePath) 82 | folder := path.Dir(key) 83 | err = os.MkdirAll(folder, 0755) 84 | if err != nil { 85 | return 86 | } 87 | 88 | if !overwrite { // not overwrite the file 89 | _, err = os.Stat(key) 90 | if err != nil { 91 | if os.IsNotExist(err) { // file not exist 92 | // do nothing and continue 93 | err = nil 94 | } else { // other error 95 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 96 | return 97 | } 98 | } 99 | } 100 | 101 | err = os.WriteFile(key, data, 0644) 102 | if err != nil { 103 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 104 | return 105 | } 106 | 107 | length = int64(len(data)) 108 | 109 | //logging.LogInfof("uploaded object [%s]", key) 110 | return 111 | } 112 | 113 | func (local *Local) DownloadObject(filePath string) (data []byte, err error) { 114 | key := path.Join(local.getCurrentRepoDirPath(), filePath) 115 | data, err = os.ReadFile(key) 116 | if err != nil { 117 | if os.IsNotExist(err) { 118 | err = ErrCloudObjectNotFound 119 | } 120 | return 121 | } 122 | 123 | //logging.LogInfof("downloaded object [%s]", key) 124 | return 125 | } 126 | 127 | func (local *Local) RemoveObject(filePath string) (err error) { 128 | key := path.Join(local.getCurrentRepoDirPath(), filePath) 129 | err = os.Remove(key) 130 | if err != nil { 131 | if os.IsNotExist(err) { 132 | err = nil 133 | } else { 134 | logging.LogErrorf("remove object [%s] failed: %s", key, err) 135 | } 136 | return 137 | } 138 | 139 | //logging.LogInfof("removed object [%s]", key) 140 | return 141 | } 142 | 143 | func (local *Local) ListObjects(pathPrefix string) (objects map[string]*entity.ObjectInfo, err error) { 144 | objects = map[string]*entity.ObjectInfo{} 145 | pathPrefix = path.Join(local.getCurrentRepoDirPath(), pathPrefix) 146 | entries, err := os.ReadDir(pathPrefix) 147 | if err != nil { 148 | logging.LogErrorf("list objects [%s] failed: %s", pathPrefix, err) 149 | return 150 | } 151 | 152 | for _, entry := range entries { 153 | entryInfo, infoErr := entry.Info() 154 | if infoErr != nil { 155 | err = infoErr 156 | logging.LogErrorf("get object [%s] info failed: %s", path.Join(pathPrefix, entry.Name()), err) 157 | return 158 | } 159 | 160 | filePath := entry.Name() 161 | objects[filePath] = &entity.ObjectInfo{ 162 | Path: filePath, 163 | Size: entryInfo.Size(), 164 | } 165 | } 166 | 167 | //logging.LogInfof("list objects [%s]", pathPrefix) 168 | return 169 | } 170 | 171 | func (local *Local) GetTags() (tags []*Ref, err error) { 172 | tags, err = local.listRepoRefs("tags") 173 | if err != nil { 174 | return 175 | } 176 | if 1 > len(tags) { 177 | tags = []*Ref{} 178 | } 179 | return 180 | } 181 | 182 | func (local *Local) GetIndexes(page int) (indexes []*entity.Index, pageCount, totalCount int, err error) { 183 | data, err := local.DownloadObject("indexes-v2.json") 184 | if err != nil { 185 | if os.IsNotExist(err) { 186 | err = nil 187 | } 188 | return 189 | } 190 | 191 | data, err = compressDecoder.DecodeAll(data, nil) 192 | if err != nil { 193 | return 194 | } 195 | 196 | indexesJSON := &Indexes{} 197 | if err = gulu.JSON.UnmarshalJSON(data, indexesJSON); err != nil { 198 | return 199 | } 200 | 201 | totalCount = len(indexesJSON.Indexes) 202 | pageCount = int(math.Ceil(float64(totalCount) / float64(pageSize))) 203 | start := (page - 1) * pageSize 204 | end := page * pageSize 205 | if end > totalCount { 206 | end = totalCount 207 | } 208 | 209 | for i := start; i < end; i++ { 210 | index, getErr := local.repoIndex(indexesJSON.Indexes[i].ID) 211 | if getErr != nil { 212 | logging.LogWarnf("get repo index [%s] failed: %s", indexesJSON.Indexes[i], getErr) 213 | continue 214 | } 215 | 216 | index.Files = nil // Optimize the performance of obtaining cloud snapshots https://github.com/siyuan-note/siyuan/issues/8387 217 | indexes = append(indexes, index) 218 | } 219 | return 220 | } 221 | 222 | func (local *Local) GetRefsFiles() (fileIDs []string, refs []*Ref, err error) { 223 | refs, err = local.listRepoRefs("") 224 | var files []string 225 | for _, ref := range refs { 226 | index, getErr := local.repoIndex(ref.ID) 227 | if getErr != nil { 228 | return 229 | } 230 | if index == nil { 231 | continue 232 | } 233 | 234 | files = append(files, index.Files...) 235 | } 236 | 237 | fileIDs = gulu.Str.RemoveDuplicatedElem(files) 238 | if 1 > len(fileIDs) { 239 | fileIDs = []string{} 240 | } 241 | return 242 | } 243 | 244 | func (local *Local) GetChunks(checkChunkIDs []string) (chunkIDs []string, err error) { 245 | repoObjectsPath := path.Join(local.getCurrentRepoDirPath(), "objects") 246 | var keys []string 247 | for _, chunkID := range checkChunkIDs { 248 | key := path.Join(repoObjectsPath, chunkID[:2], chunkID[2:]) 249 | keys = append(keys, key) 250 | } 251 | 252 | notFound, err := local.getNotFound(keys) 253 | if err != nil { 254 | return 255 | } 256 | 257 | var notFoundChunkIDs []string 258 | for _, key := range notFound { 259 | chunkID := strings.TrimPrefix(key, repoObjectsPath) 260 | chunkID = strings.ReplaceAll(chunkID, "/", "") 261 | notFoundChunkIDs = append(notFoundChunkIDs, chunkID) 262 | } 263 | 264 | chunkIDs = append(chunkIDs, notFoundChunkIDs...) 265 | chunkIDs = gulu.Str.RemoveDuplicatedElem(chunkIDs) 266 | if 1 > len(chunkIDs) { 267 | chunkIDs = []string{} 268 | } 269 | return 270 | } 271 | 272 | // func (local *Local) GetStat() (stat *Stat, err error) 273 | 274 | func (local *Local) GetIndex(id string) (index *entity.Index, err error) { 275 | index, err = local.repoIndex(id) 276 | if err != nil { 277 | logging.LogErrorf("get repo index [%s] failed: %s", id, err) 278 | return 279 | } 280 | if index == nil { 281 | err = ErrCloudObjectNotFound 282 | return 283 | } 284 | return 285 | } 286 | 287 | func (local *Local) GetConcurrentReqs() (ret int) { 288 | ret = local.Local.ConcurrentReqs 289 | if ret < 1 { 290 | ret = 16 291 | } 292 | if ret > 1024 { 293 | ret = 1024 294 | } 295 | return 296 | } 297 | 298 | func (local *Local) GetConf() *Conf { 299 | return local.Conf 300 | } 301 | 302 | func (local *Local) GetAvailableSize() int64 { 303 | return util.GetFreeDiskSpace(local.Local.Endpoint) 304 | } 305 | 306 | func (local *Local) AddTraffic(*Traffic) { 307 | return 308 | } 309 | 310 | func (local *Local) listRepos() (repos []*Repo, err error) { 311 | entries, err := os.ReadDir(local.Local.Endpoint) 312 | if err != nil { 313 | logging.LogErrorf("list repos [%s] failed: %s", local.Local.Endpoint, err) 314 | return 315 | } 316 | 317 | for _, entry := range entries { 318 | if !entry.IsDir() { 319 | continue 320 | } 321 | 322 | entryInfo, infoErr := entry.Info() 323 | if infoErr != nil { 324 | err = infoErr 325 | logging.LogErrorf("get repo [%s] info failed: %s", path.Join(local.Local.Endpoint, entry.Name()), err) 326 | return 327 | } 328 | repos = append(repos, &Repo{ 329 | Name: entry.Name(), 330 | Size: entryInfo.Size(), 331 | Updated: entryInfo.ModTime().Local().Format("2006-01-02 15:04:05"), 332 | }) 333 | } 334 | sort.Slice(repos, func(i, j int) bool { return repos[i].Name < repos[j].Name }) 335 | return 336 | } 337 | 338 | func (local *Local) listRepoRefs(refPrefix string) (refs []*Ref, err error) { 339 | keyPath := path.Join(local.getCurrentRepoDirPath(), "refs", refPrefix) 340 | entries, err := os.ReadDir(keyPath) 341 | if err != nil { 342 | logging.LogErrorf("list repo refs [%s] failed: %s", keyPath, err) 343 | return 344 | } 345 | 346 | for _, entry := range entries { 347 | if entry.IsDir() { 348 | continue 349 | } 350 | 351 | entryInfo, infoErr := entry.Info() 352 | if infoErr != nil { 353 | err = infoErr 354 | logging.LogErrorf("get repo ref [%s] info failed: %s", path.Join(local.Local.Endpoint, entry.Name()), err) 355 | return 356 | } 357 | 358 | data, readErr := os.ReadFile(path.Join(keyPath, entry.Name())) 359 | if readErr != nil { 360 | err = readErr 361 | logging.LogErrorf("get repo ref [%s] ID failed: %s", path.Join(local.Local.Endpoint, entry.Name()), err) 362 | return 363 | } 364 | 365 | id := string(data) 366 | ref := &Ref{ 367 | Name: entry.Name(), 368 | ID: id, 369 | Updated: entryInfo.ModTime().Local().Format("2006-01-02 15:04:05"), 370 | } 371 | refs = append(refs, ref) 372 | } 373 | return 374 | } 375 | 376 | func (local *Local) repoIndex(id string) (index *entity.Index, err error) { 377 | indexFilePath := path.Join(local.getCurrentRepoDirPath(), "indexes", id) 378 | indexFileInfo, err := os.Stat(indexFilePath) 379 | if err != nil { 380 | return 381 | } 382 | if 1 > indexFileInfo.Size() { 383 | return 384 | } 385 | 386 | data, err := os.ReadFile(indexFilePath) 387 | if err != nil { 388 | return 389 | } 390 | 391 | data, err = compressDecoder.DecodeAll(data, nil) 392 | if err != nil { 393 | return 394 | } 395 | 396 | index = &entity.Index{} 397 | err = gulu.JSON.UnmarshalJSON(data, index) 398 | if err != nil { 399 | return 400 | } 401 | return 402 | } 403 | 404 | func (local *Local) getNotFound(keys []string) (ret []string, err error) { 405 | if 1 > len(keys) { 406 | return 407 | } 408 | for _, key := range keys { 409 | _, statErr := os.Stat(key) 410 | if os.IsNotExist(statErr) { 411 | ret = append(ret, key) 412 | } 413 | } 414 | return 415 | } 416 | 417 | func (local *Local) getCurrentRepoDirPath() string { 418 | return path.Join(local.Local.Endpoint, local.Dir) 419 | } 420 | -------------------------------------------------------------------------------- /cloud/webdav.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package cloud 18 | 19 | import ( 20 | "errors" 21 | "io/fs" 22 | "math" 23 | "os" 24 | "path" 25 | "path/filepath" 26 | "sort" 27 | "strings" 28 | "sync" 29 | 30 | "github.com/88250/gulu" 31 | "github.com/siyuan-note/dejavu/entity" 32 | "github.com/siyuan-note/logging" 33 | "github.com/studio-b12/gowebdav" 34 | ) 35 | 36 | // WebDAV 描述了 WebDAV 云端存储服务实现。 37 | type WebDAV struct { 38 | *BaseCloud 39 | Client *gowebdav.Client 40 | 41 | lock sync.Mutex 42 | } 43 | 44 | func NewWebDAV(baseCloud *BaseCloud, client *gowebdav.Client) (ret *WebDAV) { 45 | ret = &WebDAV{ 46 | BaseCloud: baseCloud, 47 | Client: client, 48 | lock: sync.Mutex{}, 49 | } 50 | return 51 | } 52 | 53 | func (webdav *WebDAV) GetRepos() (repos []*Repo, size int64, err error) { 54 | repos, err = webdav.listRepos() 55 | if nil != err { 56 | return 57 | } 58 | 59 | for _, repo := range repos { 60 | size += repo.Size 61 | } 62 | return 63 | } 64 | 65 | func (webdav *WebDAV) UploadObject(filePath string, overwrite bool) (length int64, err error) { 66 | absFilePath := filepath.Join(webdav.Conf.RepoPath, filePath) 67 | data, err := os.ReadFile(absFilePath) 68 | if nil != err { 69 | return 70 | } 71 | 72 | length, err = webdav.UploadBytes(filePath, data, overwrite) 73 | return 74 | } 75 | 76 | func (webdav *WebDAV) UploadBytes(filePath string, data []byte, overwrite bool) (length int64, err error) { 77 | length = int64(len(data)) 78 | key := path.Join(webdav.Dir, "siyuan", "repo", filePath) 79 | folder := path.Dir(key) 80 | err = webdav.mkdirAll(folder) 81 | if nil != err { 82 | return 83 | } 84 | 85 | err = webdav.Client.Write(key, data, 0644) 86 | err = webdav.parseErr(err) 87 | if nil != err { 88 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 89 | return 90 | } 91 | //logging.LogInfof("uploaded object [%s]", key) 92 | return 93 | } 94 | 95 | func (webdav *WebDAV) DownloadObject(filePath string) (data []byte, err error) { 96 | key := path.Join(webdav.Dir, "siyuan", "repo", filePath) 97 | data, err = webdav.Client.Read(key) 98 | err = webdav.parseErr(err) 99 | if nil != err { 100 | return 101 | } 102 | 103 | //logging.LogInfof("downloaded object [%s]", key) 104 | return 105 | } 106 | 107 | func (webdav *WebDAV) RemoveObject(filePath string) (err error) { 108 | key := path.Join(webdav.Dir, "siyuan", "repo", filePath) 109 | err = webdav.Client.Remove(key) 110 | err = webdav.parseErr(err) 111 | if nil != err { 112 | return 113 | } 114 | 115 | //logging.LogInfof("removed object [%s]", key) 116 | return 117 | } 118 | 119 | func (webdav *WebDAV) GetTags() (tags []*Ref, err error) { 120 | tags, err = webdav.listRepoRefs("tags") 121 | if nil != err { 122 | err = webdav.parseErr(err) 123 | if errors.Is(err, ErrCloudObjectNotFound) { // https://ld246.com/article/1749182255326 124 | err = nil 125 | tags = []*Ref{} 126 | return 127 | } 128 | return 129 | } 130 | if 1 > len(tags) { 131 | tags = []*Ref{} 132 | } 133 | return 134 | } 135 | 136 | func (webdav *WebDAV) GetIndexes(page int) (ret []*entity.Index, pageCount, totalCount int, err error) { 137 | ret = []*entity.Index{} 138 | data, err := webdav.DownloadObject("indexes-v2.json") 139 | if nil != err { 140 | err = webdav.parseErr(err) 141 | if ErrCloudObjectNotFound == err { 142 | err = nil 143 | } 144 | return 145 | } 146 | 147 | data, err = compressDecoder.DecodeAll(data, nil) 148 | if nil != err { 149 | return 150 | } 151 | 152 | indexesJSON := &Indexes{} 153 | if err = gulu.JSON.UnmarshalJSON(data, indexesJSON); nil != err { 154 | return 155 | } 156 | 157 | totalCount = len(indexesJSON.Indexes) 158 | pageCount = int(math.Ceil(float64(totalCount) / float64(pageSize))) 159 | 160 | start := (page - 1) * pageSize 161 | end := page * pageSize 162 | if end > totalCount { 163 | end = totalCount 164 | } 165 | 166 | repoKey := path.Join(webdav.Dir, "siyuan", "repo") 167 | for i := start; i < end; i++ { 168 | index, getErr := webdav.repoIndex(repoKey, indexesJSON.Indexes[i].ID) 169 | if nil != getErr { 170 | logging.LogWarnf("get index [%s] failed: %s", indexesJSON.Indexes[i], getErr) 171 | continue 172 | } 173 | 174 | index.Files = nil // Optimize the performance of obtaining cloud snapshots https://github.com/siyuan-note/siyuan/issues/8387 175 | ret = append(ret, index) 176 | } 177 | return 178 | } 179 | 180 | func (webdav *WebDAV) GetRefsFiles() (fileIDs []string, refs []*Ref, err error) { 181 | refs, err = webdav.listRepoRefs("") 182 | repoKey := path.Join(webdav.Dir, "siyuan", "repo") 183 | var files []string 184 | for _, ref := range refs { 185 | index, getErr := webdav.repoIndex(repoKey, ref.ID) 186 | if nil != getErr { 187 | err = getErr 188 | return 189 | } 190 | if nil == index { 191 | continue 192 | } 193 | 194 | files = append(files, index.Files...) 195 | } 196 | fileIDs = gulu.Str.RemoveDuplicatedElem(files) 197 | if 1 > len(fileIDs) { 198 | fileIDs = []string{} 199 | } 200 | return 201 | } 202 | 203 | func (webdav *WebDAV) GetChunks(checkChunkIDs []string) (chunkIDs []string, err error) { 204 | repoObjects := path.Join(webdav.Dir, "siyuan", "repo", "objects") 205 | var keys []string 206 | for _, chunk := range checkChunkIDs { 207 | key := path.Join(repoObjects, chunk[:2], chunk[2:]) 208 | keys = append(keys, key) 209 | } 210 | 211 | notFound, err := webdav.getNotFound(keys) 212 | if nil != err { 213 | return 214 | } 215 | 216 | var notFoundChunkIDs []string 217 | for _, key := range notFound { 218 | chunkID := strings.TrimPrefix(key, repoObjects) 219 | chunkID = strings.ReplaceAll(chunkID, "/", "") 220 | notFoundChunkIDs = append(notFoundChunkIDs, chunkID) 221 | } 222 | 223 | chunkIDs = append(chunkIDs, notFoundChunkIDs...) 224 | chunkIDs = gulu.Str.RemoveDuplicatedElem(chunkIDs) 225 | if 1 > len(chunkIDs) { 226 | chunkIDs = []string{} 227 | } 228 | return 229 | } 230 | 231 | func (webdav *WebDAV) GetIndex(id string) (index *entity.Index, err error) { 232 | repoKey := path.Join(webdav.Dir, "siyuan", "repo") 233 | index, err = webdav.repoIndex(repoKey, id) 234 | if nil != err { 235 | logging.LogErrorf("get index [%s] failed: %s", id, err) 236 | return 237 | } 238 | if nil == index { 239 | err = ErrCloudObjectNotFound 240 | return 241 | } 242 | return 243 | } 244 | 245 | func (webdav *WebDAV) GetConcurrentReqs() (ret int) { 246 | ret = webdav.Conf.WebDAV.ConcurrentReqs 247 | if 1 > ret { 248 | ret = 1 249 | } 250 | if 16 < ret { 251 | ret = 16 252 | } 253 | return 254 | } 255 | 256 | func (webdav *WebDAV) ListObjects(pathPrefix string) (ret map[string]*entity.ObjectInfo, err error) { 257 | ret = map[string]*entity.ObjectInfo{} 258 | 259 | endWithSlash := strings.HasSuffix(pathPrefix, "/") 260 | pathPrefix = path.Join(webdav.Dir, "siyuan", "repo", pathPrefix) 261 | if endWithSlash { 262 | pathPrefix += "/" 263 | } 264 | 265 | infos, err := webdav.Client.ReadDir(pathPrefix) 266 | if nil != err { 267 | logging.LogErrorf("list objects [%s] failed: %s", pathPrefix, err) 268 | return 269 | } 270 | 271 | for _, entry := range infos { 272 | filePath := entry.Name() 273 | ret[filePath] = &entity.ObjectInfo{ 274 | Path: filePath, 275 | Size: entry.Size(), 276 | } 277 | } 278 | 279 | if nil != err { 280 | logging.LogErrorf("list objects failed: %s", err) 281 | return 282 | } 283 | return 284 | } 285 | 286 | func (webdav *WebDAV) listRepoRefs(refPrefix string) (ret []*Ref, err error) { 287 | keyPath := path.Join(webdav.Dir, "siyuan", "repo", "refs", refPrefix) 288 | infos, err := webdav.Client.ReadDir(keyPath) 289 | if nil != err { 290 | err = webdav.parseErr(err) 291 | return 292 | } 293 | 294 | for _, info := range infos { 295 | if info.IsDir() { 296 | continue 297 | } 298 | 299 | data, ReadErr := webdav.Client.Read(path.Join(keyPath, info.Name())) 300 | if nil != ReadErr { 301 | err = webdav.parseErr(ReadErr) 302 | return 303 | } 304 | id := string(data) 305 | ref := &Ref{ 306 | Name: info.Name(), 307 | ID: id, 308 | Updated: info.ModTime().Local().Format("2006-01-02 15:04:05"), 309 | } 310 | ret = append(ret, ref) 311 | } 312 | return 313 | } 314 | 315 | func (webdav *WebDAV) listRepos() (ret []*Repo, err error) { 316 | infos, err := webdav.Client.ReadDir("/") 317 | if nil != err { 318 | err = webdav.parseErr(err) 319 | if ErrCloudObjectNotFound == err { 320 | err = nil 321 | } 322 | return 323 | } 324 | 325 | for _, repoInfo := range infos { 326 | if !repoInfo.IsDir() { 327 | continue 328 | } 329 | 330 | ret = append(ret, &Repo{ 331 | Name: repoInfo.Name(), 332 | Size: 0, 333 | Updated: repoInfo.ModTime().Local().Format("2006-01-02 15:04:05"), 334 | }) 335 | } 336 | sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) 337 | return 338 | } 339 | 340 | func (webdav *WebDAV) repoLatest(repoDir string) (id string, err error) { 341 | latestPath := path.Join(repoDir, "refs", "latest") 342 | _, err = webdav.Client.Stat(latestPath) 343 | if nil != err { 344 | err = webdav.parseErr(err) 345 | return 346 | } 347 | 348 | data, err := webdav.Client.Read(latestPath) 349 | if nil != err { 350 | return 351 | } 352 | id = string(data) 353 | return 354 | } 355 | 356 | func (webdav *WebDAV) getNotFound(keys []string) (ret []string, err error) { 357 | if 1 > len(keys) { 358 | return 359 | } 360 | for _, key := range keys { 361 | _, statErr := webdav.Client.Stat(key) 362 | statErr = webdav.parseErr(statErr) 363 | if ErrCloudObjectNotFound == statErr { 364 | ret = append(ret, key) 365 | } 366 | } 367 | return 368 | } 369 | 370 | func (webdav *WebDAV) repoIndex(repoDir, id string) (ret *entity.Index, err error) { 371 | indexPath := path.Join(repoDir, "indexes", id) 372 | info, err := webdav.Client.Stat(indexPath) 373 | if nil != err { 374 | err = webdav.parseErr(err) 375 | return 376 | } 377 | if 1 > info.Size() { 378 | return 379 | } 380 | 381 | data, err := webdav.Client.Read(indexPath) 382 | if nil != err { 383 | return 384 | } 385 | data, err = compressDecoder.DecodeAll(data, nil) 386 | if nil != err { 387 | return 388 | } 389 | ret = &entity.Index{} 390 | err = gulu.JSON.UnmarshalJSON(data, ret) 391 | return 392 | } 393 | 394 | func (webdav *WebDAV) parseErr(err error) error { 395 | if nil == err { 396 | return nil 397 | } 398 | 399 | switch err.(type) { 400 | case *fs.PathError: 401 | if e := errors.Unwrap(err); nil != e { 402 | switch e.(type) { 403 | case gowebdav.StatusError: 404 | statusErr := e.(gowebdav.StatusError) 405 | if 404 == statusErr.Status { 406 | return ErrCloudObjectNotFound 407 | } else if 503 == statusErr.Status || 502 == statusErr.Status || 500 == statusErr.Status { 408 | return ErrCloudServiceUnavailable 409 | } else if 200 == statusErr.Status { 410 | return nil 411 | } 412 | } 413 | } 414 | } 415 | msg := strings.ToLower(err.Error()) 416 | if strings.Contains(msg, "404") || strings.Contains(msg, "no such file") { 417 | err = ErrCloudObjectNotFound 418 | } 419 | return err 420 | } 421 | 422 | func (webdav *WebDAV) mkdirAll(folder string) (err error) { 423 | cacheKey := "webdav.dir." + folder 424 | _, ok := cache.Get(cacheKey) 425 | if ok { 426 | return 427 | } 428 | 429 | webdav.lock.Lock() 430 | defer webdav.lock.Unlock() 431 | 432 | info, err := webdav.Client.Stat(folder) 433 | if nil != err { 434 | err = webdav.parseErr(err) 435 | if nil == err { 436 | cache.Set(cacheKey, true, 1) 437 | return 438 | } 439 | 440 | if ErrCloudObjectNotFound != err { 441 | return 442 | } 443 | } 444 | i := info.(*gowebdav.File) 445 | if nil != i && i.IsDir() { 446 | cache.Set(cacheKey, true, 1) 447 | return 448 | } 449 | 450 | paths := strings.Split(folder, "/") 451 | sub := "/" 452 | for _, e := range paths { 453 | if e == "" { 454 | continue 455 | } 456 | sub += e + "/" 457 | 458 | if _, ok := cache.Get("webdav.dir." + sub); ok { 459 | continue 460 | } 461 | 462 | err = webdav.Client.Mkdir(sub, 0755) 463 | err = webdav.parseErr(err) 464 | if nil != err { 465 | logging.LogErrorf("mkdir [%s] failed: %s", folder, err) 466 | } else { 467 | cache.Set("webdav.dir."+sub, true, 1) 468 | } 469 | } 470 | return 471 | } 472 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package dejavu 18 | 19 | import ( 20 | "errors" 21 | "os" 22 | "path/filepath" 23 | "strings" 24 | "time" 25 | 26 | "github.com/88250/gulu" 27 | "github.com/dgraph-io/ristretto" 28 | "github.com/klauspost/compress/zstd" 29 | "github.com/siyuan-note/dejavu/entity" 30 | "github.com/siyuan-note/encryption" 31 | "github.com/siyuan-note/logging" 32 | ) 33 | 34 | var ErrNotFoundObject = errors.New("not found object") 35 | 36 | // Store 描述了存储库。 37 | type Store struct { 38 | Path string // 存储库文件夹的绝对路径，如：F:\\SiYuan\\repo\\ 39 | AesKey []byte 40 | 41 | compressEncoder *zstd.Encoder 42 | compressDecoder *zstd.Decoder 43 | } 44 | 45 | func NewStore(path string, aesKey []byte) (ret *Store, err error) { 46 | ret = &Store{Path: path, AesKey: aesKey} 47 | 48 | ret.compressEncoder, err = zstd.NewWriter(nil, 49 | zstd.WithEncoderLevel(zstd.SpeedDefault), 50 | zstd.WithEncoderCRC(false), 51 | zstd.WithWindowSize(512*1024)) 52 | if nil != err { 53 | return 54 | } 55 | ret.compressDecoder, err = zstd.NewReader(nil, 56 | zstd.WithDecoderMaxMemory(16*1024*1024*1024)) 57 | return 58 | } 59 | 60 | func (store *Store) Purge(retentionIndexIDs ...string) (ret *entity.PurgeStat, err error) { 61 | logging.LogInfof("purging data repo [%s], retention indexes [%d]", store.Path, len(retentionIndexIDs)) 62 | 63 | objectsDir := filepath.Join(store.Path, "objects") 64 | if !gulu.File.IsDir(objectsDir) { 65 | logging.LogWarnf("objects dir [%s] is not a dir", objectsDir) 66 | return 67 | } 68 | 69 | entries, err := os.ReadDir(objectsDir) 70 | if nil != err { 71 | logging.LogErrorf("read objects dir [%s] failed: %s", objectsDir, err) 72 | return 73 | } 74 | 75 | // 收集所有数据对象 76 | objIDs := map[string]bool{} 77 | for _, entry := range entries { 78 | if !entry.IsDir() { 79 | continue 80 | } 81 | 82 | dirName := entry.Name() 83 | dir := filepath.Join(objectsDir, dirName) 84 | objs, readErr := os.ReadDir(dir) 85 | if nil != readErr { 86 | err = readErr 87 | logging.LogErrorf("read objects dir [%s] failed: %s", dir, err) 88 | return 89 | } 90 | 91 | for _, obj := range objs { 92 | id := dirName + obj.Name() 93 | objIDs[id] = true 94 | } 95 | } 96 | 97 | // 收集所有索引对象 98 | indexIDs := map[string]bool{} 99 | indexesDir := filepath.Join(store.Path, "indexes") 100 | if gulu.File.IsDir(indexesDir) { 101 | entries, err = os.ReadDir(indexesDir) 102 | if nil != err { 103 | logging.LogErrorf("read indexes dir [%s] failed: %s", indexesDir, err) 104 | return 105 | } 106 | 107 | for _, entry := range entries { 108 | id := entry.Name() 109 | if 40 != len(id) { 110 | continue 111 | } 112 | 113 | indexIDs[id] = true 114 | } 115 | } 116 | 117 | // 收集所有引用的索引 118 | refIndexIDs, err := store.readRefs() 119 | if nil != err { 120 | logging.LogErrorf("read refs failed: %s", err) 121 | return 122 | } 123 | for _, retentionIndexID := range retentionIndexIDs { // 指定保留的索引算作被引用 124 | refIndexIDs[retentionIndexID] = true 125 | } 126 | unreferencedIndexIDs := map[string]bool{} 127 | for indexID := range indexIDs { 128 | if !refIndexIDs[indexID] { 129 | unreferencedIndexIDs[indexID] = true 130 | } 131 | } 132 | 133 | // 收集所有引用的数据对象 134 | referencedObjIDs := map[string]bool{} 135 | for refID := range refIndexIDs { 136 | index, getErr := store.GetIndex(refID) 137 | if nil != getErr { 138 | logging.LogWarnf("get index [%s] failed: %s", refID, getErr) 139 | continue 140 | } 141 | 142 | for _, fileID := range index.Files { 143 | referencedObjIDs[fileID] = true 144 | file, getFileErr := store.GetFile(fileID) 145 | if nil != getFileErr { 146 | logging.LogWarnf("get file [%s] failed: %s", fileID, getFileErr) 147 | continue 148 | } 149 | 150 | for _, chunkID := range file.Chunks { 151 | referencedObjIDs[chunkID] = true 152 | } 153 | } 154 | } 155 | 156 | // 收集所有未引用的数据对象 157 | unreferencedObjIDs := map[string]bool{} 158 | for objID := range objIDs { 159 | if !referencedObjIDs[objID] { 160 | unreferencedObjIDs[objID] = true 161 | } 162 | } 163 | 164 | ret = &entity.PurgeStat{} 165 | ret.Indexes = len(unreferencedIndexIDs) 166 | 167 | // 清理未引用的索引对象 168 | for unreferencedIndexID := range unreferencedIndexIDs { 169 | indexPath := filepath.Join(store.Path, "indexes", unreferencedIndexID) 170 | if err = os.RemoveAll(indexPath); nil != err { 171 | logging.LogErrorf("remove unreferenced index [%s] failed: %s", unreferencedIndexID, err) 172 | return 173 | } 174 | } 175 | 176 | // 清理校验索引 177 | // Clear check index when purging data repo https://github.com/siyuan-note/siyuan/issues/9665 178 | checkIndexesDir := filepath.Join(store.Path, "check", "indexes") 179 | if gulu.File.IsDir(checkIndexesDir) { 180 | entries, err = os.ReadDir(checkIndexesDir) 181 | if nil != err { 182 | logging.LogErrorf("read check indexes dir [%s] failed: %s", checkIndexesDir, err) 183 | } else { 184 | for _, entry := range entries { 185 | id := entry.Name() 186 | if 40 != len(id) { 187 | continue 188 | } 189 | 190 | data, readErr := os.ReadFile(filepath.Join(checkIndexesDir, id)) 191 | if nil != readErr { 192 | logging.LogErrorf("read check index [%s] failed: %s", id, readErr) 193 | continue 194 | } 195 | 196 | if data, readErr = store.compressDecoder.DecodeAll(data, nil); nil != readErr { 197 | logging.LogErrorf("decode check index [%s] failed: %s", id, readErr) 198 | continue 199 | } 200 | 201 | checkIndex := &entity.CheckIndex{} 202 | if readErr = gulu.JSON.UnmarshalJSON(data, checkIndex); nil != readErr { 203 | logging.LogErrorf("unmarshal check index [%s] failed: %s", id, readErr) 204 | continue 205 | } 206 | 207 | if !unreferencedIndexIDs[checkIndex.IndexID] { 208 | continue 209 | } 210 | 211 | if _, statErr := os.Stat(filepath.Join(store.Path, "indexes", checkIndex.IndexID)); os.IsNotExist(statErr) { 212 | if removeErr := os.RemoveAll(filepath.Join(store.Path, "check", "indexes", checkIndex.ID)); nil != removeErr { 213 | logging.LogErrorf("remove check index [%s] failed: %s", checkIndex.ID, removeErr) 214 | } 215 | } 216 | } 217 | } 218 | } 219 | 220 | // 清理未引用的数据对象 221 | for unreferencedObjID := range unreferencedObjIDs { 222 | stat, statErr := store.Stat(unreferencedObjID) 223 | if nil != statErr { 224 | logging.LogErrorf("stat [%s] failed: %s", unreferencedObjID, statErr) 225 | continue 226 | } 227 | 228 | ret.Size += stat.Size() 229 | ret.Objects++ 230 | 231 | if err = store.Remove(unreferencedObjID); nil != err { 232 | logging.LogErrorf("remove unreferenced object [%s] failed: %s", unreferencedObjID, err) 233 | return 234 | } 235 | } 236 | 237 | fileCache.Clear() 238 | indexCache.Clear() 239 | 240 | logging.LogInfof("purged data repo [%s], [%d] indexes, [%d] objects, [%d] bytes", store.Path, ret.Indexes, ret.Objects, ret.Size) 241 | return 242 | } 243 | 244 | func (store *Store) readRefs() (ret map[string]bool, err error) { 245 | ret = map[string]bool{} 246 | refsDir := filepath.Join(store.Path, "refs") 247 | if !gulu.File.IsDir(refsDir) { 248 | return 249 | } 250 | 251 | err = filepath.Walk(refsDir, func(path string, info os.FileInfo, err error) error { 252 | if nil != err { 253 | return err 254 | } 255 | 256 | if info.IsDir() { 257 | return nil 258 | } 259 | 260 | if 42 < info.Size() { 261 | logging.LogWarnf("ref file [%s] is invalid", path) 262 | return nil 263 | } 264 | 265 | data, err := os.ReadFile(path) 266 | if nil != err { 267 | return err 268 | } 269 | 270 | content := strings.TrimSpace(string(data)) 271 | if 40 != len(content) { 272 | logging.LogWarnf("ref file [%s] is invalid", path) 273 | return nil 274 | } 275 | 276 | ret[content] = true 277 | return nil 278 | }) 279 | return 280 | } 281 | 282 | func (store *Store) PutIndex(index *entity.Index) (err error) { 283 | if "" == index.ID { 284 | return errors.New("invalid id") 285 | } 286 | dir, file := store.IndexAbsPath(index.ID) 287 | if err = os.MkdirAll(dir, 0755); nil != err { 288 | return errors.New("put index failed: " + err.Error()) 289 | } 290 | 291 | data, err := gulu.JSON.MarshalJSON(index) 292 | if nil != err { 293 | return errors.New("put index failed: " + err.Error()) 294 | } 295 | 296 | // Index 仅压缩，不加密 297 | data = store.compressEncoder.EncodeAll(data, nil) 298 | 299 | err = gulu.File.WriteFileSafer(file, data, 0644) 300 | if nil != err { 301 | return errors.New("put index failed: " + err.Error()) 302 | } 303 | 304 | created := time.UnixMilli(index.Created) 305 | if err = os.Chtimes(file, created, created); nil != err { 306 | logging.LogWarnf("change index [%s] time failed: %s", index.ID, err.Error()) 307 | } 308 | 309 | indexCache.Set(index.ID, index, int64(len(data))) 310 | return 311 | } 312 | 313 | func (store *Store) GetIndex(id string) (ret *entity.Index, err error) { 314 | cached, _ := indexCache.Get(id) 315 | if nil != cached { 316 | ret = cached.(*entity.Index) 317 | return 318 | } 319 | 320 | _, file := store.IndexAbsPath(id) 321 | var data []byte 322 | data, err = os.ReadFile(file) 323 | if nil != err { 324 | return 325 | } 326 | 327 | // Index 没有加密，直接解压 328 | data, err = store.compressDecoder.DecodeAll(data, nil) 329 | if nil == err { 330 | ret = &entity.Index{} 331 | err = gulu.JSON.UnmarshalJSON(data, ret) 332 | } 333 | if nil != err { 334 | return 335 | } 336 | 337 | indexCache.Set(id, ret, int64(len(data))) 338 | return 339 | } 340 | 341 | func (store *Store) PutFile(file *entity.File) (err error) { 342 | if "" == file.ID { 343 | return errors.New("invalid id") 344 | } 345 | dir, f := store.AbsPath(file.ID) 346 | if gulu.File.IsExist(f) { 347 | return 348 | } 349 | if err = os.MkdirAll(dir, 0755); nil != err { 350 | return errors.New("put failed: " + err.Error()) 351 | } 352 | 353 | data, err := gulu.JSON.MarshalJSON(file) 354 | if nil != err { 355 | return errors.New("put file failed: " + err.Error()) 356 | } 357 | if data, err = store.encodeData(data); nil != err { 358 | return 359 | } 360 | 361 | err = gulu.File.WriteFileSafer(f, data, 0644) 362 | if nil != err { 363 | return errors.New("put file failed: " + err.Error()) 364 | } 365 | 366 | fileCache.Set(file.ID, file, int64(len(data))) 367 | return 368 | } 369 | 370 | func (store *Store) GetFile(id string) (ret *entity.File, err error) { 371 | cached, _ := fileCache.Get(id) 372 | if nil != cached { 373 | ret = cached.(*entity.File) 374 | return 375 | } 376 | 377 | _, file := store.AbsPath(id) 378 | data, err := os.ReadFile(file) 379 | if nil != err { 380 | return 381 | } 382 | if data, err = store.decodeData(data); nil != err { 383 | return 384 | } 385 | ret = &entity.File{} 386 | err = gulu.JSON.UnmarshalJSON(data, ret) 387 | if nil != err { 388 | ret = nil 389 | return 390 | } 391 | 392 | fileCache.Set(id, ret, int64(len(data))) 393 | return 394 | } 395 | 396 | func (store *Store) PutChunk(chunk *entity.Chunk) (err error) { 397 | if "" == chunk.ID { 398 | return errors.New("invalid id") 399 | } 400 | dir, file := store.AbsPath(chunk.ID) 401 | if gulu.File.IsExist(file) { 402 | return 403 | } 404 | 405 | if err = os.MkdirAll(dir, 0755); nil != err { 406 | return errors.New("put chunk failed: " + err.Error()) 407 | } 408 | 409 | data := chunk.Data 410 | if data, err = store.encodeData(data); nil != err { 411 | return 412 | } 413 | 414 | err = gulu.File.WriteFileSafer(file, data, 0644) 415 | if nil != err { 416 | return errors.New("put chunk failed: " + err.Error()) 417 | } 418 | return 419 | } 420 | 421 | func (store *Store) GetChunk(id string) (ret *entity.Chunk, err error) { 422 | _, file := store.AbsPath(id) 423 | data, err := os.ReadFile(file) 424 | if nil != err { 425 | return 426 | } 427 | if data, err = store.decodeData(data); nil != err { 428 | return 429 | } 430 | ret = &entity.Chunk{ID: id, Data: data} 431 | return 432 | } 433 | 434 | func (store *Store) Remove(id string) (err error) { 435 | _, file := store.AbsPath(id) 436 | err = os.RemoveAll(file) 437 | return 438 | } 439 | 440 | func (store *Store) Stat(id string) (stat os.FileInfo, err error) { 441 | _, file := store.AbsPath(id) 442 | stat, err = os.Stat(file) 443 | return 444 | } 445 | 446 | func (store *Store) IndexAbsPath(id string) (dir, file string) { 447 | dir = filepath.Join(store.Path, "indexes") 448 | file = filepath.Join(dir, id) 449 | return 450 | } 451 | 452 | func (store *Store) AbsPath(id string) (dir, file string) { 453 | dir, file = id[0:2], id[2:] 454 | dir = filepath.Join(store.Path, "objects", dir) 455 | file = filepath.Join(dir, file) 456 | return 457 | } 458 | 459 | func (store *Store) encodeData(data []byte) ([]byte, error) { 460 | data = store.compressEncoder.EncodeAll(data, nil) 461 | return encryption.AesEncrypt(data, store.AesKey) 462 | } 463 | 464 | func (store *Store) decodeData(data []byte) (ret []byte, err error) { 465 | ret, err = encryption.AesDecrypt(data, store.AesKey) 466 | if nil != err { 467 | return 468 | } 469 | ret, err = store.compressDecoder.DecodeAll(ret, nil) 470 | return 471 | } 472 | 473 | var fileCache, _ = ristretto.NewCache(&ristretto.Config{ 474 | NumCounters: 200000, 475 | MaxCost: 1000 * 1000 * 32, // 1 个文件按 300 字节计算，32MB 大概可以缓存 10W 个文件实例 476 | BufferItems: 64, 477 | }) 478 | 479 | var indexCache, _ = ristretto.NewCache(&ristretto.Config{ 480 | NumCounters: 200000, 481 | MaxCost: 1000 * 1000 * 128, // 1 个文件按 300K 字节（大约 1.5W 个文件）计算，128MB 大概可以缓存 400 个索引 482 | BufferItems: 64, 483 | }) 484 | 485 | func (store *Store) cacheFile(file *entity.File) { 486 | fileCache.Set(file.ID, file, 256 /* 直接使用合理的均值以免进行实际计算消耗性能 */) 487 | } 488 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7 h1:MafIFwSS0x6A4hqNtl0ObDG2cx8kcafqWu2xxkwZ3rI= 2 | github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7/go.mod h1:HrKCCTin3YNDSLBD02K0AOljjV6eNwc3/zyEI+xyV1I= 3 | github.com/88250/gulu v1.2.3-0.20251119142510-7b1583ab4aa0 h1:ip0IQCJCLtJEDHil+2XSnh3NP39i98SYV5qhWoUeMnA= 4 | github.com/88250/gulu v1.2.3-0.20251119142510-7b1583ab4aa0/go.mod h1:IQ5dXW9CjVmx6B7OfK1Y4ZBKTPMe9q1AkVoLGGzRbS8= 5 | github.com/88250/lute v1.7.7-0.20250801084148-32f2ef961381 h1:On7e0kQ5WCCbY+B+ItjkUyqwzChPXiMpI9hlpbTYwfc= 6 | github.com/88250/lute v1.7.7-0.20250801084148-32f2ef961381/go.mod h1:WYyUw//5yVw9BJnoVjx7rI/3szsISxNZCYGOqTIrV0o= 7 | github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 8 | github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= 9 | github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= 10 | github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= 11 | github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= 12 | github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= 13 | github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= 14 | github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= 15 | github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= 16 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= 17 | github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= 18 | github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= 19 | github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= 20 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= 21 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= 22 | github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= 23 | github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= 24 | github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= 25 | github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= 26 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= 27 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= 28 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= 29 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= 30 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= 31 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= 32 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= 33 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= 34 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic= 35 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c= 36 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= 37 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= 38 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso= 39 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo= 40 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= 41 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= 42 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU= 43 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A= 44 | github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 h1:SWTxh/EcUCDVqi/0s26V6pVUq0BBG7kx0tDTmF/hCgA= 45 | github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8= 46 | github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= 47 | github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= 48 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= 49 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= 50 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= 51 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= 52 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= 53 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= 54 | github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= 55 | github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= 56 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 57 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 58 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 59 | github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= 60 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 61 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 62 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 63 | github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= 64 | github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= 65 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= 66 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 67 | github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= 68 | github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= 69 | github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= 70 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 71 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 72 | github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= 73 | github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= 74 | github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= 75 | github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= 76 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 77 | github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= 78 | github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= 79 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 80 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 81 | github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= 82 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 83 | github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= 84 | github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= 85 | github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= 86 | github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 87 | github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= 88 | github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= 89 | github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= 90 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 91 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 92 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 93 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 94 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 95 | github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= 96 | github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= 97 | github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= 98 | github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= 99 | github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= 100 | github.com/icholy/digest v1.1.0 h1:HfGg9Irj7i+IX1o1QAmPfIBNu/Q5A5Tu3n/MED9k9H4= 101 | github.com/icholy/digest v1.1.0/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y= 102 | github.com/imroc/req/v3 v3.57.0 h1:LMTUjNRUybUkTPn8oJDq8Kg3JRBOBTcnDhKu7mzupKI= 103 | github.com/imroc/req/v3 v3.57.0/go.mod h1:JL62ey1nvSLq81HORNcosvlf7SxZStONNqOprg0Pz00= 104 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 105 | github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= 106 | github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= 107 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 108 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 109 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 110 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 111 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 112 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 113 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 114 | github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= 115 | github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= 116 | github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= 117 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 118 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 119 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 120 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 121 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 122 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= 123 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= 124 | github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= 125 | github.com/qiniu/go-sdk/v7 v7.25.5 h1:BZAZhrYC7vrw9NPnNbFi1K8xucoz29s7sEGWap77i2Q= 126 | github.com/qiniu/go-sdk/v7 v7.25.5/go.mod h1:dmKtJ2ahhPWFVi9o1D5GemmWoh/ctuB9peqTowyTO8o= 127 | github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= 128 | github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= 129 | github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= 130 | github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= 131 | github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= 132 | github.com/refraction-networking/utls v1.8.1 h1:yNY1kapmQU8JeM1sSw2H2asfTIwWxIkrMJI0pRUOCAo= 133 | github.com/refraction-networking/utls v1.8.1/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= 134 | github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw= 135 | github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw= 136 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 137 | github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= 138 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= 139 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= 140 | github.com/shirou/gopsutil/v4 v4.25.11 h1:X53gB7muL9Gnwwo2evPSE+SfOrltMoR6V3xJAXZILTY= 141 | github.com/shirou/gopsutil/v4 v4.25.11/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU= 142 | github.com/siyuan-note/dataparser v0.0.0-20251203120213-59c16535cb56 h1:aKzEVlOOSwvYqpbWTAQpGrpAZT8JrpAvaY9ek4PXIAQ= 143 | github.com/siyuan-note/dataparser v0.0.0-20251203120213-59c16535cb56/go.mod h1:UGIytXL3Ge9iFj9RVDNAnfEOCPmzMPzpYTZyLtGC6tQ= 144 | github.com/siyuan-note/encryption v0.0.0-20251120032857-3ddc3c2cc49f h1:HSgJKIAMgokJDAvBBfRj47SzRSm6mNGssY0Wv7rcEtg= 145 | github.com/siyuan-note/encryption v0.0.0-20251120032857-3ddc3c2cc49f/go.mod h1:JE3S9VuJqTggyfhjesNDuqvqrRvwG3IctFjXXchLx1M= 146 | github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8= 147 | github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97/go.mod h1:1/nGgthl89FPA7GzAcEWKl6zRRnfgyTjzLZj9bW7kuw= 148 | github.com/siyuan-note/filelock v0.0.0-20251212095217-08318833e008 h1:3wEmNS4eZkxwm1rhXDhVK5Y0o/GKAZtfe1VV584BF+A= 149 | github.com/siyuan-note/filelock v0.0.0-20251212095217-08318833e008/go.mod h1:9OhXAyOkSXwuLvNCZk2aFMo0nOldyO3f2hMJEnkuT30= 150 | github.com/siyuan-note/httpclient v0.0.0-20251217011734-7f49de93158a h1:AHhTKVbZ1lCssLu6qgj2VgGwwnhMoMa3HfoEXVgGQp8= 151 | github.com/siyuan-note/httpclient v0.0.0-20251217011734-7f49de93158a/go.mod h1:USEy1/f1vT64cXQHd6GsiP0UkjDUxgMobg4RE1f91Fc= 152 | github.com/siyuan-note/logging v0.0.0-20251209020516-52f1a2f65ec5 h1:bIMoJAAf3tV0xYcN+N2Vw7Ot/LbVxuz715o1rn1GDto= 153 | github.com/siyuan-note/logging v0.0.0-20251209020516-52f1a2f65ec5/go.mod h1:U6DyWKvtIPW9WrUoUikPCwFUzUoHGtEJjjeLNYae1nc= 154 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 155 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 156 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 157 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 158 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 159 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 160 | github.com/studio-b12/gowebdav v0.11.0 h1:qbQzq4USxY28ZYsGJUfO5jR+xkFtcnwWgitp4Zp1irU= 161 | github.com/studio-b12/gowebdav v0.11.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= 162 | github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= 163 | github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= 164 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 165 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 166 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 167 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 168 | github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 169 | github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 170 | go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= 171 | go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= 172 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 173 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 174 | golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= 175 | golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= 176 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 177 | golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= 178 | golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= 179 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 180 | golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= 181 | golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= 182 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 183 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 184 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 185 | golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 186 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 187 | golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= 188 | golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 189 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 190 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 191 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 192 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 193 | golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= 194 | golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= 195 | golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= 196 | golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= 197 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 198 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 199 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 200 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 201 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 202 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 203 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 204 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 205 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 206 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 207 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 208 | gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= 209 | gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= 210 | modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= 211 | modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= 212 | modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= 213 | -------------------------------------------------------------------------------- /cloud/s3.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package cloud 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "errors" 23 | "fmt" 24 | "io" 25 | "math" 26 | "net/http" 27 | "os" 28 | "path" 29 | "path/filepath" 30 | "sort" 31 | "strings" 32 | "sync" 33 | "time" 34 | 35 | "github.com/88250/gulu" 36 | "github.com/aws/aws-sdk-go-v2/aws" 37 | asSigner "github.com/aws/aws-sdk-go-v2/aws/signer/v4" 38 | "github.com/aws/aws-sdk-go-v2/config" 39 | "github.com/aws/aws-sdk-go-v2/credentials" 40 | as3 "github.com/aws/aws-sdk-go-v2/service/s3" 41 | as3Types "github.com/aws/aws-sdk-go-v2/service/s3/types" 42 | "github.com/aws/smithy-go" 43 | "github.com/aws/smithy-go/middleware" 44 | smithyhttp "github.com/aws/smithy-go/transport/http" 45 | "github.com/panjf2000/ants/v2" 46 | "github.com/siyuan-note/dejavu/entity" 47 | "github.com/siyuan-note/logging" 48 | ) 49 | 50 | // S3 描述了 S3 协议兼容的对象存储服务实现。 51 | type S3 struct { 52 | *BaseCloud 53 | HTTPClient *http.Client 54 | service *as3.Client // 用于缓存 S3 客户端 55 | mux sync.Mutex // 用于保护 service 字段的并发访问 56 | } 57 | 58 | func NewS3(baseCloud *BaseCloud, httpClient *http.Client) *S3 { 59 | return &S3{BaseCloud: baseCloud, HTTPClient: httpClient} 60 | } 61 | 62 | func (s3 *S3) GetRepos() (repos []*Repo, size int64, err error) { 63 | repos, err = s3.listRepos() 64 | if nil != err { 65 | return 66 | } 67 | 68 | for _, repo := range repos { 69 | size += repo.Size 70 | } 71 | return 72 | } 73 | 74 | func (s3 *S3) UploadObject(filePath string, overwrite bool) (length int64, err error) { 75 | svc := s3.getService() 76 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 77 | defer cancelFn() 78 | 79 | absFilePath := filepath.Join(s3.Conf.RepoPath, filePath) 80 | info, err := os.Stat(absFilePath) 81 | if nil != err { 82 | logging.LogErrorf("stat failed: %s", err) 83 | return 84 | } 85 | length = info.Size() 86 | 87 | file, err := os.Open(absFilePath) 88 | if nil != err { 89 | return 90 | } 91 | defer file.Close() 92 | key := path.Join("repo", filePath) 93 | _, err = svc.PutObject(ctx, &as3.PutObjectInput{ 94 | Bucket: aws.String(s3.Conf.S3.Bucket), 95 | Key: aws.String(key), 96 | CacheControl: aws.String("no-cache"), 97 | Body: file, 98 | }) 99 | if nil != err { 100 | return 101 | } 102 | 103 | //logging.LogInfof("uploaded object [%s]", key) 104 | return 105 | } 106 | 107 | func (s3 *S3) UploadBytes(filePath string, data []byte, overwrite bool) (length int64, err error) { 108 | length = int64(len(data)) 109 | svc := s3.getService() 110 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 111 | defer cancelFn() 112 | 113 | key := path.Join("repo", filePath) 114 | _, err = svc.PutObject(ctx, &as3.PutObjectInput{ 115 | Bucket: aws.String(s3.Conf.S3.Bucket), 116 | Key: aws.String(key), 117 | CacheControl: aws.String("no-cache"), 118 | Body: bytes.NewReader(data), 119 | }) 120 | if nil != err { 121 | return 122 | } 123 | 124 | //logging.LogInfof("uploaded object [%s]", key) 125 | return 126 | } 127 | 128 | func (s3 *S3) DownloadObject(filePath string) (data []byte, err error) { 129 | svc := s3.getService() 130 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 131 | defer cancelFn() 132 | key := path.Join("repo", filePath) 133 | input := &as3.GetObjectInput{ 134 | Bucket: aws.String(s3.Conf.S3.Bucket), 135 | Key: aws.String(key), 136 | ResponseCacheControl: aws.String("no-cache"), 137 | } 138 | resp, err := svc.GetObject(ctx, input) 139 | if nil != err { 140 | if s3.isErrNotFound(err) { 141 | err = ErrCloudObjectNotFound 142 | } 143 | return 144 | } 145 | defer resp.Body.Close() 146 | data, err = io.ReadAll(resp.Body) 147 | if nil != err { 148 | return 149 | } 150 | 151 | //logging.LogInfof("downloaded object [%s]", key) 152 | return 153 | } 154 | 155 | func (s3 *S3) RemoveObject(key string) (err error) { 156 | key = path.Join("repo", key) 157 | svc := s3.getService() 158 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 159 | defer cancelFn() 160 | _, err = svc.DeleteObject(ctx, &as3.DeleteObjectInput{ 161 | Bucket: aws.String(s3.Conf.S3.Bucket), 162 | Key: aws.String(key), 163 | }) 164 | if nil != err { 165 | return 166 | } 167 | 168 | //logging.LogInfof("removed object [%s]", key) 169 | return 170 | } 171 | 172 | func (s3 *S3) GetTags() (tags []*Ref, err error) { 173 | tags, err = s3.listRepoRefs("tags") 174 | if nil != err { 175 | logging.LogErrorf("list repo tags failed: %s", err) 176 | return 177 | } 178 | if 1 > len(tags) { 179 | tags = []*Ref{} 180 | } 181 | return 182 | } 183 | 184 | const pageSize = 32 185 | 186 | func (s3 *S3) GetIndexes(page int) (ret []*entity.Index, pageCount, totalCount int, err error) { 187 | ret = []*entity.Index{} 188 | data, err := s3.DownloadObject("indexes-v2.json") 189 | if nil != err { 190 | if s3.isErrNotFound(err) { 191 | err = nil 192 | } 193 | return 194 | } 195 | 196 | data, err = compressDecoder.DecodeAll(data, nil) 197 | if nil != err { 198 | return 199 | } 200 | 201 | indexesJSON := &Indexes{} 202 | if err = gulu.JSON.UnmarshalJSON(data, indexesJSON); nil != err { 203 | return 204 | } 205 | 206 | totalCount = len(indexesJSON.Indexes) 207 | pageCount = int(math.Ceil(float64(totalCount) / float64(pageSize))) 208 | 209 | start := (page - 1) * pageSize 210 | end := page * pageSize 211 | if end > totalCount { 212 | end = totalCount 213 | } 214 | 215 | for i := start; i < end; i++ { 216 | index, getErr := s3.repoIndex(indexesJSON.Indexes[i].ID) 217 | if nil != getErr { 218 | logging.LogWarnf("get index [%s] failed: %s", indexesJSON.Indexes[i], getErr) 219 | continue 220 | } 221 | if nil == index { 222 | continue 223 | } 224 | 225 | index.Files = nil // Optimize the performance of obtaining cloud snapshots https://github.com/siyuan-note/siyuan/issues/8387 226 | ret = append(ret, index) 227 | } 228 | return 229 | } 230 | 231 | func (s3 *S3) GetRefsFiles() (fileIDs []string, refs []*Ref, err error) { 232 | refs, err = s3.listRepoRefs("") 233 | if nil != err { 234 | logging.LogErrorf("list repo refs failed: %s", err) 235 | return 236 | } 237 | 238 | var files []string 239 | for _, ref := range refs { 240 | index, getErr := s3.repoIndex(ref.ID) 241 | if nil != getErr { 242 | err = getErr 243 | return 244 | } 245 | if nil == index { 246 | continue 247 | } 248 | 249 | files = append(files, index.Files...) 250 | } 251 | fileIDs = gulu.Str.RemoveDuplicatedElem(files) 252 | if 1 > len(fileIDs) { 253 | fileIDs = []string{} 254 | } 255 | return 256 | } 257 | 258 | func (s3 *S3) GetChunks(checkChunkIDs []string) (chunkIDs []string, err error) { 259 | var keys []string 260 | repoObjects := path.Join("repo", "objects") 261 | for _, chunk := range checkChunkIDs { 262 | key := path.Join(repoObjects, chunk[:2], chunk[2:]) 263 | keys = append(keys, key) 264 | } 265 | 266 | notFound, err := s3.getNotFound(keys) 267 | if nil != err { 268 | return 269 | } 270 | 271 | var notFoundChunkIDs []string 272 | for _, key := range notFound { 273 | chunkID := strings.TrimPrefix(key, repoObjects) 274 | chunkID = strings.ReplaceAll(chunkID, "/", "") 275 | notFoundChunkIDs = append(notFoundChunkIDs, chunkID) 276 | } 277 | 278 | chunkIDs = append(chunkIDs, notFoundChunkIDs...) 279 | chunkIDs = gulu.Str.RemoveDuplicatedElem(chunkIDs) 280 | if 1 > len(chunkIDs) { 281 | chunkIDs = []string{} 282 | } 283 | return 284 | } 285 | 286 | func (s3 *S3) GetIndex(id string) (index *entity.Index, err error) { 287 | index, err = s3.repoIndex(id) 288 | if nil != err { 289 | logging.LogErrorf("get index [%s] failed: %s", id, err) 290 | return 291 | } 292 | if nil == index { 293 | err = ErrCloudObjectNotFound 294 | return 295 | } 296 | return 297 | } 298 | 299 | func (s3 *S3) GetConcurrentReqs() (ret int) { 300 | ret = s3.S3.ConcurrentReqs 301 | if 1 > ret { 302 | ret = 8 303 | } 304 | if 16 < ret { 305 | ret = 16 306 | } 307 | return 308 | } 309 | 310 | func (s3 *S3) ListObjects(pathPrefix string) (ret map[string]*entity.ObjectInfo, err error) { 311 | ret = map[string]*entity.ObjectInfo{} 312 | svc := s3.getService() 313 | 314 | endWithSlash := strings.HasSuffix(pathPrefix, "/") 315 | pathPrefix = path.Join("repo", pathPrefix) 316 | if endWithSlash { 317 | pathPrefix += "/" 318 | } 319 | limit := int32(1000) 320 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 321 | defer cancelFn() 322 | 323 | paginator := as3.NewListObjectsV2Paginator(svc, &as3.ListObjectsV2Input{ 324 | Bucket: &s3.Conf.S3.Bucket, 325 | Prefix: &pathPrefix, 326 | MaxKeys: &limit, 327 | }) 328 | 329 | for paginator.HasMorePages() { 330 | output, pErr := paginator.NextPage(ctx) 331 | if nil != pErr { 332 | logging.LogErrorf("list objects failed: %s", pErr) 333 | return nil, pErr 334 | } 335 | 336 | for _, entry := range output.Contents { 337 | filePath := strings.TrimPrefix(*entry.Key, pathPrefix) 338 | if "" == filePath { 339 | logging.LogWarnf("skip empty file path for key [%s]", *entry.Key) 340 | continue 341 | } 342 | 343 | ret[filePath] = &entity.ObjectInfo{ 344 | Path: filePath, 345 | Size: *entry.Size, 346 | } 347 | } 348 | } 349 | return 350 | } 351 | 352 | func (s3 *S3) repoIndex(id string) (ret *entity.Index, err error) { 353 | indexPath := path.Join("repo", "indexes", id) 354 | info, err := s3.statFile(indexPath) 355 | if nil != err { 356 | if s3.isErrNotFound(err) { 357 | err = nil 358 | } 359 | return 360 | } 361 | if 1 > info.Size { 362 | return 363 | } 364 | 365 | data, err := s3.DownloadObject(path.Join("indexes", id)) 366 | if nil != err { 367 | logging.LogErrorf("download index [%s] failed: %s", id, err) 368 | return 369 | } 370 | data, err = compressDecoder.DecodeAll(data, nil) 371 | if nil != err { 372 | logging.LogErrorf("decompress index [%s] failed: %s", id, err) 373 | return 374 | } 375 | ret = &entity.Index{} 376 | err = gulu.JSON.UnmarshalJSON(data, ret) 377 | return 378 | } 379 | 380 | func (s3 *S3) listRepoRefs(refPrefix string) (ret []*Ref, err error) { 381 | svc := s3.getService() 382 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 383 | defer cancelFn() 384 | 385 | prefix := path.Join("repo", "refs", refPrefix) 386 | limit := int32(32) 387 | marker := "" 388 | for { 389 | output, listErr := svc.ListObjects(ctx, &as3.ListObjectsInput{ 390 | Bucket: &s3.Conf.S3.Bucket, 391 | Prefix: &prefix, 392 | Marker: &marker, 393 | MaxKeys: &limit, 394 | }) 395 | if nil != listErr { 396 | return 397 | } 398 | 399 | marker = *output.Marker 400 | 401 | for _, entry := range output.Contents { 402 | filePath := strings.TrimPrefix(*entry.Key, "repo/") 403 | data, getErr := s3.DownloadObject(filePath) 404 | if nil != getErr { 405 | err = getErr 406 | return 407 | } 408 | 409 | id := string(data) 410 | info, statErr := s3.statFile(path.Join("repo", "indexes", id)) 411 | if nil != statErr { 412 | err = statErr 413 | return 414 | } 415 | if 1 > info.Size { 416 | continue 417 | } 418 | 419 | ret = append(ret, &Ref{ 420 | Name: path.Base(*entry.Key), 421 | ID: id, 422 | Updated: entry.LastModified.Format("2006-01-02 15:04:05"), 423 | }) 424 | } 425 | 426 | if !(*output.IsTruncated) { 427 | break 428 | } 429 | } 430 | return 431 | } 432 | 433 | func (s3 *S3) listRepos() (ret []*Repo, err error) { 434 | svc := s3.getService() 435 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 436 | defer cancelFn() 437 | 438 | output, err := svc.ListBuckets(ctx, &as3.ListBucketsInput{}) 439 | if nil != err { 440 | return 441 | } 442 | 443 | ret = []*Repo{} 444 | for _, bucket := range output.Buckets { 445 | if *bucket.Name != s3.S3.Bucket { 446 | continue 447 | } 448 | 449 | ret = append(ret, &Repo{ 450 | Name: *bucket.Name, 451 | Size: 0, 452 | Updated: (*bucket.CreationDate).Format("2006-01-02 15:04:05"), 453 | }) 454 | } 455 | sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) 456 | return 457 | } 458 | 459 | func (s3 *S3) statFile(key string) (info *objectInfo, err error) { 460 | svc := s3.getService() 461 | ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(s3.S3.Timeout)*time.Second) 462 | defer cancelFn() 463 | 464 | header, err := svc.HeadObject(ctx, &as3.HeadObjectInput{ 465 | Bucket: &s3.Conf.S3.Bucket, 466 | Key: &key, 467 | }) 468 | if nil != err { 469 | return 470 | } 471 | 472 | updated := time.Now().Format("2006-01-02 15:04:05") 473 | info = &objectInfo{Key: key, Updated: updated, Size: 0} 474 | if nil == header { 475 | logging.LogWarnf("stat file [%s] header is nil", key) 476 | return 477 | } 478 | info.Size = *header.ContentLength 479 | if 1 > info.Size { 480 | logging.LogWarnf("stat file [%s] size is [%d]", key, info.Size) 481 | } 482 | if nil == header.LastModified { 483 | logging.LogWarnf("stat file [%s] header last modified is nil", key) 484 | } else { 485 | updated = header.LastModified.Format("2006-01-02 15:04:05") 486 | } 487 | info.Updated = updated 488 | return 489 | } 490 | 491 | func (s3 *S3) getNotFound(keys []string) (ret []string, err error) { 492 | if 1 > len(keys) { 493 | return 494 | } 495 | 496 | poolSize := s3.GetConcurrentReqs() 497 | if poolSize > len(keys) { 498 | poolSize = len(keys) 499 | } 500 | 501 | waitGroup := &sync.WaitGroup{} 502 | p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { 503 | defer waitGroup.Done() 504 | key := arg.(string) 505 | info, statErr := s3.statFile(key) 506 | if nil == info || nil != statErr { 507 | ret = append(ret, key) 508 | } 509 | }) 510 | 511 | for _, key := range keys { 512 | waitGroup.Add(1) 513 | err = p.Invoke(key) 514 | if nil != err { 515 | logging.LogErrorf("invoke failed: %s", err) 516 | return 517 | } 518 | } 519 | waitGroup.Wait() 520 | p.Release() 521 | return 522 | } 523 | 524 | func (s3 *S3) getService() *as3.Client { 525 | s3.mux.Lock() 526 | defer s3.mux.Unlock() 527 | 528 | if nil != s3.service { 529 | return s3.service 530 | } 531 | 532 | cfg, err := config.LoadDefaultConfig(context.TODO()) 533 | if err != nil { 534 | logging.LogErrorf("load default config failed: %s", err) 535 | } 536 | 537 | s3.service = as3.NewFromConfig(cfg, func(o *as3.Options) { 538 | o.Credentials = aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(s3.Conf.S3.AccessKey, s3.Conf.S3.SecretKey, "")) 539 | o.BaseEndpoint = aws.String(s3.Conf.S3.Endpoint) 540 | o.Region = s3.Conf.S3.Region 541 | o.UsePathStyle = s3.Conf.S3.PathStyle 542 | o.HTTPClient = s3.HTTPClient 543 | o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired 544 | o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired 545 | 546 | // --- START: S3 Compatibility Fix for SigV4 (Cloudflare Tunnel/Proxies) --- 547 | // https://github.com/siyuan-note/siyuan/issues/16199 548 | // This fix addresses the 'SignatureDoesNotMatch' error encountered when using 549 | // S3-compatible endpoints proxied through services like Cloudflare Tunnel. 550 | // Proxies may modify headers (like Accept-Encoding), which invalidates the 551 | // AWS Signature Version 4 calculation. 552 | endpoint := strings.ToLower(s3.Conf.S3.Endpoint) 553 | 554 | // Only apply the compatibility middleware if the endpoint is NOT an official AWS S3 endpoint. 555 | if !strings.Contains(endpoint, "amazonaws.com") { 556 | // ignoreSigningHeaders and HeadersToIgnore are defined in s3_middleware.go (same package). 557 | ignoreSigningHeaders(o, HeadersToIgnore) 558 | // logging.LogDebugf("applied S3 compatibility fix for non-AWS endpoint: %s", s3.Conf.S3.Endpoint) 559 | } 560 | // --- END: S3 Compatibility Fix --- 561 | }) 562 | return s3.service 563 | } 564 | 565 | func (s3 *S3) isErrNotFound(err error) bool { 566 | var nsk *as3Types.NoSuchKey 567 | if errors.As(err, &nsk) { 568 | return true 569 | } 570 | 571 | var nf *as3Types.NotFound 572 | if errors.As(err, &nf) { 573 | return true 574 | } 575 | 576 | var apiErr smithy.APIError 577 | if errors.As(err, &apiErr) { 578 | msg := strings.ToLower(apiErr.ErrorMessage()) 579 | return strings.Contains(msg, "does not exist") || strings.Contains(msg, "404") || strings.Contains(msg, "no such file or directory") 580 | } 581 | return false 582 | } 583 | 584 | // HeadersToIgnore lists headers that frequently cause SignatureDoesNotMatch errors 585 | // when used with S3-compatible providers behind proxies (like Cloudflare Tunnel or GCS). 586 | // These headers are temporarily removed before the SigV4 signing process and restored afterwards. 587 | var HeadersToIgnore = []string{ 588 | "Accept-Encoding", // The primary culprit, often modified by proxies. 589 | "Amz-Sdk-Invocation-Id", 590 | "Amz-Sdk-Request", 591 | } 592 | 593 | type ignoredHeadersKey struct{} 594 | 595 | // ignoreSigningHeaders is a helper to inject middleware that excludes specified headers 596 | // from the Signature Version 4 calculation by temporarily removing them. 597 | // This function should be called only for non-AWS S3 endpoints. 598 | func ignoreSigningHeaders(o *as3.Options, headers []string) { 599 | o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { 600 | // 1. Insert ignoreHeaders BEFORE the "Signing" middleware 601 | if err := stack.Finalize.Insert(ignoreHeaders(headers), "Signing", middleware.Before); err != nil { 602 | return fmt.Errorf("failed to insert S3CompatIgnoreHeaders: %w", err) 603 | } 604 | 605 | // 2. Insert restoreIgnored AFTER the "Signing" middleware 606 | if err := stack.Finalize.Insert(restoreIgnored(), "Signing", middleware.After); err != nil { 607 | return fmt.Errorf("failed to insert S3CompatRestoreHeaders: %w", err) 608 | } 609 | return nil 610 | }) 611 | } 612 | 613 | // ignoreHeaders removes specified headers and stores them in context for later restoration. 614 | func ignoreHeaders(headers []string) middleware.FinalizeMiddleware { 615 | return middleware.FinalizeMiddlewareFunc( 616 | "S3CompatIgnoreHeaders", 617 | func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) { 618 | req, ok := in.Request.(*smithyhttp.Request) 619 | if !ok { 620 | return out, metadata, &asSigner.SigningError{Err: errors.New("unexpected request middleware type for ignoreHeaders")} 621 | } 622 | 623 | // Store removed headers and their values 624 | ignored := make(map[string]string, len(headers)) 625 | for _, h := range headers { 626 | // Use canonical form for map key (e.g., "Accept-Encoding") 627 | // strings.Title is necessary for older Go versions to ensure canonicalization. 628 | canonicalKey := strings.Title(strings.ToLower(h)) 629 | ignored[canonicalKey] = req.Header.Get(h) 630 | req.Header.Del(h) // Remove header before signing 631 | } 632 | 633 | // Store the ignored headers in the context 634 | ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored) 635 | return next.HandleFinalize(ctx, in) 636 | }, 637 | ) 638 | } 639 | 640 | // restoreIgnored retrieves headers from context and restores them to the request 641 | // after the signing (Finalize) and before sending. 642 | func restoreIgnored() middleware.FinalizeMiddleware { 643 | return middleware.FinalizeMiddlewareFunc( 644 | "S3CompatRestoreHeaders", 645 | func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) { 646 | req, ok := in.Request.(*smithyhttp.Request) 647 | if !ok { 648 | return out, metadata, errors.New("unexpected request middleware type for restoreIgnored") 649 | } 650 | 651 | // Execute the next Handler (which includes signing and the actual network request) 652 | out, metadata, err = next.HandleFinalize(ctx, in) 653 | 654 | // Retrieve ignored headers from the context 655 | ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string) 656 | // Restore the headers to the request 657 | for k, v := range ignored { 658 | if v != "" { 659 | req.Header.Set(k, v) 660 | } 661 | } 662 | return out, metadata, err 663 | }, 664 | ) 665 | } 666 | -------------------------------------------------------------------------------- /cloud/siyuan.go: -------------------------------------------------------------------------------- 1 | // DejaVu - Data snapshot and sync. 2 | // Copyright (c) 2022-present, b3log.org 3 | // 4 | // This program is free software: you can redistribute it and/or modify 5 | // it under the terms of the GNU Affero General Public License as published by 6 | // the Free Software Foundation, either version 3 of the License, or 7 | // (at your option) any later version. 8 | // 9 | // This program is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU Affero General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Affero General Public License 15 | // along with this program. If not, see . 16 | 17 | package cloud 18 | 19 | import ( 20 | "bytes" 21 | "context" 22 | "fmt" 23 | "os" 24 | "path" 25 | "path/filepath" 26 | "sort" 27 | "strings" 28 | "sync" 29 | "time" 30 | 31 | "github.com/88250/gulu" 32 | "github.com/qiniu/go-sdk/v7/client" 33 | "github.com/qiniu/go-sdk/v7/storage" 34 | "github.com/siyuan-note/dejavu/entity" 35 | "github.com/siyuan-note/httpclient" 36 | "github.com/siyuan-note/logging" 37 | ) 38 | 39 | var clientInit = sync.Once{} 40 | 41 | // SiYuan 描述了思源笔记官方云端存储服务实现。 42 | type SiYuan struct { 43 | *BaseCloud 44 | } 45 | 46 | func NewSiYuan(baseCloud *BaseCloud) *SiYuan { 47 | clientInit.Do(func() { 48 | client.DefaultClient = client.Client{Client: httpclient.GetCloudFileClient2Min()} 49 | storage.DefaultClient = client.DefaultClient 50 | }) 51 | return &SiYuan{BaseCloud: baseCloud} 52 | } 53 | 54 | func (siyuan *SiYuan) UploadObject(filePath string, overwrite bool) (length int64, err error) { 55 | absFilePath := filepath.Join(siyuan.Conf.RepoPath, filePath) 56 | info, err := os.Stat(absFilePath) 57 | if nil != err { 58 | logging.LogErrorf("stat failed: %s", err) 59 | return 60 | } 61 | length = info.Size() 62 | 63 | key := path.Join("siyuan", siyuan.Conf.UserID, "repo", siyuan.Conf.Dir, filePath) 64 | keyUploadToken, scopeUploadToken, err := siyuan.requestScopeKeyUploadToken(key, overwrite) 65 | if nil != err { 66 | return 67 | } 68 | 69 | uploadToken := keyUploadToken 70 | if !overwrite { 71 | uploadToken = scopeUploadToken 72 | } 73 | 74 | formUploader := storage.NewFormUploader(&storage.Config{UseHTTPS: true}) 75 | ret := storage.PutRet{} 76 | err = formUploader.PutFile(context.Background(), &ret, uploadToken, key, absFilePath, nil) 77 | if nil != err { 78 | if msg := fmt.Sprintf("%s", err); strings.Contains(msg, "file exists") { 79 | err = nil 80 | return 81 | } 82 | 83 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 84 | if e, ok := err.(*client.ErrorInfo); ok { 85 | if 614 == e.Code || strings.Contains(e.Err, "file exists") { 86 | err = nil 87 | return 88 | } 89 | logging.LogErrorf("error detail: %s", e.ErrorDetail()) 90 | } 91 | 92 | time.Sleep(1 * time.Second) 93 | err = formUploader.PutFile(context.Background(), &ret, uploadToken, key, absFilePath, nil) 94 | if nil != err { 95 | if msg := fmt.Sprintf("%s", err); strings.Contains(msg, "file exists") { 96 | err = nil 97 | return 98 | } 99 | 100 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 101 | if e, ok := err.(*client.ErrorInfo); ok { 102 | if 614 == e.Code || strings.Contains(e.Err, "file exists") { 103 | err = nil 104 | return 105 | } 106 | 107 | logging.LogErrorf("error detail: %s", e.ErrorDetail()) 108 | } 109 | } 110 | return 111 | } 112 | 113 | //logging.LogInfof("uploaded object [%s]", key) 114 | return 115 | } 116 | 117 | func (siyuan *SiYuan) UploadBytes(filePath string, data []byte, overwrite bool) (length int64, err error) { 118 | length = int64(len(data)) 119 | 120 | key := path.Join("siyuan", siyuan.Conf.UserID, "repo", siyuan.Conf.Dir, filePath) 121 | keyUploadToken, scopeUploadToken, err := siyuan.requestScopeKeyUploadToken(key, overwrite) 122 | if nil != err { 123 | return 124 | } 125 | 126 | uploadToken := keyUploadToken 127 | if !overwrite { 128 | uploadToken = scopeUploadToken 129 | } 130 | 131 | formUploader := storage.NewFormUploader(&storage.Config{UseHTTPS: true}) 132 | ret := storage.PutRet{} 133 | err = formUploader.Put(context.Background(), &ret, uploadToken, key, bytes.NewReader(data), length, &storage.PutExtra{}) 134 | if nil != err { 135 | if msg := fmt.Sprintf("%s", err); strings.Contains(msg, "file exists") { 136 | err = nil 137 | return 138 | } 139 | 140 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 141 | if e, ok := err.(*client.ErrorInfo); ok { 142 | if 614 == e.Code || strings.Contains(e.Err, "file exists") { 143 | err = nil 144 | return 145 | } 146 | logging.LogErrorf("error detail: %s", e.ErrorDetail()) 147 | } 148 | 149 | time.Sleep(1 * time.Second) 150 | err = formUploader.Put(context.Background(), &ret, uploadToken, key, bytes.NewReader(data), length, &storage.PutExtra{}) 151 | if nil != err { 152 | if msg := fmt.Sprintf("%s", err); strings.Contains(msg, "file exists") { 153 | err = nil 154 | return 155 | } 156 | 157 | logging.LogErrorf("upload object [%s] failed: %s", key, err) 158 | if e, ok := err.(*client.ErrorInfo); ok { 159 | if 614 == e.Code || strings.Contains(e.Err, "file exists") { 160 | err = nil 161 | return 162 | } 163 | 164 | logging.LogErrorf("error detail: %s", e.ErrorDetail()) 165 | } 166 | } 167 | return 168 | } 169 | 170 | //logging.LogInfof("uploaded object [%s]", key) 171 | return 172 | } 173 | 174 | func (siyuan *SiYuan) DownloadObject(filePath string) (ret []byte, err error) { 175 | key := path.Join("siyuan", siyuan.Conf.UserID, "repo", siyuan.Conf.Dir, filePath) 176 | resp, err := httpclient.NewCloudFileRequest2m().Get(siyuan.Endpoint + key) 177 | if nil != err { 178 | err = fmt.Errorf("download object [%s] failed: %s", key, err) 179 | return 180 | } 181 | if 200 != resp.StatusCode { 182 | if 404 == resp.StatusCode { 183 | err = ErrCloudObjectNotFound 184 | return 185 | } 186 | err = fmt.Errorf("download object [%s] failed [%d]", key, resp.StatusCode) 187 | return 188 | } 189 | 190 | ret, err = resp.ToBytes() 191 | if nil != err { 192 | err = fmt.Errorf("download read data failed: %s", err) 193 | return 194 | } 195 | 196 | //logging.LogInfof("downloaded object [%s]", key) 197 | return 198 | } 199 | 200 | func (siyuan *SiYuan) RemoveObject(filePath string) (err error) { 201 | userId := siyuan.Conf.UserID 202 | dir := siyuan.Conf.Dir 203 | token := siyuan.Conf.Token 204 | server := siyuan.Conf.Server 205 | 206 | key := path.Join("siyuan", userId, "repo", dir, filePath) 207 | result := gulu.Ret.NewResult() 208 | request := httpclient.NewCloudRequest30s() 209 | resp, err := request. 210 | SetSuccessResult(&result). 211 | SetBody(map[string]string{"repo": dir, "token": token, "key": key}). 212 | Post(server + "/apis/siyuan/dejavu/removeRepoObject?uid=" + userId) 213 | if nil != err { 214 | return 215 | } 216 | 217 | if 200 != resp.StatusCode { 218 | if 401 == resp.StatusCode { 219 | err = ErrCloudAuthFailed 220 | return 221 | } 222 | err = fmt.Errorf("remove cloud repo object failed [%d]", resp.StatusCode) 223 | return 224 | } 225 | 226 | if 0 != result.Code { 227 | err = fmt.Errorf("remove cloud repo object failed: %s", result.Msg) 228 | return 229 | } 230 | 231 | //logging.LogInfof("removed object [%s]", key) 232 | return 233 | } 234 | 235 | func (siyuan *SiYuan) ListObjects(pathPrefix string) (objInfos map[string]*entity.ObjectInfo, err error) { 236 | objInfos = map[string]*entity.ObjectInfo{} 237 | 238 | token := siyuan.Conf.Token 239 | dir := siyuan.Conf.Dir 240 | userId := siyuan.Conf.UserID 241 | server := siyuan.Conf.Server 242 | 243 | result := gulu.Ret.NewResult() 244 | request := httpclient.NewCloudRequest30s() 245 | resp, err := request. 246 | SetSuccessResult(&result). 247 | SetBody(map[string]string{"repo": dir, "token": token, "pathPrefix": pathPrefix}). 248 | Post(server + "/apis/siyuan/dejavu/listRepoObjects?uid=" + userId) 249 | if nil != err { 250 | err = fmt.Errorf("list cloud repo objects failed: %s", err) 251 | return 252 | } 253 | 254 | if 200 != resp.StatusCode { 255 | if 401 == resp.StatusCode { 256 | err = ErrCloudAuthFailed 257 | return 258 | } 259 | err = fmt.Errorf("list cloud repo objects failed [%d]", resp.StatusCode) 260 | return 261 | } 262 | 263 | if 0 != result.Code { 264 | err = fmt.Errorf("list cloud repo objects failed: %s", result.Msg) 265 | return 266 | } 267 | 268 | retData := result.Data.(map[string]interface{}) 269 | retObjects := retData["objects"].([]interface{}) 270 | for _, retObj := range retObjects { 271 | data, marshalErr := gulu.JSON.MarshalJSON(retObj) 272 | if nil != marshalErr { 273 | logging.LogErrorf("marshal obj failed: %s", marshalErr) 274 | continue 275 | } 276 | obj := &entity.ObjectInfo{} 277 | if unmarshalErr := gulu.JSON.UnmarshalJSON(data, obj); nil != unmarshalErr { 278 | logging.LogErrorf("unmarshal obj failed: %s", unmarshalErr) 279 | continue 280 | } 281 | objInfos[obj.Path] = obj 282 | } 283 | return 284 | } 285 | 286 | func (siyuan *SiYuan) GetTags() (tags []*Ref, err error) { 287 | token := siyuan.Conf.Token 288 | dir := siyuan.Conf.Dir 289 | userId := siyuan.Conf.UserID 290 | server := siyuan.Conf.Server 291 | 292 | result := gulu.Ret.NewResult() 293 | request := httpclient.NewCloudRequest30s() 294 | resp, err := request. 295 | SetSuccessResult(&result). 296 | SetBody(map[string]string{"repo": dir, "token": token}). 297 | Post(server + "/apis/siyuan/dejavu/getRepoTags?uid=" + userId) 298 | if nil != err { 299 | err = fmt.Errorf("get cloud repo tags failed: %s", err) 300 | return 301 | } 302 | 303 | if 200 != resp.StatusCode { 304 | if 401 == resp.StatusCode { 305 | err = ErrCloudAuthFailed 306 | return 307 | } 308 | err = fmt.Errorf("get cloud repo tags failed [%d]", resp.StatusCode) 309 | return 310 | } 311 | 312 | if 0 != result.Code { 313 | err = fmt.Errorf("get cloud repo tags failed: %s", result.Msg) 314 | return 315 | } 316 | 317 | retData := result.Data.(map[string]interface{}) 318 | retTags := retData["tags"].([]interface{}) 319 | for _, retTag := range retTags { 320 | data, marshalErr := gulu.JSON.MarshalJSON(retTag) 321 | if nil != marshalErr { 322 | logging.LogErrorf("marshal tag failed: %s", marshalErr) 323 | continue 324 | } 325 | tag := &Ref{} 326 | if unmarshalErr := gulu.JSON.UnmarshalJSON(data, tag); nil != unmarshalErr { 327 | logging.LogErrorf("unmarshal tag failed: %s", unmarshalErr) 328 | continue 329 | } 330 | tags = append(tags, tag) 331 | } 332 | return 333 | } 334 | 335 | func (siyuan *SiYuan) GetIndexes(page int) (indexes []*entity.Index, pageCount, totalCount int, err error) { 336 | token := siyuan.Conf.Token 337 | dir := siyuan.Conf.Dir 338 | userId := siyuan.Conf.UserID 339 | server := siyuan.Conf.Server 340 | 341 | result := gulu.Ret.NewResult() 342 | request := httpclient.NewCloudRequest30s() 343 | resp, err := request. 344 | SetSuccessResult(&result). 345 | SetBody(map[string]interface{}{"repo": dir, "token": token, "page": page}). 346 | Post(server + "/apis/siyuan/dejavu/getRepoIndexes?uid=" + userId) 347 | if nil != err { 348 | err = fmt.Errorf("get cloud repo indexes failed: %s", err) 349 | return 350 | } 351 | 352 | if 200 != resp.StatusCode { 353 | if 401 == resp.StatusCode { 354 | err = ErrCloudAuthFailed 355 | return 356 | } 357 | err = fmt.Errorf("get cloud repo indexes failed [%d]", resp.StatusCode) 358 | return 359 | } 360 | 361 | if 0 != result.Code { 362 | err = fmt.Errorf("get cloud repo indexes failed: %s", result.Msg) 363 | return 364 | } 365 | 366 | retData := result.Data.(map[string]interface{}) 367 | retIndexes := retData["indexes"].([]interface{}) 368 | for _, retIndex := range retIndexes { 369 | data, marshalErr := gulu.JSON.MarshalJSON(retIndex) 370 | if nil != marshalErr { 371 | logging.LogErrorf("marshal index failed: %s", marshalErr) 372 | continue 373 | } 374 | index := &entity.Index{} 375 | if unmarshalErr := gulu.JSON.UnmarshalJSON(data, index); nil != unmarshalErr { 376 | logging.LogErrorf("unmarshal index failed: %s", unmarshalErr) 377 | continue 378 | } 379 | indexes = append(indexes, index) 380 | } 381 | 382 | pageCount = int(retData["pageCount"].(float64)) 383 | totalCount = int(retData["totalCount"].(float64)) 384 | return 385 | } 386 | 387 | func (siyuan *SiYuan) GetRefsFiles() (fileIDs []string, refs []*Ref, err error) { 388 | token := siyuan.Conf.Token 389 | dir := siyuan.Conf.Dir 390 | userId := siyuan.Conf.UserID 391 | server := siyuan.Conf.Server 392 | 393 | result := gulu.Ret.NewResult() 394 | request := httpclient.NewCloudFileRequest2m() 395 | resp, err := request. 396 | SetSuccessResult(&result). 397 | SetBody(map[string]string{"repo": dir, "token": token}). 398 | Post(server + "/apis/siyuan/dejavu/getRepoRefsFiles?uid=" + userId) 399 | if nil != err { 400 | err = fmt.Errorf("get cloud repo refs files failed: %s", err) 401 | return 402 | } 403 | 404 | if 200 != resp.StatusCode { 405 | if 401 == resp.StatusCode { 406 | err = ErrCloudAuthFailed 407 | return 408 | } 409 | err = fmt.Errorf("get cloud repo refs files failed [%d]", resp.StatusCode) 410 | return 411 | } 412 | 413 | if 0 != result.Code { 414 | err = fmt.Errorf("get cloud repo refs files failed: %s", result.Msg) 415 | return 416 | } 417 | 418 | retData := result.Data.(map[string]interface{}) 419 | retFiles := retData["files"].([]interface{}) 420 | for _, retFile := range retFiles { 421 | fileIDs = append(fileIDs, retFile.(string)) 422 | } 423 | retRefs := retData["refs"].([]interface{}) 424 | for _, retRef := range retRefs { 425 | data, marshalErr := gulu.JSON.MarshalJSON(retRef) 426 | if nil != marshalErr { 427 | logging.LogErrorf("marshal ref failed: %s", marshalErr) 428 | continue 429 | } 430 | ref := &Ref{} 431 | if unmarshalErr := gulu.JSON.UnmarshalJSON(data, ref); nil != unmarshalErr { 432 | logging.LogErrorf("unmarshal ref failed: %s", unmarshalErr) 433 | continue 434 | } 435 | refs = append(refs, ref) 436 | } 437 | return 438 | } 439 | 440 | func (siyuan *SiYuan) GetChunks(excludeChunkIDs []string) (chunkIDs []string, err error) { 441 | if 1 > len(excludeChunkIDs) { 442 | return 443 | } 444 | 445 | token := siyuan.Conf.Token 446 | dir := siyuan.Conf.Dir 447 | userId := siyuan.Conf.UserID 448 | server := siyuan.Conf.Server 449 | 450 | result := gulu.Ret.NewResult() 451 | request := httpclient.NewCloudFileRequest2m() 452 | resp, err := request. 453 | SetSuccessResult(&result). 454 | SetBody(map[string]interface{}{"repo": dir, "token": token, "chunks": excludeChunkIDs}). 455 | Post(server + "/apis/siyuan/dejavu/getRepoUploadChunks?uid=" + userId) 456 | if nil != err { 457 | return 458 | } 459 | 460 | if 200 != resp.StatusCode { 461 | if 401 == resp.StatusCode { 462 | err = ErrCloudAuthFailed 463 | return 464 | } 465 | err = fmt.Errorf("get cloud repo refs chunks failed [%d]", resp.StatusCode) 466 | return 467 | } 468 | 469 | if 0 != result.Code { 470 | err = fmt.Errorf("get cloud repo refs chunks failed: %s", result.Msg) 471 | return 472 | } 473 | 474 | retData := result.Data.(map[string]interface{}) 475 | retChunks := retData["chunks"].([]interface{}) 476 | for _, retChunk := range retChunks { 477 | chunkIDs = append(chunkIDs, retChunk.(string)) 478 | } 479 | return 480 | } 481 | 482 | func (siyuan *SiYuan) GetStat() (stat *Stat, err error) { 483 | token := siyuan.Conf.Token 484 | dir := siyuan.Conf.Dir 485 | userId := siyuan.Conf.UserID 486 | server := siyuan.Conf.Server 487 | 488 | result := gulu.Ret.NewResult() 489 | request := httpclient.NewCloudRequest30s() 490 | resp, err := request. 491 | SetSuccessResult(&result). 492 | SetBody(map[string]string{"repo": dir, "token": token}). 493 | Post(server + "/apis/siyuan/dejavu/getRepoStat?uid=" + userId) 494 | if nil != err { 495 | err = fmt.Errorf("get cloud repo stat failed: %s", err) 496 | return 497 | } 498 | 499 | if 200 != resp.StatusCode { 500 | if 401 == resp.StatusCode { 501 | err = ErrCloudAuthFailed 502 | return 503 | } 504 | err = fmt.Errorf("get cloud repo stat failed [%d]", resp.StatusCode) 505 | return 506 | } 507 | 508 | if 0 != result.Code { 509 | err = fmt.Errorf("get cloud repo stat failed: %s", result.Msg) 510 | return 511 | } 512 | 513 | data, marshalErr := gulu.JSON.MarshalJSON(result.Data) 514 | if nil != marshalErr { 515 | err = fmt.Errorf("marshal stat failed: %s", marshalErr) 516 | return 517 | } 518 | stat = &Stat{} 519 | if unmarshalErr := gulu.JSON.UnmarshalJSON(data, stat); nil != unmarshalErr { 520 | err = fmt.Errorf("unmarshal stat failed: %s", unmarshalErr) 521 | return 522 | } 523 | return 524 | } 525 | 526 | func (siyuan *SiYuan) AddTraffic(traffic *Traffic) { 527 | if nil == traffic { 528 | return 529 | } 530 | 531 | if 0 == traffic.UploadBytes && 0 == traffic.DownloadBytes && 0 == traffic.APIGet && 0 == traffic.APIPut { 532 | return 533 | } 534 | 535 | token := siyuan.Conf.Token 536 | server := siyuan.Conf.Server 537 | 538 | request := httpclient.NewCloudRequest30s() 539 | resp, err := request. 540 | SetBody(map[string]interface{}{ 541 | "token": token, 542 | "uploadBytes": traffic.UploadBytes, 543 | "downloadBytes": traffic.DownloadBytes, 544 | "apiGet": traffic.APIGet, 545 | "apiPut": traffic.APIPut, 546 | }). 547 | Post(server + "/apis/siyuan/dejavu/addTraffic") 548 | if nil != err { 549 | logging.LogErrorf("add traffic failed: %s", err) 550 | return 551 | } 552 | 553 | if 200 != resp.StatusCode { 554 | logging.LogErrorf("add traffic failed: %d", resp.StatusCode) 555 | return 556 | } 557 | return 558 | } 559 | 560 | func (siyuan *SiYuan) RemoveRepo(name string) (err error) { 561 | token := siyuan.Conf.Token 562 | server := siyuan.Conf.Server 563 | 564 | request := httpclient.NewCloudFileRequest2m() 565 | resp, err := request. 566 | SetBody(map[string]string{"name": name, "token": token}). 567 | Post(server + "/apis/siyuan/dejavu/removeRepo") 568 | if nil != err { 569 | err = fmt.Errorf("remove cloud repo failed: %s", err) 570 | return 571 | } 572 | 573 | if 200 != resp.StatusCode { 574 | if 401 == resp.StatusCode { 575 | err = ErrCloudAuthFailed 576 | return 577 | } 578 | err = fmt.Errorf("remove cloud repo failed [%d]", resp.StatusCode) 579 | return 580 | } 581 | return 582 | } 583 | 584 | func (siyuan *SiYuan) CreateRepo(name string) (err error) { 585 | token := siyuan.Conf.Token 586 | server := siyuan.Conf.Server 587 | 588 | result := map[string]interface{}{} 589 | request := httpclient.NewCloudRequest30s() 590 | resp, err := request. 591 | SetSuccessResult(&result). 592 | SetBody(map[string]string{"name": name, "token": token}). 593 | Post(server + "/apis/siyuan/dejavu/createRepo") 594 | if nil != err { 595 | err = fmt.Errorf("create cloud repo failed: %s", err) 596 | return 597 | } 598 | 599 | if 200 != resp.StatusCode { 600 | if 401 == resp.StatusCode { 601 | err = ErrCloudAuthFailed 602 | return 603 | } 604 | err = fmt.Errorf("create cloud repo failed [%d]", resp.StatusCode) 605 | return 606 | } 607 | 608 | code := result["code"].(float64) 609 | if 0 != code { 610 | err = fmt.Errorf("create cloud repo failed: %s", result["msg"]) 611 | return 612 | } 613 | return 614 | } 615 | 616 | func (siyuan *SiYuan) GetRepos() (repos []*Repo, size int64, err error) { 617 | token := siyuan.Conf.Token 618 | server := siyuan.Conf.Server 619 | userId := siyuan.Conf.UserID 620 | 621 | result := map[string]interface{}{} 622 | request := httpclient.NewCloudRequest30s() 623 | resp, err := request. 624 | SetBody(map[string]interface{}{"token": token}). 625 | SetSuccessResult(&result). 626 | Post(server + "/apis/siyuan/dejavu/getRepos?uid=" + userId) 627 | if nil != err { 628 | err = fmt.Errorf("get cloud repos failed: %s", err) 629 | return 630 | } 631 | 632 | if 200 != resp.StatusCode { 633 | if 401 == resp.StatusCode { 634 | err = ErrCloudAuthFailed 635 | return 636 | } 637 | err = fmt.Errorf("request cloud repo list failed [%d]", resp.StatusCode) 638 | return 639 | } 640 | 641 | code := result["code"].(float64) 642 | if 0 != code { 643 | err = fmt.Errorf("request cloud repo list failed: %s", result["msg"].(string)) 644 | return 645 | } 646 | 647 | retData := result["data"].(map[string]interface{}) 648 | retRepos := retData["repos"].([]interface{}) 649 | for _, d := range retRepos { 650 | data, marshalErr := gulu.JSON.MarshalJSON(d) 651 | if nil != marshalErr { 652 | logging.LogErrorf("marshal repo failed: %s", marshalErr) 653 | continue 654 | } 655 | repo := &Repo{} 656 | if unmarshalErr := gulu.JSON.UnmarshalJSON(data, repo); nil != unmarshalErr { 657 | logging.LogErrorf("unmarshal repo failed: %s", unmarshalErr) 658 | continue 659 | } 660 | 661 | repos = append(repos, repo) 662 | } 663 | if 1 > len(repos) { 664 | repos = []*Repo{} 665 | } 666 | sort.Slice(repos, func(i, j int) bool { return repos[i].Name < repos[j].Name }) 667 | size = int64(retData["size"].(float64)) 668 | return 669 | } 670 | 671 | type UploadToken struct { 672 | key, token string 673 | expired int64 674 | } 675 | 676 | var ( 677 | keyUploadTokenMap = map[string]*UploadToken{} 678 | scopeUploadTokenMap = map[string]*UploadToken{} 679 | uploadTokenMapLock = &sync.Mutex{} 680 | ) 681 | 682 | func (siyuan *SiYuan) requestScopeKeyUploadToken(key string, overwrite bool) (keyToken, scopeToken string, err error) { 683 | userId := siyuan.Conf.UserID 684 | now := time.Now().UnixMilli() 685 | keyPrefix := path.Join("siyuan", userId) 686 | 687 | uploadTokenMapLock.Lock() 688 | cachedKeyToken := keyUploadTokenMap[key] 689 | if nil != cachedKeyToken { 690 | if now < cachedKeyToken.expired { 691 | keyToken = cachedKeyToken.token 692 | } else { 693 | delete(keyUploadTokenMap, key) 694 | } 695 | } 696 | if overwrite && "" != keyToken { 697 | uploadTokenMapLock.Unlock() 698 | return 699 | } 700 | 701 | cachedScopeToken := scopeUploadTokenMap[keyPrefix] 702 | if nil != cachedScopeToken { 703 | if now < cachedScopeToken.expired { 704 | scopeToken = cachedScopeToken.token 705 | } else { 706 | delete(scopeUploadTokenMap, keyPrefix) 707 | } 708 | } 709 | if !overwrite && "" != scopeToken { 710 | uploadTokenMapLock.Unlock() 711 | return 712 | } 713 | uploadTokenMapLock.Unlock() 714 | 715 | token := siyuan.Conf.Token 716 | server := siyuan.Conf.Server 717 | var result map[string]interface{} 718 | req := httpclient.NewCloudRequest30s().SetSuccessResult(&result) 719 | req.SetBody(map[string]interface{}{ 720 | "token": token, 721 | "key": key, 722 | "keyPrefix": keyPrefix, 723 | "time": now, // 数据同步加入系统时间校验 https://github.com/siyuan-note/siyuan/issues/7669 724 | }) 725 | resp, err := req.Post(server + "/apis/siyuan/dejavu/getRepoScopeKeyUploadToken?uid=" + userId) 726 | if nil != err { 727 | err = fmt.Errorf("request repo upload token failed: %s", err) 728 | return 729 | } 730 | 731 | if 200 != resp.StatusCode { 732 | if 401 == resp.StatusCode { 733 | err = ErrCloudAuthFailed 734 | return 735 | } 736 | err = fmt.Errorf("request repo upload token failed [%d]", resp.StatusCode) 737 | return 738 | } 739 | 740 | code := result["code"].(float64) 741 | if 0 != code { 742 | msg := result["msg"].(string) 743 | err = fmt.Errorf("request repo upload token failed: %s", msg) 744 | switch code { 745 | case 1: 746 | err = ErrSystemTimeIncorrect 747 | case 2: 748 | err = ErrDeprecatedVersion 749 | case -1: 750 | err = ErrCloudCheckFailed 751 | } 752 | return 753 | } 754 | 755 | resultData := result["data"].(map[string]interface{}) 756 | keyToken = resultData["keyToken"].(string) 757 | scopeToken = resultData["scopeToken"].(string) 758 | expired := now + 1000*60*60*24 - 60*1000 759 | uploadTokenMapLock.Lock() 760 | keyUploadTokenMap[key] = &UploadToken{ 761 | key: key, 762 | token: keyToken, 763 | expired: expired, 764 | } 765 | scopeUploadTokenMap[keyPrefix] = &UploadToken{ 766 | key: keyPrefix, 767 | token: scopeToken, 768 | expired: expired, 769 | } 770 | uploadTokenMapLock.Unlock() 771 | return 772 | } 773 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published by 637 | the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | --------------------------------------------------------------------------------