├── .editorconfig ├── .github └── workflows │ └── go_build.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README-zh_CN.md ├── README.md ├── cmd ├── main.go ├── wire.go └── wire_gen.go ├── config ├── config-offline.yaml └── config.yaml ├── docker ├── Dockerfile ├── Dockerfile-simple └── docker-compose.yaml ├── go.mod ├── go.sum ├── internal ├── dao │ ├── dao.go │ ├── file_dao.go │ └── meta_dao.go ├── data │ └── data.go ├── downloader │ ├── bitset.go │ ├── broadcaster.go │ ├── cache_task.go │ ├── downloader.go │ ├── file.go │ ├── file_manager.go │ ├── file_test.go │ ├── remote_task.go │ └── resp_notice.go ├── handler │ ├── file_handler.go │ ├── handler.go │ ├── meta_handler.go │ └── sys_handler.go ├── model │ └── sysinfo.go ├── router │ ├── http_router.go │ └── router.go ├── server │ ├── http.go │ ├── server.go │ └── templates │ │ └── repos.html └── service │ ├── file_service.go │ ├── meta_service.go │ ├── service.go │ └── sys_service.go ├── pkg ├── app │ ├── app.go │ ├── context.go │ └── options.go ├── common │ ├── common.go │ ├── pool.go │ └── safe_map.go ├── config │ └── config.go ├── consts │ └── const.go ├── error │ └── error.go ├── logger │ └── log.go ├── middleware │ └── queue_limit.go ├── prom │ └── prometheus.go ├── server │ └── server.go └── util │ ├── compress.go │ ├── http_util.go │ ├── repo_util.go │ ├── response.go │ └── util.go ├── png ├── architecture_en.png ├── downloading_models_en.png ├── img.png ├── img_download.png ├── img_store.png └── storing_models_en.png └── repair └── data_repair.go /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = false 9 | max_line_length = 120 10 | tab_width = 4 11 | ij_continuation_indent_size = 8 12 | ij_formatter_off_tag = @formatter:off 13 | ij_formatter_on_tag = @formatter:on 14 | ij_formatter_tags_enabled = true 15 | ij_smart_tabs = false 16 | ij_visual_guides = none 17 | ij_wrap_on_typing = false 18 | 19 | [*.proto] 20 | ij_continuation_indent_size = 4 21 | ij_protobuf_keep_blank_lines_in_code = 2 22 | ij_protobuf_keep_indents_on_empty_lines = false 23 | ij_protobuf_keep_line_breaks = true 24 | ij_protobuf_space_after_comma = true 25 | ij_protobuf_space_before_comma = false 26 | ij_protobuf_spaces_around_assignment_operators = true 27 | ij_protobuf_spaces_within_braces = false 28 | ij_protobuf_spaces_within_brackets = false 29 | 30 | [{*.bash,*.sh,*.zsh}] 31 | ij_shell_binary_ops_start_line = false 32 | ij_shell_keep_column_alignment_padding = false 33 | ij_shell_minify_program = false 34 | ij_shell_redirect_followed_by_space = false 35 | ij_shell_switch_cases_indented = false 36 | ij_shell_use_unix_line_separator = true 37 | 38 | [{*.go,*.go2}] 39 | indent_size = 4 40 | indent_style = tab 41 | ij_continuation_indent_size = 4 42 | ij_go_GROUP_CURRENT_PROJECT_IMPORTS = false 43 | ij_go_add_leading_space_to_comments = true 44 | ij_go_add_parentheses_for_single_import = false 45 | ij_go_call_parameters_new_line_after_left_paren = true 46 | ij_go_call_parameters_right_paren_on_new_line = true 47 | ij_go_call_parameters_wrap = off 48 | ij_go_fill_paragraph_width = 80 49 | ij_go_group_stdlib_imports = true 50 | ij_go_import_sorting = goimports 51 | ij_go_keep_indents_on_empty_lines = false 52 | ij_go_local_group_mode = project 53 | ij_go_move_all_imports_in_one_declaration = true 54 | ij_go_move_all_stdlib_imports_in_one_group = true 55 | ij_go_remove_redundant_import_aliases = false 56 | ij_go_run_go_fmt_on_reformat = true 57 | ij_go_use_back_quotes_for_imports = false 58 | ij_go_wrap_comp_lit = off 59 | ij_go_wrap_comp_lit_newline_after_lbrace = true 60 | ij_go_wrap_comp_lit_newline_before_rbrace = true 61 | ij_go_wrap_func_params = off 62 | ij_go_wrap_func_params_newline_after_lparen = true 63 | ij_go_wrap_func_params_newline_before_rparen = true 64 | ij_go_wrap_func_result = off 65 | ij_go_wrap_func_result_newline_after_lparen = true 66 | ij_go_wrap_func_result_newline_before_rparen = true 67 | 68 | [{*.har,*.jsb2,*.jsb3,*.json,.babelrc,.eslintrc,.stylelintrc,bowerrc,jest.config}] 69 | indent_size = 2 70 | ij_json_array_wrapping = split_into_lines 71 | ij_json_keep_blank_lines_in_code = 0 72 | ij_json_keep_indents_on_empty_lines = false 73 | ij_json_keep_line_breaks = true 74 | ij_json_keep_trailing_comma = false 75 | ij_json_object_wrapping = split_into_lines 76 | ij_json_property_alignment = do_not_align 77 | ij_json_space_after_colon = true 78 | ij_json_space_after_comma = true 79 | ij_json_space_before_colon = false 80 | ij_json_space_before_comma = false 81 | ij_json_spaces_within_braces = false 82 | ij_json_spaces_within_brackets = false 83 | ij_json_wrap_long_lines = false 84 | 85 | [{*.markdown,*.md}] 86 | ij_markdown_force_one_space_after_blockquote_symbol = true 87 | ij_markdown_force_one_space_after_header_symbol = true 88 | ij_markdown_force_one_space_after_list_bullet = true 89 | ij_markdown_force_one_space_between_words = true 90 | ij_markdown_format_tables = true 91 | ij_markdown_insert_quote_arrows_on_wrap = true 92 | ij_markdown_keep_indents_on_empty_lines = false 93 | ij_markdown_keep_line_breaks_inside_text_blocks = true 94 | ij_markdown_max_lines_around_block_elements = 1 95 | ij_markdown_max_lines_around_header = 1 96 | ij_markdown_max_lines_between_paragraphs = 1 97 | ij_markdown_min_lines_around_block_elements = 1 98 | ij_markdown_min_lines_around_header = 1 99 | ij_markdown_min_lines_between_paragraphs = 1 100 | ij_markdown_wrap_text_if_long = true 101 | ij_markdown_wrap_text_inside_blockquotes = true 102 | 103 | [{*.yaml,*.yml}] 104 | ij_yaml_align_values_properties = do_not_align 105 | ij_yaml_autoinsert_sequence_marker = true 106 | ij_yaml_block_mapping_on_new_line = false 107 | ij_yaml_indent_sequence_value = true 108 | ij_yaml_keep_indents_on_empty_lines = false 109 | ij_yaml_keep_line_breaks = true 110 | ij_yaml_sequence_on_new_line = false 111 | ij_yaml_space_before_colon = false 112 | ij_yaml_spaces_within_braces = true 113 | ij_yaml_spaces_within_brackets = true 114 | -------------------------------------------------------------------------------- /.github/workflows/go_build.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v4 21 | with: 22 | go-version: '1.24.1' 23 | 24 | - name: Build 25 | run: go build -v ./... 26 | 27 | - name: Test 28 | run: go test -v ./... 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### JetBrains template 2 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 3 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 4 | 5 | # User-specific stuff 6 | .idea/**/workspace.xml 7 | .idea/**/tasks.xml 8 | .idea/**/usage.statistics.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # AWS User-specific 13 | .idea/**/aws.xml 14 | 15 | # Generated files 16 | .idea/**/contentModel.xml 17 | 18 | # Sensitive or high-churn files 19 | .idea/**/dataSources/ 20 | .idea/**/dataSources.ids 21 | .idea/**/dataSources.local.xml 22 | .idea/**/sqlDataSources.xml 23 | .idea/**/dynamic.xml 24 | .idea/**/uiDesigner.xml 25 | .idea/**/dbnavigator.xml 26 | 27 | # Gradle 28 | .idea/**/gradle.xml 29 | .idea/**/libraries 30 | 31 | # Gradle and Maven with auto-import 32 | # When using Gradle or Maven with auto-import, you should exclude module files, 33 | # since they will be recreated, and may cause churn. Uncomment if using 34 | # auto-import. 35 | # .idea/artifacts 36 | # .idea/compiler.xml 37 | # .idea/jarRepositories.xml 38 | # .idea/modules.xml 39 | # .idea/*.iml 40 | # .idea/modules 41 | # *.iml 42 | # *.ipr 43 | log 44 | # CMake 45 | cmake-build-*/ 46 | 47 | # Mongo Explorer plugin 48 | .idea/**/mongoSettings.xml 49 | 50 | # File-based project format 51 | *.iws 52 | 53 | # IntelliJ 54 | out/ 55 | bin 56 | repos 57 | # mpeltonen/sbt-idea plugin 58 | .idea_modules/ 59 | 60 | # JIRA plugin 61 | atlassian-ide-plugin.xml 62 | 63 | # Cursive Clojure plugin 64 | .idea/replstate.xml 65 | 66 | # SonarLint plugin 67 | .idea/sonarlint/ 68 | 69 | # Crashlytics plugin (for Android Studio and IntelliJ) 70 | com_crashlytics_export_strings.xml 71 | crashlytics.properties 72 | crashlytics-build.properties 73 | fabric.properties 74 | 75 | # Editor-based Rest Client 76 | .idea/httpRequests 77 | 78 | # Android studio 3.1+ serialized cache file 79 | .idea/caches/build_file_checksums.ser 80 | 81 | ### Example user template template 82 | ### Example user template 83 | 84 | # IntelliJ project files 85 | .idea 86 | *.iml 87 | out 88 | gen 89 | ### Go template 90 | # If you prefer the allow list template instead of the deny list, see community template: 91 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 92 | # 93 | # Binaries for programs and plugins 94 | *.exe 95 | *.exe~ 96 | *.dll 97 | *.so 98 | *.dylib 99 | 100 | # Test binary, built with `go test -c` 101 | *.test 102 | 103 | # Output of the go coverage tool, specifically when used with LiteIDE 104 | *.out 105 | 106 | # Dependency directories (remove the comment below to include it) 107 | # vendor/ 108 | 109 | # Go workspace file 110 | go.work 111 | 112 | ### Windows template 113 | # Windows thumbnail cache files 114 | Thumbs.db 115 | Thumbs.db:encryptable 116 | ehthumbs.db 117 | ehthumbs_vista.db 118 | 119 | # Dump file 120 | *.stackdump 121 | 122 | # Folder config file 123 | [Dd]esktop.ini 124 | 125 | # Recycle Bin used on file shares 126 | $RECYCLE.BIN/ 127 | 128 | # Windows Installer files 129 | *.cab 130 | *.msi 131 | *.msix 132 | *.msm 133 | *.msp 134 | 135 | # Windows shortcuts 136 | *.lnk 137 | 138 | ### GoLand template 139 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 140 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 141 | 142 | # Gradle and Maven with auto-import 143 | # When using Gradle or Maven with auto-import, you should exclude module files, 144 | # since they will be recreated, and may cause churn. Uncomment if using 145 | # auto-import. 146 | # .idea/artifacts 147 | # .idea/compiler.xml 148 | # .idea/jarRepositories.xml 149 | # .idea/modules.xml 150 | # .idea/*.iml 151 | # .idea/modules 152 | # *.iml 153 | # *.ipr 154 | 155 | ### macOS template 156 | # General 157 | .DS_Store 158 | .AppleDouble 159 | .LSOverride 160 | 161 | # Icon must end with two \router 162 | Icon 163 | 164 | # Thumbnails 165 | ._* 166 | 167 | # Files that might appear in the root of a volume 168 | .DocumentRevisions-V100 169 | .fseventsd 170 | .Spotlight-V100 171 | .TemporaryItems 172 | .Trashes 173 | .VolumeIcon.icns 174 | .com.apple.timemachine.donotpresent 175 | 176 | # Directories potentially created on remote AFP share 177 | .AppleDB 178 | .AppleDesktop 179 | Network Trash Folder 180 | Temporary Items 181 | .apdisk 182 | 183 | 184 | *.log -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http:www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOHOSTOS:=$(shell go env GOHOSTOS) 2 | GOPATH:=$(shell go env GOPATH) 3 | VERSION=$(shell git describe --tags --always) 4 | PACKAGES=$(shell go list ./... | grep -v /vendor/) 5 | CURRENTTIME=$(shell date +"%Y%m%d%H%M%S") 6 | 7 | REMOTE_DIR=/root/hub-download/dingospeed 8 | 9 | ifeq ($(GOHOSTOS), windows) 10 | #the `find.exe` is different from `find` in bash/shell. 11 | #to see https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/find. 12 | #changed to use git-bash.exe to run find cli or other cli friendly, caused of every developer has a Git. 13 | #Git_Bash= $(subst cmd\,bin\bash.exe,$(dir $(shell where git))) 14 | Git_Bash=$(shell which bash) 15 | PROTO_FILES=$(shell $(Git_Bash) -c "find . -name *.proto") 16 | TEST_DIRS=$(shell $(Git_Bash) -c "find . -name '*_test.go' | awk -F '/[^/]*$$' '{print $$1}' | sort -u") 17 | GO_FILES=$(shell $(Git_Bash) -c "find . -name '*.go' -type f -not -path './vendor/*'") 18 | else 19 | PROTO_FILES=$(shell find . -name *.proto) 20 | TEST_DIRS=$(shell find . -name '*_test.go' | awk -F '/[^/]*$$' '{print $$1}' | sort -u) 21 | GO_FILES=$(shell find . -name '*.go' -type f -not -path './vendor/*') 22 | endif 23 | 24 | .PHONY: init 25 | # init env 26 | init: 27 | go install github.com/google/wire/cmd/wire@latest 28 | 29 | .PHONY: wire 30 | # wire 31 | wire: 32 | cd cmd/ && wire gen ./... 33 | 34 | .PHONY: generate 35 | # generate 36 | generate: 37 | go mod tidy 38 | go get github.com/google/wire/cmd/wire@latest 39 | go generate ./... 40 | 41 | .PHONY: proto 42 | # generate proto 43 | proto: 44 | protoc --proto_path=./pkg/proto \ 45 | --go_out=paths=source_relative:./pkg/proto \ 46 | --go-grpc_out=paths=source_relative:./pkg/proto \ 47 | $(PROTO_FILES) 48 | 49 | .PHONY: test 50 | # test 51 | test: 52 | @go clean -testcache && go test -cover -v ${TEST_DIRS} -gcflags="all=-N -l" 53 | 54 | .PHONY: vet 55 | # vet 56 | vet: 57 | @go vet --unsafeptr=false $(PACKAGES) 58 | 59 | .PHONY: build 60 | # build 61 | build: 62 | mkdir -p bin/ && go build -ldflags "-s -w -X main.Version=$(VERSION)" -o ./bin/dingospeed dingospeed/cmd 63 | 64 | .PHONY: macbuild 65 | macbuild: 66 | mkdir -p bin/ && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.Version=$(VERSION)" -o ./bin/dingospeed dingospeed/cmd 67 | 68 | .PHONY: repairbuild 69 | repairbuild: 70 | mkdir -p bin/ && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.Version=$(VERSION)" -o ./bin/repair dingospeed/repair 71 | 72 | .PHONY: repairScpDev 73 | repairScpDev: 74 | scp bin/repair root@172.30.14.123:/root/hub-download/dingospeed 75 | 76 | 77 | .PHONY: scpDev 78 | scpDev: 79 | scp bin/dingospeed root@172.30.14.123:/root/hub-download/dingospeed 80 | 81 | .PHONY: scpTest 82 | scpTest: 83 | scp bin/dingospeed root@10.220.70.124:/root/hub-download/dingospeed 84 | 85 | # go install github.com/superproj/addlicense@latest 86 | .PHONY: license 87 | license: 88 | addlicense -v -f LICENSE cmd pkg internal 89 | 90 | .PHONY: docker 91 | docker: 92 | make macbuild; 93 | docker build -f docker/Dockerfile-simple -t dingospeed:$(CURRENTTIME) . 94 | 95 | .PHONY: all 96 | # generate all 97 | all: 98 | make init; 99 | make generate; 100 | #make proto; 101 | make vet; 102 | #make test; 103 | make build 104 | 105 | # show help 106 | help: 107 | @echo '' 108 | @echo 'Usage:' 109 | @echo ' make [target]' 110 | @echo '' 111 | @echo 'Targets:' 112 | @awk '/^[a-zA-Z\-\_0-9]+:/ { \ 113 | helpMessage = match(lastLine, /^# (.*)/); \ 114 | if (helpMessage) { \ 115 | helpCommand = substr($$1, 0, index($$1, ":")); \ 116 | helpMessage = substr(lastLine, RSTART + 2, RLENGTH); \ 117 | printf "\033[36m%-22s\033[0m %s\n", helpCommand,helpMessage; \ 118 | } \ 119 | } \ 120 | { lastLine = $$0 }' $(MAKEFILE_LIST) 121 | 122 | .DEFAULT_GOAL := help -------------------------------------------------------------------------------- /README-zh_CN.md: -------------------------------------------------------------------------------- 1 | # DingoSpeed 2 | [English](README.md) | 简体中文 3 | 4 | DingoSpeed 是一个自托管的 Hugging Face 镜像服务，旨在为用户提供便捷、高效的模型资源访问和管理解决方案。通过本地镜像，用户可以减少对远程 Hugging Face 服务器的依赖，提高资源获取速度，同时实现数据的本地化存储和管理。 5 | 6 | # 产品特性 7 | DingoSpeed具备以下主要产品特性： 8 | * 镜像加速：将首次下载的资源做缓存，客户端下次请求时将从缓存读取并返回，极大提升下载速率； 9 | * 便捷访问：无需科学上网及复杂的网络配置，只需部署DingoSpeed服务，并将其作为代理地址，即能方便的完成下载； 10 | * 缩流减载：一次下载多次使用，减少重复下载带来的流量浪费，高效且省流； 11 | * 本地化管理：实现镜像服务本地编译、部署、监控及使用的全流程覆盖，带来灵活可控的卓越体验，避免了对外部网络和公共镜像仓库的依赖，显著提升了系统的响应速度和数据安全性。 12 | 13 | # 功能清单 14 | 1. [x] 实现了 HTTP RESTful API（兼容 HF Hub 规范），支持模型和数据集下载； 15 | 2. [x] 实现了多种缓存清理策略（LRU/FIFO/LARGE_FIRST）、定时任务、阈值触发； 16 | 3. [x] 支持 HTTP Range 请求，实现客户端断点续传，服务端分块下载大文件，降低内存占用； 17 | 4. [x] 支持跨多个镜像节点同步缓存数据，避免多节点重复下载同一文件； 18 | 5. [x] 支持大文件分块存储、多副本存储节点； 19 | 6. [x] 下载耗时较低，并发下载成功率较高； 20 | 7. [x] 内存占用率较低； 21 | 8. [x] 下载速度稳定； 22 | 23 | # 系统架构 24 | ![系统架构图](png/img.png) 25 | 26 | # 安装 27 | 项目会使用wire命令生成所需的依赖代码，安装wire命令如下： 28 | ```bash 29 | # 导入到项目中 30 | go get -u github.com/google/wire 31 | 32 | # 安装命令 33 | go install github.com/google/wire/cmd/wire 34 | ``` 35 | 36 | Wire 是一个灵活的依赖注入工具，通过自动生成代码的方式在编译期完成依赖注入。 在各个组件之间的依赖关系中，通常显式初始化，而不是全局变量传递。 所以通过 Wire 进行初始化代码，可以很好地解决组件之间的耦合，以及提高代码维护性。 37 | 38 | > 本项目使用go mod管理依赖，需要go1.23以上版本。使用makefile管理项目，需要make命令 39 | 40 | ```bash 41 | # 1. 安装依赖 42 | make init 43 | 44 | # 2. 代码生成 45 | make wire 46 | 47 | # 3. 编译可执行文件，当前系统版本 48 | make build 49 | 50 | # 4. mac上编译linux可执行文件 51 | make macbuild 52 | 53 | # 5. 为每个文件添加licence 54 | make license 55 | 56 | ``` 57 | # 快速开始 58 | 将编译生成的二进制部署文件，执行./dingospeed启动。然后将环境变量`HF_ENDPOINT`设置为镜像站点(这里是http://localhost:8090/)。 59 | 60 | Linux: 61 | ```shell 62 | export HF_ENDPOINT=http://localhost:8090 63 | ``` 64 | Windows Powershell: 65 | ```shell 66 | $env:HF_ENDPOINT = "http://localhost:8090" 67 | ``` 68 | 从现在开始，HuggingFace库中的所有下载操作都将通过此镜像站点代理进行。可以安装python库试用： 69 | ```shell 70 | pip install -U huggingface_hub 71 | ``` 72 | ```shell 73 | from huggingface_hub import snapshot_download 74 | 75 | snapshot_download(repo_id='Qwen/Qwen-7B', repo_type='model', 76 | local_dir='./model_dir', resume_download=True, 77 | max_workers=8) 78 | 79 | ``` 80 | 或者你也可以使用huggingface cli直接下载模型和数据集. 81 | 下载GPT2: 82 | ```shell 83 | huggingface-cli download --resume-download openai-community/gpt2 --local-dir gpt2 84 | ``` 85 | 下载单个文件： 86 | ```shell 87 | huggingface-cli download --resume-download --force-download HuggingFaceTB/SmolVLM-256M-Instruct config.json 88 | ``` 89 | 下载WikiText: 90 | ```shell 91 | huggingface-cli download --repo-type dataset --resume-download Salesforce/wikitext --local-dir wikitext 92 | ``` 93 | 您可以查看路径./repos，其中存储了所有数据集和模型的缓存。 94 | 95 | # 下载模型 96 | 97 | 通过将文件按一定的大小切分成数量不等的文件段，由调度工具将任务提交到协程池执行下载任务，每个协程任务将所分配的长度提交到远端请求，按照一个chunk大小来循环读取响应 98 | 结果，并将结果缓存的协程独有的工作队列，由推送协程将其推送的客户端。同时每次检查当前chunk是否满足一个block的大小，若满足则将该block写入文件。 99 | 100 | ![下载模型](png/img_download.png) 101 | 102 | # 存储模型 103 | 104 | 仓库缓存数据文件由HEADER和数据块量两部分构成，其中HEADER作用： 105 | 1.提高缓存文件的可读性，当配置文件被修改或程序升级，都不会影响已缓存文件的读取； 106 | 2.能高效的检查块是否存在，无需读取真实的数据库，提高操作效率。 107 | 108 | ![存储模型](png/img_store.png) 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DingoSpeed 2 | English | [简体中文](README-zh_CN.md) 3 | 4 | DingoSpeed is a self-hosted Hugging Face mirror service designed to provide users with a convenient and efficient solution for accessing and managing model resources. Through local mirroring, users can reduce their reliance on remote Hugging Face servers, improve resource acquisition speed, and achieve local storage and management of data. 5 | 6 | # Product Features 7 | DingoSpeed has the following main product features: 8 | * Mirror Acceleration: Cache the resources downloaded for the first time. When the client makes a subsequent request, the data will be read from the cache and returned, greatly improving the download rate. 9 | * Convenient Access: There is no need for scientific internet access or complex network configuration. Simply deploy the DingoSpeed service and use it as the proxy address to easily complete the download. 10 | * Traffic Reduction and Load Alleviation: Download once and use multiple times, reducing the traffic waste caused by repeated downloads, which is efficient and saves traffic. 11 | * Localized Management: Cover the entire process of local compilation, deployment, monitoring, and usage of the mirror service, bringing an excellent and flexible experience. It avoids reliance on external networks and public mirror repositories, significantly improving the system's response speed and data security. 12 | 13 | # Function List 14 | 1. [X] Implemented an HTTP RESTful API (compatible with the HF Hub specification) to support model and dataset downloads. 15 | 2. [X] Implemented multiple cache cleaning strategies (LRU/FIFO/LARGE_FIRST), scheduled tasks, and threshold triggers. 16 | 3. [X] Supports HTTP Range requests, enabling clients to resume interrupted downloads and allowing the server to download large files in chunks, reducing memory usage. 17 | 4. [X] Supports synchronizing cache data across multiple mirror nodes to avoid repeated downloads of the same file on multiple nodes. 18 | 5. [X] Supports storing large files in chunks and using multiple replica storage nodes. 19 | 6. [X] Low download time and high concurrent download success rate. 20 | 7. [X] Low memory usage. 21 | 8. [X] Stable download speed. 22 | 23 | 24 | # System Architecture 25 | ![System Architecture](png/architecture_en.png) 26 | 27 | # Installation 28 | The project uses the wire command to generate the required dependency code. Install the wire command as follows： 29 | ```bash 30 | # Import into the project 31 | go get -u github.com/google/wire 32 | 33 | # Install the command 34 | go install github.com/google/wire/cmd/wire 35 | ``` 36 | 37 | Wire is a flexible dependency injection tool that completes dependency injection at compile time by automatically generating code. In the dependency relationships between various components, explicit initialization is usually used instead of passing global variables. Therefore, using Wire to initialize the code can effectively solve the coupling between components and improve code maintainability. 38 | > This project uses go mod to manage dependencies and requires Go version 1.23 or higher. It uses makefile to manage the project and requires the make command. 39 | 40 | ```bash 41 | # 1. Install dependencies 42 | make init 43 | 44 | # 2. Generate code 45 | make wire 46 | 47 | # 3. Compile the executable file for the current system version 48 | make build 49 | 50 | # 4. Compile the Linux executable file on macOS 51 | make macbuild 52 | 53 | # 5. Add a license to each file 54 | make license 55 | 56 | ``` 57 | # Quick Start 58 | Deploy the compiled binary file and execute ./dingospeed to start the service. Then set the environment variable HF_ENDPOINT to the mirror site (here it is http://localhost:8090/). 59 | 60 | Linux: 61 | ```shell 62 | export HF_ENDPOINT=http://localhost:8090 63 | ``` 64 | Windows Powershell: 65 | ```shell 66 | $env:HF_ENDPOINT = "http://localhost:8090" 67 | ``` 68 | From now on, all download operations in the Hugging Face library will be proxied through this mirror site. You can install the Python library to try it out: 69 | 70 | ```shell 71 | pip install -U huggingface_hub 72 | ``` 73 | ```shell 74 | from huggingface_hub import snapshot_download 75 | 76 | snapshot_download(repo_id='Qwen/Qwen-7B', repo_type='model', 77 | local_dir='./model_dir', resume_download=True, 78 | max_workers=8) 79 | 80 | ``` 81 | Alternatively, you can use the Hugging Face CLI to directly download models and datasets. 82 | Download GPT2: 83 | ```shell 84 | huggingface-cli download --resume-download openai-community/gpt2 --local-dir gpt2 85 | ``` 86 | Download a single file: 87 | 88 | ```shell 89 | huggingface-cli download --resume-download --force-download HuggingFaceTB/SmolVLM-256M-Instruct config.json 90 | ``` 91 | Download WikiText: 92 | ```shell 93 | huggingface-cli download --repo-type dataset --resume-download Salesforce/wikitext --local-dir wikitext 94 | ``` 95 | You can view the path ./repos, where the caches of all datasets and models are stored. 96 | 97 | # Downloading Models 98 | The file is divided into different segments of a certain size. The scheduling tool submits the tasks to the coroutine pool for execution. Each coroutine task submits the assigned length to the remote server for a request, reads the response results in chunks, and caches the results in the coroutine's exclusive work queue. The push coroutine then pushes the data to the client. At the same time, it checks whether the current chunk meets the size of a block. If it does, the block is written to the file. 99 | 100 | ![Downloading Models](png/downloading_models_en.png) 101 | 102 | # Storing Models 103 | 104 | The repository cache data file consists of a HEADER and data blocks. The functions of the HEADER are as follows: 105 | 1. Improve the readability of the cache file. Even if the configuration file is modified or the program is upgraded, it will not affect the reading of the cached file. 106 | 2. Efficiently check the existence of blocks without reading the actual database, improving operation efficiency. 107 | 108 | ![Storing Models](png/storing_models_en.png) 109 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | "net/http" 21 | _ "net/http/pprof" 22 | "os" 23 | "runtime" 24 | 25 | "dingospeed/internal/server" 26 | "dingospeed/pkg/app" 27 | "dingospeed/pkg/config" 28 | log "dingospeed/pkg/logger" 29 | ) 30 | 31 | var ( 32 | configPath string 33 | id, _ = os.Hostname() //nolint:errcheck 34 | Name = "dingospeed" 35 | Version string 36 | ) 37 | 38 | func init() { 39 | flag.StringVar(&configPath, "config", "./config/config.yaml", "配置文件路径") 40 | flag.Parse() 41 | } 42 | 43 | func newApp(s *server.HTTPServer) *app.App { 44 | app := app.New(app.ID(id), app.Name(Name), app.Version(Version), 45 | app.Server(s)) 46 | return app 47 | } 48 | 49 | func main() { 50 | conf, err := config.Scan(configPath) 51 | if err != nil { 52 | panic(err) 53 | } 54 | 55 | log.InitLogger() 56 | myapp, f, err := wireApp(conf) 57 | if err != nil { 58 | panic(err) 59 | } 60 | 61 | if config.SysConfig.Server.PProf { 62 | runtime.SetBlockProfileRate(1) 63 | runtime.SetMutexProfileFraction(1) 64 | 65 | go func() { 66 | panic(http.ListenAndServe(fmt.Sprintf(":%d", config.SysConfig.Server.PProfPort), nil)) 67 | }() 68 | } 69 | 70 | defer f() 71 | 72 | err = myapp.Run() 73 | if err != nil { 74 | panic(err) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cmd/wire.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build wireinject 16 | // +build wireinject 17 | 18 | // The build tag makes sure the stub is not built in the final build. 19 | 20 | package main 21 | 22 | import ( 23 | "dingospeed/internal/dao" 24 | "dingospeed/internal/handler" 25 | "dingospeed/internal/router" 26 | "dingospeed/internal/server" 27 | "dingospeed/internal/service" 28 | "dingospeed/pkg/app" 29 | "dingospeed/pkg/config" 30 | 31 | "github.com/google/wire" 32 | ) 33 | 34 | func wireApp(*config.Config) (*app.App, func(), error) { 35 | panic(wire.Build(server.ServerProvider, router.RouterProvider, handler.HandlerProvider, service.ServiceProvider, dao.DaoProvider, newApp)) 36 | } 37 | -------------------------------------------------------------------------------- /cmd/wire_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by Wire. DO NOT EDIT. 2 | 3 | //go:generate go run -mod=mod github.com/google/wire/cmd/wire 4 | //go:build !wireinject 5 | // +build !wireinject 6 | 7 | package main 8 | 9 | import ( 10 | "dingospeed/internal/dao" 11 | "dingospeed/internal/handler" 12 | "dingospeed/internal/router" 13 | "dingospeed/internal/server" 14 | "dingospeed/internal/service" 15 | "dingospeed/pkg/app" 16 | "dingospeed/pkg/config" 17 | ) 18 | 19 | import ( 20 | _ "net/http/pprof" 21 | ) 22 | 23 | // Injectors from wire.go: 24 | 25 | func wireApp(configConfig *config.Config) (*app.App, func(), error) { 26 | echo := server.NewEngine() 27 | fileDao := dao.NewFileDao() 28 | fileService := service.NewFileService(fileDao) 29 | sysService := service.NewSysService() 30 | fileHandler := handler.NewFileHandler(fileService, sysService) 31 | metaDao := dao.NewMetaDao(fileDao) 32 | metaService := service.NewMetaService(fileDao, metaDao) 33 | metaHandler := handler.NewMetaHandler(metaService) 34 | sysHandler := handler.NewSysHandler(sysService) 35 | httpRouter := router.NewHttpRouter(echo, fileHandler, metaHandler, sysHandler) 36 | httpServer := server.NewServer(configConfig, echo, httpRouter) 37 | appApp := newApp(httpServer) 38 | return appApp, func() { 39 | }, nil 40 | } 41 | -------------------------------------------------------------------------------- /config/config-offline.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | mode: debug 3 | host: localhost 4 | port: 8091 5 | pprof: true 6 | online: false #true表示hf-mirror找不到，去hfNetLoc地址查找并下载模型数据，false表示本地如果没有，直接返回没有 7 | repos: ./repos 8 | hfNetLoc: hf-mirror.com # huggingface.co hf-mirror.com 9 | hfScheme: https 10 | hfLfsNetLoc : cdn-lfs.huggingface.co 11 | 12 | download: 13 | blockSize: 8388608 #默认文件块大小为8MB（8388608），单位字节，1048576（1MB） 14 | reqTimeout: 0 #远端请求超时时间，单位秒，默认为0，不超时。 15 | respChunkSize: 8192 #默认对响应结果的读取大小8192，单位字节。 16 | respChanSize: 30 #响应队列大小 17 | remoteFileBufferSize: 8388608 #每个分区文件的结果Queue的缓存大小，即当前文件下载时，缓存64MB的数据 18 | remoteFileRangeSize: 0 #按照这个长度分块下载，0为不切分,测试选项：8388608（8M），67108864（64M），134217728（128M）,536870912(512M),1GB（1073741824） 19 | remoteFileRangeWaitTime: 1 #每个分区文件下载任务提交时间间隔，默认1s，单位（s）。 20 | goroutineMaxNumPerFile: 8 #远程下载任务启动的最大协程数量 21 | 22 | 23 | cache: 24 | enabled: true 25 | collectTimePeriod: 5 #定期检测内存使用量时间周期，单位秒（S） 26 | prefetchMemoryUsedThreshold: 90 #当内存使用量达到该值，将不会预读取，不缓存数据块 27 | prefetchBlocks: 16 #离线下载时，预先读取块的数量 28 | prefetchBlockTTL: 30 #离线下载时，预先读取块的存活时间，单位秒（S） 29 | 30 | retry: 31 | delay: 1 #重试间隔时间，单位秒，默认为1 32 | attempts: 3 #重试次数，默认为3 33 | 34 | log: 35 | maxSize: 1 # 日志文件最大的尺寸（MB） 36 | maxBackups: 10 #保留旧文件的最大个数 37 | maxAge: 90 #保留旧文件的最大天数 38 | 39 | tokenBucketLimit: 40 | handlerCapacity: 50 #提交处理任务的超时时间 41 | 42 | diskClean: 43 | enabled: true 44 | cacheSizeLimit: 41781441855488 #38T 45 | cacheCleanStrategy: "LRU" #LRU,FIFO,LARGE_FIRST 46 | collectTimePeriod: 1 #定期检测磁盘使用量时间周期，单位小时（H） 47 | -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | mode: debug 3 | host: 0.0.0.0 4 | port: 8090 5 | pprof: true 6 | pprofPort: 6060 7 | metrics: true 8 | online: true #true表示本地找不到，去hfNetLoc地址查找并下载模型数据，false表示本地如果没有，直接返回没有 9 | repos: ./repos 10 | hfNetLoc: hf-mirror.com # huggingface.co hf-mirror.com 11 | hfScheme: https 12 | 13 | download: 14 | blockSize: 8388608 #默认文件块大小为8MB（8388608），单位字节，1048576（1MB） 15 | reqTimeout: 0 #远端请求超时时间，单位秒，默认为0，不超时。 16 | respChunkSize: 8192 #默认对响应结果的读取大小8192，单位字节。 17 | respChanSize: 30 #响应队列大小 18 | remoteFileBufferSize: 8388608 #每个分区文件的结果Queue的缓存大小，即当前文件下载时，缓存8MB的数据 19 | remoteFileRangeSize: 0 #按照这个长度分块下载，0为不切分,测试选项：8388608（8M），67108864（64M），134217728（128M）,536870912(512M),1GB（1073741824） 20 | remoteFileRangeWaitTime: 0 #每个分区文件下载任务提交时间间隔，单位（ms）。 21 | goroutineMaxNumPerFile: 8 #远程下载任务启动的最大协程数量 22 | 23 | 24 | cache: 25 | enabled: false #是否启用缓存 26 | type: 1 # 采用哪种组件缓存，可选项0(map), 1(Ristretto) 27 | collectTimePeriod: 5 #定期检测内存使用量时间周期，单位秒（S） 28 | prefetchMemoryUsedThreshold: 90 #当内存使用量达到该值，将不会预读取，不缓存数据块 29 | prefetchBlocks: 16 #离线下载时，预先读取块的数量 30 | prefetchBlockTTL: 30 #离线下载时，预先读取块的存活时间，单位秒（S） 31 | 32 | retry: 33 | delay: 1 #重试间隔时间，单位秒，默认为1 34 | attempts: 3 #重试次数，默认为3 35 | 36 | log: 37 | maxSize: 20 # 日志文件最大的尺寸（MB） 38 | maxBackups: 10 #保留旧文件的最大个数 39 | maxAge: 90 #保留旧文件的最大天数 40 | 41 | tokenBucketLimit: 42 | handlerCapacity: 50 #提交处理任务的超时时间 43 | 44 | diskClean: 45 | enabled: true 46 | cacheSizeLimit: 41781441855488 #38T 47 | cacheCleanStrategy: "LRU" #LRU,FIFO,LARGE_FIRST 48 | collectTimePeriod: 1 #定期检测磁盘使用量时间周期，单位小时（H） 49 | 50 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.0 AS builder 2 | 3 | LABEL stage=gobuilder 4 | 5 | ENV CGO_ENABLED=0 6 | ENV GOPROXY=https://goproxy.cn,direct 7 | 8 | WORKDIR /app 9 | 10 | COPY .. . 11 | 12 | # Installation of dependency packages and environment preparation 13 | RUN go install github.com/google/wire/cmd/wire@latest 14 | RUN cd cmd/ && wire gen ./... 15 | RUN go mod tidy 16 | RUN go get github.com/google/wire/cmd/wire@latest 17 | RUN go generate ./... 18 | RUN mkdir "repos" 19 | 20 | # Compile Go project and generate executable file 21 | RUN mkdir -p bin/ && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.Version=$(VERSION)" -o ./bin/dingospeed dingospeed/cmd 22 | 23 | # Expose the 8090 port of the container for external acces 24 | EXPOSE 8090 25 | 26 | VOLUME /app/repos 27 | VOLUME /app/log 28 | 29 | CMD ["./bin/dingospeed"] 30 | -------------------------------------------------------------------------------- /docker/Dockerfile-simple: -------------------------------------------------------------------------------- 1 | FROM golang:1.23.0 AS builder 2 | 3 | LABEL stage=gobuilder 4 | 5 | WORKDIR /app 6 | 7 | COPY ./bin/dingospeed . 8 | 9 | # Expose the 8090 port of the container for external acces 10 | EXPOSE 8090 11 | 12 | VOLUME /app/repos 13 | VOLUME /app/log 14 | 15 | CMD ["./dingospeed"] 16 | -------------------------------------------------------------------------------- /docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | dingospeed: 5 | image: harbor.dev01.zetyun.cn/dingofs/dingospeed:v4 6 | container_name: dingospeed 7 | #environment: 8 | #- http_proxy=http://100.64.1.68:1080 9 | #- https_proxy=http://100.64.1.68:1080 10 | #- http_proxy=http://100.64.1.72:3128 11 | #- https_proxy=http://100.64.1.72:3128 12 | ports: 13 | - "8090:8090" 14 | volumes: 15 | - ./repos:/app/repos 16 | - ./log:/app/log 17 | restart: always 18 | environment: 19 | - TZ=Asia/Shanghai 20 | cpus: '25' # 限制 CPU 为 20 核 21 | mem_limit: '50g' # 限制内存为 50G 22 | dingospeed-online: 23 | image: harbor.dev01.zetyun.cn/dingofs/dingospeed-online:v4 24 | container_name: dingospeed-online 25 | environment: 26 | - http_proxy=http://10.201.44.68:1080 27 | - https_proxy=http://10.201.44.68:1080 28 | - TZ=Asia/Shanghai 29 | ports: 30 | - "8091:8091" 31 | volumes: 32 | - ./repos:/app/repos 33 | - ./log:/app/log 34 | restart: always 35 | cpus: '25' # 限制 CPU 为 20 核 36 | mem_limit: '50g' # 限制内存为 50G 37 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module dingospeed 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.1 6 | 7 | require ( 8 | github.com/andybalholm/brotli v1.1.1 9 | github.com/avast/retry-go v3.0.0+incompatible 10 | github.com/bytedance/sonic v1.13.2 11 | github.com/go-playground/validator/v10 v10.26.0 12 | github.com/google/uuid v1.3.0 13 | github.com/google/wire v0.6.0 14 | github.com/klauspost/compress v1.18.0 15 | github.com/labstack/echo/v4 v4.13.3 16 | github.com/shirou/gopsutil v3.21.11+incompatible 17 | go.uber.org/zap v1.24.0 18 | golang.org/x/sync v0.13.0 19 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 20 | gopkg.in/yaml.v3 v3.0.1 21 | 22 | ) 23 | 24 | require ( 25 | github.com/beorn7/perks v1.0.1 // indirect 26 | github.com/bytedance/sonic/loader v0.2.4 // indirect 27 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 28 | github.com/cloudwego/base64x v0.1.5 // indirect 29 | github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect 30 | github.com/dustin/go-humanize v1.0.1 // indirect 31 | github.com/gabriel-vasile/mimetype v1.4.8 // indirect 32 | github.com/go-ole/go-ole v1.2.6 // indirect 33 | github.com/go-playground/locales v0.14.1 // indirect 34 | github.com/go-playground/universal-translator v0.18.1 // indirect 35 | github.com/klauspost/cpuid/v2 v2.2.10 // indirect 36 | github.com/labstack/gommon v0.4.2 // indirect 37 | github.com/leodido/go-urn v1.4.0 // indirect 38 | github.com/mattn/go-colorable v0.1.14 // indirect 39 | github.com/mattn/go-isatty v0.0.20 // indirect 40 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 41 | github.com/pkg/errors v0.9.1 // indirect 42 | github.com/prometheus/client_golang v1.22.0 // indirect 43 | github.com/prometheus/client_model v0.6.1 // indirect 44 | github.com/prometheus/common v0.62.0 // indirect 45 | github.com/prometheus/procfs v0.15.1 // indirect 46 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 47 | github.com/valyala/bytebufferpool v1.0.0 // indirect 48 | github.com/valyala/fasttemplate v1.2.2 // indirect 49 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 50 | go.uber.org/atomic v1.10.0 // indirect 51 | go.uber.org/multierr v1.9.0 // indirect 52 | golang.org/x/arch v0.15.0 // indirect 53 | golang.org/x/crypto v0.37.0 // indirect 54 | golang.org/x/net v0.39.0 // indirect 55 | golang.org/x/sys v0.32.0 // indirect 56 | golang.org/x/text v0.24.0 // indirect 57 | google.golang.org/protobuf v1.36.5 // indirect 58 | ) 59 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 2 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 3 | github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= 4 | github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= 5 | github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= 6 | github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= 10 | github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= 11 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 12 | github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= 13 | github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= 14 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 15 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 16 | github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= 17 | github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 18 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= 23 | github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= 24 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 25 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 26 | github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= 27 | github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= 28 | github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= 29 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 30 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= 31 | github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 32 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 33 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 34 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 35 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 36 | github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= 37 | github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= 38 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 39 | github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= 40 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 41 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 42 | github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= 43 | github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= 44 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 45 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 46 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 47 | github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= 48 | github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 49 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 50 | github.com/labstack/echo/v4 v4.13.3 h1:pwhpCPrTl5qry5HRdM5FwdXnhXSLSY+WE+YQSeCaafY= 51 | github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAfg+Xox9q5g= 52 | github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= 53 | github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= 54 | github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= 55 | github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= 56 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 57 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 58 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 59 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 60 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 61 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 62 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 63 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 64 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 65 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 66 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 67 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 68 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 69 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 70 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 71 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 72 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 73 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 74 | github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= 75 | github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 76 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 77 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 78 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 79 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 80 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 81 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 82 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 83 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 84 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 85 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 86 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 87 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 88 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 89 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 90 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 91 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 92 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 93 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 94 | github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 95 | github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 96 | go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= 97 | go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 98 | go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= 99 | go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= 100 | go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= 101 | go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= 102 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 103 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 104 | golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= 105 | golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= 106 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 107 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 108 | golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= 109 | golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= 110 | golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= 111 | golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= 112 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 113 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 114 | golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 115 | golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 116 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 117 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 118 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 119 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 120 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 121 | golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= 122 | golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= 123 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= 124 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= 125 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 126 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 127 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 128 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 129 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 130 | golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= 131 | golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 132 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 133 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 134 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 135 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 136 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 137 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 138 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 139 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 140 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 141 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 142 | golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 143 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 144 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 145 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 146 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 147 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 148 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 149 | golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= 150 | golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= 151 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 152 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 153 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 154 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 155 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 156 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 157 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 158 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 159 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 160 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 161 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 162 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 163 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 164 | golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= 165 | golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= 166 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 167 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 168 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 169 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 170 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 171 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= 172 | gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= 173 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 174 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 175 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 176 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 177 | -------------------------------------------------------------------------------- /internal/dao/dao.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dao 16 | 17 | import "github.com/google/wire" 18 | 19 | var DaoProvider = wire.NewSet(NewFileDao, NewMetaDao) 20 | -------------------------------------------------------------------------------- /internal/dao/file_dao.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dao 16 | 17 | import ( 18 | "context" 19 | "encoding/hex" 20 | "fmt" 21 | "io" 22 | "net/http" 23 | "net/url" 24 | "path" 25 | "path/filepath" 26 | "strings" 27 | "time" 28 | 29 | cache "dingospeed/internal/data" 30 | "dingospeed/internal/downloader" 31 | "dingospeed/pkg/common" 32 | "dingospeed/pkg/config" 33 | "dingospeed/pkg/consts" 34 | myerr "dingospeed/pkg/error" 35 | "dingospeed/pkg/util" 36 | 37 | "github.com/bytedance/sonic" 38 | "github.com/labstack/echo/v4" 39 | "go.uber.org/zap" 40 | ) 41 | 42 | type CommitHfSha struct { 43 | Sha string `json:"sha"` 44 | } 45 | 46 | type FileDao struct { 47 | } 48 | 49 | func NewFileDao() *FileDao { 50 | if config.SysConfig.Cache.Enabled { 51 | cache.InitCache() // 初始化缓存 52 | } 53 | return &FileDao{} 54 | } 55 | 56 | func (f *FileDao) CheckCommitHf(repoType, org, repo, commit, authorization string) (int, error) { 57 | orgRepo := util.GetOrgRepo(org, repo) 58 | var reqUrl string 59 | if commit == "" { 60 | reqUrl = fmt.Sprintf("%s/api/%s/%s", config.SysConfig.GetHFURLBase(), repoType, orgRepo) 61 | } else { 62 | reqUrl = fmt.Sprintf("%s/api/%s/%s/revision/%s", config.SysConfig.GetHFURLBase(), repoType, orgRepo, commit) 63 | } 64 | headers := map[string]string{} 65 | if authorization != "" { 66 | headers["authorization"] = authorization 67 | } 68 | resp, err := util.RetryRequest(func() (*common.Response, error) { 69 | return util.Head(reqUrl, headers, config.SysConfig.GetReqTimeOut()) 70 | }) 71 | if err != nil { 72 | zap.S().Errorf("call %s error.%v", reqUrl, err) 73 | return http.StatusInternalServerError, err 74 | } 75 | if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusTemporaryRedirect { 76 | return resp.StatusCode, nil 77 | } 78 | zap.S().Errorf("CheckCommitHf statusCode:%d", resp.StatusCode) 79 | return resp.StatusCode, myerr.New("request commit err") 80 | } 81 | 82 | // 若为离线或在线请求失败，将进行本地仓库查找。 83 | 84 | func (f *FileDao) GetCommitHf(repoType, org, repo, commit, authorization string) (string, error) { 85 | if !config.SysConfig.Online() { 86 | return f.getCommitHfOffline(repoType, org, repo, commit) 87 | } 88 | orgRepo := util.GetOrgRepo(org, repo) 89 | var reqUrl string 90 | reqUrl = fmt.Sprintf("%s/api/%s/%s/revision/%s", config.SysConfig.GetHFURLBase(), repoType, orgRepo, commit) 91 | headers := map[string]string{} 92 | if authorization != "" { 93 | headers["authorization"] = authorization 94 | } 95 | resp, err := util.RetryRequest(func() (*common.Response, error) { 96 | return util.Get(reqUrl, headers, config.SysConfig.GetReqTimeOut()) 97 | }) 98 | if err != nil { 99 | zap.S().Errorf("call %s error.%v", reqUrl, err) 100 | return f.getCommitHfOffline(repoType, org, repo, commit) 101 | } 102 | var sha CommitHfSha 103 | if err = sonic.Unmarshal(resp.Body, &sha); err != nil { 104 | zap.S().Errorf("unmarshal content:%s, error:%v", string(resp.Body), err) 105 | return f.getCommitHfOffline(repoType, org, repo, commit) 106 | } 107 | return sha.Sha, nil 108 | } 109 | 110 | func (f *FileDao) getCommitHfOffline(repoType, org, repo, commit string) (string, error) { 111 | orgRepo := util.GetOrgRepo(org, repo) 112 | apiPath := fmt.Sprintf("%s/api/%s/%s/revision/%s/meta_get.json", config.SysConfig.Repos(), repoType, orgRepo, commit) 113 | if util.FileExists(apiPath) { 114 | cacheContent, err := f.ReadCacheRequest(apiPath) 115 | if err != nil { 116 | return "", err 117 | } 118 | var sha CommitHfSha 119 | if err = sonic.Unmarshal(cacheContent.OriginContent, &sha); err != nil { 120 | zap.S().Errorf("unmarshal error.%v", err) 121 | return "", myerr.Wrap("Unmarshal err", err) 122 | } 123 | return sha.Sha, nil 124 | } 125 | return "", myerr.New(fmt.Sprintf("apiPath file not exist, %s", apiPath)) 126 | } 127 | 128 | func (f *FileDao) FileGetGenerator(c echo.Context, repoType, org, repo, commit, fileName, method string) error { 129 | orgRepo := util.GetOrgRepo(org, repo) 130 | filesDir := fmt.Sprintf("%s/files/%s/%s/resolve/%s", config.SysConfig.Repos(), repoType, orgRepo, commit) 131 | filesPath := fmt.Sprintf("%s/%s", filesDir, fileName) 132 | err := util.MakeDirs(filesPath) 133 | if err != nil { 134 | zap.S().Errorf("create %s dir err.%v", filesPath, err) 135 | return util.ErrorProxyError(c) 136 | } 137 | var hfUrl string 138 | if repoType == "models" { 139 | hfUrl = fmt.Sprintf("%s/%s/resolve/%s/%s", config.SysConfig.GetHFURLBase(), orgRepo, commit, fileName) 140 | } else { 141 | hfUrl = fmt.Sprintf("%s/%s/%s/resolve/%s/%s", config.SysConfig.GetHFURLBase(), repoType, orgRepo, commit, fileName) 142 | } 143 | reqHeaders := map[string]string{} 144 | for k, _ := range c.Request().Header { 145 | if k == "host" { 146 | domain, _ := util.GetDomain(hfUrl) 147 | reqHeaders[strings.ToLower(k)] = domain 148 | } else { 149 | reqHeaders[strings.ToLower(k)] = c.Request().Header.Get(k) 150 | } 151 | } 152 | authorization := reqHeaders["authorization"] 153 | // _file_realtime_stream 154 | pathsInfos, err := f.pathsInfoGenerator(repoType, org, repo, commit, authorization, []string{fileName}, "post") 155 | if err != nil { 156 | if e, ok := err.(myerr.Error); ok { 157 | zap.S().Errorf("pathsInfoGenerator code:%d, err:%v", e.StatusCode(), err) 158 | return util.ErrorEntryUnknown(c, e.StatusCode(), e.Error()) 159 | } 160 | zap.S().Errorf("pathsInfoGenerator err:%v", err) 161 | return util.ErrorProxyError(c) 162 | } 163 | if len(pathsInfos) == 0 { 164 | zap.S().Errorf("pathsInfos is null. org:%s, repo:%s, commit:%s, fileName:%s", org, repo, commit, fileName) 165 | return util.ErrorEntryNotFound(c) 166 | } 167 | if len(pathsInfos) != 1 { 168 | zap.S().Errorf("pathsInfos not equal to 1. org:%s, repo:%s, commit:%s, fileName:%s", org, repo, commit, fileName) 169 | return util.ErrorProxyTimeout(c) 170 | } 171 | respHeaders := map[string]string{} 172 | pathInfo := pathsInfos[0] 173 | var startPos, endPos int64 174 | if pathInfo.Size > 0 { // There exists a file of size 0 175 | var headRange = reqHeaders["range"] 176 | if headRange == "" { 177 | headRange = fmt.Sprintf("bytes=%d-%d", 0, pathInfo.Size-1) 178 | } 179 | startPos, endPos = parseRangeParams(headRange, pathInfo.Size) 180 | endPos = endPos + 1 181 | } else if pathInfo.Size == 0 { 182 | zap.S().Warnf("file %s size: %d", fileName, pathInfo.Size) 183 | } 184 | respHeaders["content-length"] = util.Itoa(endPos - startPos) 185 | if commit != "" { 186 | respHeaders[strings.ToLower(consts.HUGGINGFACE_HEADER_X_REPO_COMMIT)] = commit 187 | } 188 | var etag string 189 | if pathInfo.Lfs.Oid != "" { 190 | etag = pathInfo.Lfs.Oid 191 | } else { 192 | etag = pathInfo.Oid 193 | } 194 | respHeaders["etag"] = etag 195 | blobsDir := fmt.Sprintf("%s/files/%s/%s/blobs", config.SysConfig.Repos(), repoType, orgRepo) 196 | blobsFile := fmt.Sprintf("%s/%s", blobsDir, etag) 197 | err = util.MakeDirs(blobsFile) 198 | if err != nil { 199 | zap.S().Errorf("create %s dir err.%v", blobsDir, err) 200 | return util.ErrorProxyError(c) 201 | } 202 | if method == consts.RequestTypeHead { 203 | return util.ResponseHeaders(c, respHeaders) 204 | } else if method == consts.RequestTypeGet { 205 | return f.FileChunkGet(c, hfUrl, blobsFile, filesPath, orgRepo, fileName, authorization, pathInfo.Size, startPos, endPos, respHeaders) 206 | } else { 207 | return util.ErrorMethodError(c) 208 | } 209 | } 210 | 211 | func (f *FileDao) pathsInfoGenerator(repoType, org, repo, commit, authorization string, paths []string, method string) ([]common.PathsInfo, error) { 212 | orgRepo := util.GetOrgRepo(org, repo) 213 | remoteReqFilePathMap := make(map[string]string, 0) 214 | ret := make([]common.PathsInfo, 0) 215 | for _, pathFileName := range paths { 216 | apiDir := fmt.Sprintf("%s/api/%s/%s/paths-info/%s/%s", config.SysConfig.Repos(), repoType, orgRepo, commit, pathFileName) 217 | apiPathInfoPath := fmt.Sprintf("%s/%s", apiDir, fmt.Sprintf("paths-info_%s.json", method)) 218 | hitCache := util.FileExists(apiPathInfoPath) 219 | if hitCache { 220 | cacheContent, err := f.ReadCacheRequest(apiPathInfoPath) 221 | if err != nil { 222 | zap.S().Errorf("ReadCacheRequest err.%v", err) 223 | continue 224 | } 225 | pathsInfos := make([]common.PathsInfo, 0) 226 | err = sonic.Unmarshal(cacheContent.OriginContent, &pathsInfos) 227 | if err != nil { 228 | zap.S().Errorf("pathsInfo Unmarshal err.%v", err) 229 | continue 230 | } 231 | if cacheContent.StatusCode == http.StatusOK { 232 | ret = append(ret, pathsInfos...) 233 | } 234 | } else { 235 | remoteReqFilePathMap[pathFileName] = apiPathInfoPath 236 | } 237 | } 238 | if len(remoteReqFilePathMap) > 0 { 239 | filePaths := make([]string, 0) 240 | for k := range remoteReqFilePathMap { 241 | filePaths = append(filePaths, k) 242 | } 243 | pathsInfoUrl := fmt.Sprintf("%s/api/%s/%s/paths-info/%s", config.SysConfig.GetHFURLBase(), repoType, orgRepo, commit) 244 | response, err := f.pathsInfoProxy(pathsInfoUrl, authorization, filePaths) 245 | if err != nil { 246 | zap.S().Errorf("req %s err.%v", pathsInfoUrl, err) 247 | return nil, myerr.NewAppendCode(http.StatusInternalServerError, fmt.Sprintf("%v", err)) 248 | } 249 | if response.StatusCode != http.StatusOK { 250 | var errorResp common.ErrorResp 251 | if len(response.Body) > 0 { 252 | err = sonic.Unmarshal(response.Body, &errorResp) 253 | if err != nil { 254 | return nil, myerr.NewAppendCode(response.StatusCode, fmt.Sprintf("response code %d, %v", response.StatusCode, err)) 255 | } 256 | } 257 | return nil, myerr.NewAppendCode(response.StatusCode, errorResp.Error) 258 | } 259 | remoteRespPathsInfos := make([]common.PathsInfo, 0) 260 | err = sonic.Unmarshal(response.Body, &remoteRespPathsInfos) 261 | if err != nil { 262 | zap.S().Errorf("req %s remoteRespPathsInfos Unmarshal err.%v", pathsInfoUrl, err) 263 | return nil, myerr.NewAppendCode(http.StatusInternalServerError, fmt.Sprintf("%v", err)) 264 | } 265 | for _, item := range remoteRespPathsInfos { 266 | // 对单个文件pathsInfo做存储 267 | if apiPath, ok := remoteReqFilePathMap[item.Path]; ok { 268 | if err = util.MakeDirs(apiPath); err != nil { 269 | zap.S().Errorf("create %s dir err.%v", apiPath, err) 270 | continue 271 | } 272 | b, _ := sonic.Marshal([]common.PathsInfo{item}) // 转成单个文件的切片 273 | if err = f.WriteCacheRequest(apiPath, response.StatusCode, response.ExtractHeaders(response.Headers), b); err != nil { 274 | zap.S().Errorf("WriteCacheRequest err.%s,%v", apiPath, err) 275 | continue 276 | } 277 | } 278 | } 279 | ret = append(ret, remoteRespPathsInfos...) 280 | } 281 | return ret, nil 282 | } 283 | 284 | func (f *FileDao) pathsInfoProxy(targetUrl, authorization string, filePaths []string) (*common.Response, error) { 285 | data := map[string]interface{}{ 286 | "paths": filePaths, 287 | } 288 | jsonData, err := sonic.Marshal(data) 289 | if err != nil { 290 | return nil, err 291 | } 292 | headers := map[string]string{} 293 | if authorization != "" { 294 | headers["authorization"] = authorization 295 | } 296 | return util.RetryRequest(func() (*common.Response, error) { 297 | return util.Post(targetUrl, "application/json", jsonData, headers) 298 | }) 299 | } 300 | 301 | func (f *FileDao) FileChunkGet(c echo.Context, hfUrl, blobsFile, filesPath, orgRepo, fileName, authorization string, fileSize, startPos, endPos int64, respHeaders map[string]string) error { 302 | responseChan := make(chan []byte, config.SysConfig.Download.RespChanSize) 303 | source := util.Itoa(c.Get(consts.PromSource)) 304 | bgCtx := context.WithValue(c.Request().Context(), consts.PromSource, source) 305 | ctx, cancel := context.WithCancel(bgCtx) 306 | defer func() { 307 | cancel() 308 | }() 309 | go downloader.FileDownload(ctx, hfUrl, blobsFile, filesPath, orgRepo, fileName, authorization, fileSize, startPos, endPos, responseChan) 310 | if err := util.ResponseStream(c, fmt.Sprintf("%s/%s", orgRepo, fileName), respHeaders, responseChan); err != nil { 311 | zap.S().Warnf("FileChunkGet stream err.%v", err) 312 | return util.ErrorProxyTimeout(c) 313 | } 314 | return nil 315 | } 316 | 317 | func (f *FileDao) WhoamiV2Generator(c echo.Context) error { 318 | newHeaders := make(http.Header) 319 | for k, vv := range c.Request().Header { 320 | lowerKey := strings.ToLower(k) 321 | if lowerKey == "host" { 322 | continue 323 | } 324 | newHeaders[lowerKey] = vv 325 | } 326 | 327 | targetURL, err := url.Parse(config.SysConfig.GetHFURLBase()) 328 | if err != nil { 329 | zap.L().Error("Failed to parse base URL", zap.Error(err)) 330 | return echo.NewHTTPError(http.StatusInternalServerError, "Internal Server Error") 331 | } 332 | targetURL.Path = path.Join(targetURL.Path, "/api/whoami-v2") 333 | 334 | // targetURL := "https://huggingface.co/api/whoami-v2" 335 | zap.S().Debugf("exec WhoamiV2Generator:targetURL:%s,host:%s", targetURL.String(), config.SysConfig.GetHfNetLoc()) 336 | // Creating a proxy request 337 | req, err := http.NewRequest("GET", targetURL.String(), nil) 338 | if err != nil { 339 | zap.L().Error("Failed to create request", zap.Error(err)) 340 | return echo.NewHTTPError(http.StatusInternalServerError, "Internal Server Error") 341 | } 342 | req.Header = newHeaders 343 | // req.Host = "huggingface.co" 344 | req.Host = config.SysConfig.GetHfNetLoc() 345 | 346 | client := &http.Client{ 347 | Timeout: 10 * time.Second, 348 | } 349 | resp, err := client.Do(req) 350 | if err != nil { 351 | zap.L().Error("Failed to forward request", zap.Error(err)) 352 | return echo.NewHTTPError(http.StatusBadGateway, "Bad Gateway") 353 | } 354 | defer resp.Body.Close() 355 | 356 | // Processing Response Headers 357 | responseHeaders := make(http.Header) 358 | for k, vv := range resp.Header { 359 | lowerKey := strings.ToLower(k) 360 | if lowerKey == "content-encoding" || lowerKey == "content-length" { 361 | continue 362 | } 363 | for _, v := range vv { 364 | responseHeaders.Add(lowerKey, v) 365 | } 366 | } 367 | 368 | // Setting the response header 369 | for k, vv := range responseHeaders { 370 | for _, v := range vv { 371 | c.Response().Header().Add(k, v) 372 | } 373 | } 374 | 375 | c.Response().WriteHeader(resp.StatusCode) 376 | 377 | // Streaming response content 378 | _, err = io.Copy(c.Response().Writer, resp.Body) 379 | if err != nil { 380 | zap.L().Error("Failed to stream response", zap.Error(err)) 381 | } 382 | 383 | return nil 384 | } 385 | 386 | func (f *FileDao) WriteCacheRequest(apiPath string, statusCode int, headers map[string]string, content []byte) error { 387 | cacheContent := common.CacheContent{ 388 | StatusCode: statusCode, 389 | Headers: headers, 390 | Content: hex.EncodeToString(content), 391 | } 392 | return util.WriteDataToFile(apiPath, cacheContent) 393 | } 394 | 395 | func (f *FileDao) ReadCacheRequest(apiPath string) (*common.CacheContent, error) { 396 | cacheContent := common.CacheContent{} 397 | bytes, err := util.ReadFileToBytes(apiPath) 398 | if err != nil { 399 | return nil, myerr.Wrap("ReadFileToBytes err.", err) 400 | } 401 | if err = sonic.Unmarshal(bytes, &cacheContent); err != nil { 402 | return nil, err 403 | } 404 | decodeByte, err := hex.DecodeString(cacheContent.Content) 405 | if err != nil { 406 | return nil, myerr.Wrap("DecodeString err.", err) 407 | } 408 | cacheContent.OriginContent = decodeByte 409 | return &cacheContent, nil 410 | } 411 | 412 | func (f *FileDao) ReposGenerator(c echo.Context) error { 413 | reposPath := config.SysConfig.Repos() 414 | 415 | datasets, _ := filepath.Glob(filepath.Join(reposPath, "api/datasets/*/*")) 416 | datasetsRepos := util.ProcessPaths(datasets) 417 | 418 | models, _ := filepath.Glob(filepath.Join(reposPath, "api/models/*/*")) 419 | modelsRepos := util.ProcessPaths(models) 420 | 421 | spaces, _ := filepath.Glob(filepath.Join(reposPath, "api/spaces/*/*")) 422 | spacesRepos := util.ProcessPaths(spaces) 423 | 424 | return c.Render(http.StatusOK, "repos.html", map[string]interface{}{ 425 | "datasets_repos": datasetsRepos, 426 | "models_repos": modelsRepos, 427 | "spaces_repos": spacesRepos, 428 | }) 429 | } 430 | 431 | func parseRangeParams(fileRange string, fileSize int64) (int64, int64) { 432 | if strings.Contains(fileRange, "/") { 433 | split := strings.SplitN(fileRange, "/", 2) 434 | fileRange = split[0] 435 | } 436 | if strings.HasPrefix(fileRange, "bytes=") { 437 | fileRange = fileRange[6:] 438 | } 439 | parts := strings.Split(fileRange, "-") 440 | if len(parts) != 2 { 441 | panic("file range err.") 442 | } 443 | var startPos, endPos int64 444 | if len(parts[0]) != 0 { 445 | startPos = util.Atoi64(parts[0]) 446 | } else { 447 | startPos = 0 448 | } 449 | if len(parts[1]) != 0 { 450 | endPos = util.Atoi64(parts[1]) 451 | } else { 452 | endPos = fileSize - 1 453 | } 454 | return startPos, endPos 455 | } 456 | -------------------------------------------------------------------------------- /internal/dao/meta_dao.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dao 16 | 17 | import ( 18 | "fmt" 19 | 20 | "dingospeed/pkg/common" 21 | "dingospeed/pkg/config" 22 | "dingospeed/pkg/consts" 23 | "dingospeed/pkg/util" 24 | 25 | "github.com/labstack/echo/v4" 26 | "go.uber.org/zap" 27 | ) 28 | 29 | type MetaDao struct { 30 | fileDao *FileDao 31 | } 32 | 33 | func NewMetaDao(fileDao *FileDao) *MetaDao { 34 | return &MetaDao{ 35 | fileDao: fileDao, 36 | } 37 | } 38 | 39 | func (m *MetaDao) MetaGetGenerator(c echo.Context, repoType, org, repo, commit, method string, writeResp bool) error { 40 | orgRepo := util.GetOrgRepo(org, repo) 41 | apiDir := fmt.Sprintf("%s/api/%s/%s/revision/%s", config.SysConfig.Repos(), repoType, orgRepo, commit) 42 | apiMetaPath := fmt.Sprintf("%s/%s", apiDir, fmt.Sprintf("meta_%s.json", method)) 43 | err := util.MakeDirs(apiMetaPath) 44 | if err != nil { 45 | zap.S().Errorf("create %s dir err.%v", apiDir, err) 46 | return err 47 | } 48 | request := c.Request() 49 | authorization := request.Header.Get("authorization") 50 | // 若缓存文件存在，且为离线模式，从缓存读取 51 | if util.FileExists(apiMetaPath) && !config.SysConfig.Online() { 52 | return m.MetaCacheGenerator(c, repo, apiMetaPath) 53 | } else { 54 | return m.MetaProxyGenerator(c, repoType, org, repo, commit, method, authorization, apiMetaPath, writeResp) 55 | } 56 | } 57 | 58 | func (m *MetaDao) MetaCacheGenerator(c echo.Context, repo, apiMetaPath string) error { 59 | cacheContent, err := m.fileDao.ReadCacheRequest(apiMetaPath) 60 | if err != nil { 61 | return err 62 | } 63 | var bodyStreamChan = make(chan []byte, consts.RespChanSize) 64 | bodyStreamChan <- cacheContent.OriginContent 65 | close(bodyStreamChan) 66 | err = util.ResponseStream(c, repo, cacheContent.Headers, bodyStreamChan) 67 | if err != nil { 68 | return err 69 | } 70 | return nil 71 | } 72 | 73 | // 请求api文件 74 | 75 | func (m *MetaDao) MetaProxyGenerator(c echo.Context, repoType, org, repo, commit, method, authorization, apiMetaPath string, writeResp bool) error { 76 | orgRepo := util.GetOrgRepo(org, repo) 77 | metaUrl := fmt.Sprintf("%s/api/%s/%s/revision/%s", config.SysConfig.GetHFURLBase(), repoType, orgRepo, commit) 78 | headers := map[string]string{} 79 | if authorization != "" { 80 | headers["authorization"] = authorization 81 | } 82 | if method == consts.RequestTypeHead { 83 | resp, err := util.RetryRequest(func() (*common.Response, error) { 84 | return util.Head(metaUrl, headers, config.SysConfig.GetReqTimeOut()) 85 | }) 86 | if err != nil { 87 | zap.S().Errorf("head %s err.%v", metaUrl, err) 88 | return util.ErrorEntryNotFound(c) 89 | } 90 | extractHeaders := resp.ExtractHeaders(resp.Headers) 91 | return util.ResponseHeaders(c, extractHeaders) 92 | } else if method == consts.RequestTypeGet { 93 | resp, err := util.RetryRequest(func() (*common.Response, error) { 94 | return util.Get(metaUrl, headers, config.SysConfig.GetReqTimeOut()) 95 | }) 96 | if err != nil { 97 | zap.S().Errorf("get %s err.%v", metaUrl, err) 98 | return util.ErrorEntryNotFound(c) 99 | } 100 | extractHeaders := resp.ExtractHeaders(resp.Headers) 101 | if writeResp { 102 | var bodyStreamChan = make(chan []byte, consts.RespChanSize) 103 | bodyStreamChan <- resp.Body 104 | close(bodyStreamChan) 105 | if err = util.ResponseStream(c, repo, extractHeaders, bodyStreamChan); err != nil { 106 | return err 107 | } 108 | } 109 | if err = m.fileDao.WriteCacheRequest(apiMetaPath, resp.StatusCode, extractHeaders, resp.Body); err != nil { 110 | zap.S().Errorf("writeCacheRequest err.%v", err) 111 | return nil 112 | } 113 | } 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /internal/data/data.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cache 16 | 17 | import ( 18 | "dingospeed/pkg/common" 19 | "dingospeed/pkg/config" 20 | 21 | "github.com/dgraph-io/ristretto/v2" 22 | ) 23 | 24 | // 缓存预读取的文件块，默认每个文件16个块 25 | 26 | var FileBlockCache Cache[string, []byte] 27 | 28 | func InitCache() { 29 | if FileBlockCache == nil { 30 | // 默认使用ristretto 31 | if config.SysConfig.Cache.Type == 0 { 32 | cache, err := ristretto.NewCache(&ristretto.Config[string, []byte]{ 33 | NumCounters: 1e7, // 计数器数量，用于预估缓存项的使用频率 34 | MaxCost: 1 << 30, // 缓存的最大成本，这里设置为 1GB 35 | BufferItems: 64, // 每个分片的缓冲区大小 36 | }) 37 | if err != nil { 38 | panic(err) 39 | } 40 | FileBlockCache = &RistrettoCache[string, []byte]{ 41 | RCache: cache, 42 | cost: 1, 43 | } 44 | } else { 45 | FileBlockCache = common.NewSafeMap[string, []byte]() 46 | } 47 | } 48 | } 49 | 50 | type Cache[K comparable, V any] interface { 51 | Set(key K, value V) 52 | Get(key K) (V, bool) 53 | Delete(key K) 54 | Wait() 55 | } 56 | 57 | type RistrettoCache[K string, V any] struct { 58 | RCache *ristretto.Cache[K, V] 59 | cost int64 60 | } 61 | 62 | func (f *RistrettoCache[K, V]) Set(key K, value V) { 63 | f.RCache.SetWithTTL(key, value, f.cost, config.SysConfig.GetPrefetchBlockTTL()) 64 | } 65 | 66 | func (f *RistrettoCache[K, V]) Get(key K) (V, bool) { 67 | return f.RCache.Get(key) 68 | } 69 | 70 | func (f *RistrettoCache[K, V]) Delete(key K) { 71 | f.RCache.Del(key) 72 | } 73 | func (f *RistrettoCache[K, V]) Wait() { 74 | f.RCache.Wait() 75 | } 76 | -------------------------------------------------------------------------------- /internal/downloader/bitset.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | ) 21 | 22 | // Bitset 结构体表示一个位集合 23 | type Bitset struct { 24 | size int64 25 | bits []byte 26 | } 27 | 28 | // NewBitset 创建一个新的 Bitset 对象 29 | func NewBitset(size int64) *Bitset { 30 | return &Bitset{ 31 | size: size, 32 | bits: make([]byte, (size+7)/8), 33 | } 34 | } 35 | 36 | // Set 将指定索引的位设置为 1 37 | func (b *Bitset) Set(index int64) error { 38 | if index < 0 || index >= b.size { 39 | return errors.New("Index out of range") 40 | } 41 | byteIndex := index / 8 42 | bitIndex := index % 8 43 | b.bits[byteIndex] |= 1 << bitIndex 44 | return nil 45 | } 46 | 47 | // Clear 将指定索引的位设置为 0 48 | func (b *Bitset) Clear(index int64) error { 49 | if index < 0 || index >= b.size { 50 | return errors.New("Index out of range") 51 | } 52 | byteIndex := index / 8 53 | bitIndex := index % 8 54 | b.bits[byteIndex] &= ^(1 << bitIndex) 55 | return nil 56 | } 57 | 58 | // Test 检查指定索引的位的值 59 | func (b *Bitset) Test(index int64) (bool, error) { 60 | if index < 0 || index >= b.size { 61 | return false, errors.New("Index out of range") 62 | } 63 | byteIndex := index / 8 64 | bitIndex := index % 8 65 | return (b.bits[byteIndex] & (1 << bitIndex)) != 0, nil 66 | } 67 | 68 | // String 返回 Bitset 的字符串表示 69 | func (b *Bitset) String() string { 70 | result := "" 71 | for _, byteVal := range b.bits { 72 | result += fmt.Sprintf("%08b", byteVal) 73 | } 74 | return result 75 | } 76 | -------------------------------------------------------------------------------- /internal/downloader/broadcaster.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | ) 22 | 23 | type Broadcaster struct { 24 | once sync.Once 25 | ctx context.Context 26 | msgChan chan bool 27 | listeners []chan bool 28 | mu sync.RWMutex 29 | cancel context.CancelFunc 30 | } 31 | 32 | // 每个下载请求需要注册广播实例 33 | 34 | func NewBroadcaster(ctx context.Context) *Broadcaster { 35 | ctx, cancel := context.WithCancel(ctx) 36 | b := &Broadcaster{ 37 | msgChan: make(chan bool, 1), 38 | listeners: make([]chan bool, 0), 39 | ctx: ctx, 40 | cancel: cancel, 41 | } 42 | return b 43 | } 44 | 45 | func (b *Broadcaster) AddListener() chan bool { 46 | b.once.Do(func() { 47 | go func() { 48 | for { 49 | select { 50 | case state, ok := <-b.msgChan: 51 | if !ok { 52 | return 53 | } 54 | b.mu.RLock() 55 | // 复制监听器列表以避免并发修改问题 56 | listenersCopy := make([]chan bool, len(b.listeners)) 57 | copy(listenersCopy, b.listeners) 58 | b.mu.RUnlock() 59 | for _, listener := range listenersCopy { 60 | select { 61 | case listener <- state: 62 | case <-b.ctx.Done(): 63 | return 64 | } 65 | } 66 | case <-b.ctx.Done(): 67 | return 68 | } 69 | } 70 | }() 71 | }) 72 | listener := make(chan bool) 73 | b.mu.Lock() 74 | b.listeners = append(b.listeners, listener) 75 | b.mu.Unlock() 76 | return listener 77 | } 78 | 79 | func (b *Broadcaster) SendMsg(msg bool) { 80 | b.mu.RLock() 81 | if len(b.listeners) == 0 { 82 | return 83 | } 84 | b.mu.RUnlock() 85 | select { 86 | case b.msgChan <- msg: 87 | case <-b.ctx.Done(): 88 | } 89 | } 90 | 91 | func (b *Broadcaster) Close() { 92 | close(b.msgChan) 93 | b.cancel() 94 | b.mu.RLock() 95 | for _, listener := range b.listeners { 96 | close(listener) 97 | } 98 | b.mu.RUnlock() 99 | } 100 | 101 | func worker(ctx context.Context, listener chan bool, id int) { 102 | for { 103 | select { 104 | case newState := <-listener: 105 | fmt.Printf("Worker %d: Received new state %v\n", id, newState) 106 | // default: 107 | // fmt.Printf("Worker %d: Waiting for state change...\n", id) 108 | // time.Sleep(500 * time.Millisecond) 109 | case <-ctx.Done(): 110 | return 111 | } 112 | } 113 | } 114 | 115 | // 116 | // func main() { 117 | // // ctx, cancel := context.WithCancel(context.Background()) 118 | // ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 119 | // 120 | // b := NewBroadcaster(ctx) 121 | // var wg sync.WaitGroup 122 | // 123 | // for i := 0; i < 3; i++ { 124 | // wg.Add(1) 125 | // listener := b.addListener() 126 | // go func(i int) { 127 | // defer wg.Done() 128 | // worker(ctx, listener, i) 129 | // }(i) 130 | // } 131 | // time.Sleep(1 * time.Second) 132 | // b.sendMsg(false) 133 | // b.sendMsg(true) 134 | // time.Sleep(1 * time.Second) 135 | // fmt.Println("执行取消") 136 | // cancel() 137 | // time.Sleep(2 * time.Second) 138 | // wg.Wait() 139 | // } 140 | -------------------------------------------------------------------------------- /internal/downloader/cache_task.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "context" 19 | 20 | "dingospeed/pkg/util" 21 | 22 | "go.uber.org/zap" 23 | ) 24 | 25 | type DownloadTask struct { 26 | TaskNo int 27 | RangeStartPos int64 28 | RangeEndPos int64 29 | TaskSize int 30 | FileName string 31 | DingFile *DingCache `json:"-"` 32 | ResponseChan chan []byte `json:"-"` 33 | Context context.Context `json:"-"` 34 | blobsFile string 35 | filesPath string 36 | orgRepo string 37 | } 38 | 39 | type CacheFileTask struct { 40 | DownloadTask 41 | } 42 | 43 | func NewCacheFileTask(taskNo int, rangeStartPos int64, rangeEndPos int64) *CacheFileTask { 44 | c := &CacheFileTask{} 45 | c.TaskNo = taskNo 46 | c.RangeStartPos = rangeStartPos 47 | c.RangeEndPos = rangeEndPos 48 | return c 49 | } 50 | 51 | func (c CacheFileTask) DoTask() { 52 | } 53 | 54 | func (c CacheFileTask) OutResult() { 55 | startBlock := c.RangeStartPos / c.DingFile.getBlockSize() 56 | endBlock := (c.RangeEndPos - 1) / c.DingFile.getBlockSize() 57 | curPos := c.RangeStartPos 58 | for curBlock := startBlock; curBlock <= endBlock; curBlock++ { 59 | if c.Context.Err() != nil { 60 | zap.S().Warnf("for cache ctx err :%s", c.FileName) 61 | return 62 | } 63 | _, blockStartPos, blockEndPos := getBlockInfo(curPos, c.DingFile.getBlockSize(), c.DingFile.GetFileSize()) 64 | hasBlockBool, err := c.DingFile.HasBlock(curBlock) 65 | if err != nil { 66 | zap.S().Errorf("HasBlock err. file:%s, curBlock:%d, curPos:%d, %v", c.FileName, curBlock, curPos, err) 67 | continue 68 | } 69 | if !hasBlockBool { 70 | zap.S().Errorf("block not exist. file:%s, curBlock:%d,curPos:%d", c.FileName, curBlock, curPos) 71 | break 72 | } 73 | rawBlock, err := c.DingFile.ReadBlock(curBlock) 74 | if err != nil { 75 | zap.S().Errorf("ReadBlock err file:%s, %v", c.FileName, err) 76 | continue 77 | } 78 | sPos := max(c.RangeStartPos, blockStartPos) - blockStartPos 79 | ePos := min(c.RangeEndPos, blockEndPos) - blockStartPos 80 | rawLen := int64(len(rawBlock)) 81 | if rawLen == 0 || sPos > rawLen { 82 | zap.S().Errorf("read rawBlock err,%s, rawLen:%d, sPos:%d,ePos:%d, %v", c.FileName, rawLen, sPos, ePos, err) 83 | continue 84 | } 85 | if ePos > rawLen { 86 | zap.S().Warnf("block incomplete,%s, rawLen:%d, sPos:%d,ePos:%d, %v", c.FileName, rawLen, sPos, ePos, err) 87 | ePos = rawLen 88 | } 89 | chunk := rawBlock[sPos:ePos] 90 | select { 91 | case c.ResponseChan <- chunk: 92 | case <-c.Context.Done(): 93 | return 94 | } 95 | curPos += int64(len(chunk)) 96 | } 97 | if curPos != c.RangeEndPos { 98 | zap.S().Errorf("file:%s, cache range from %d to %d is incomplete.", c.FileName, c.RangeStartPos, c.RangeEndPos) 99 | } 100 | zap.S().Infof("cache file out:%s/%s, taskNo:%d, size:%d, startPos:%d, endPos:%d", c.orgRepo, c.FileName, c.TaskNo, c.TaskSize, c.RangeStartPos, c.RangeEndPos) 101 | if err := util.CreateSymlinkIfNotExists(c.blobsFile, c.filesPath); err != nil { 102 | zap.S().Errorf("filesPath:%s is not link", c.filesPath) 103 | } 104 | } 105 | 106 | func (c CacheFileTask) GetResponseChan() chan []byte { 107 | return c.ResponseChan 108 | } 109 | -------------------------------------------------------------------------------- /internal/downloader/downloader.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | "time" 21 | 22 | "dingospeed/pkg/common" 23 | "dingospeed/pkg/config" 24 | 25 | "go.uber.org/zap" 26 | ) 27 | 28 | // 整个文件 29 | func FileDownload(ctx context.Context, hfUrl, blobsFile, filesPath, orgRepo, fileName, authorization string, fileSize, startPos, endPos int64, responseChan chan []byte) { 30 | var ( 31 | remoteTasks []*RemoteFileTask 32 | wg sync.WaitGroup 33 | ) 34 | defer close(responseChan) 35 | dingCacheManager := GetInstance() 36 | dingFile, err := dingCacheManager.GetDingFile(blobsFile, fileSize) 37 | if err != nil { 38 | zap.S().Errorf("GetDingFile err.%v", err) 39 | return 40 | } 41 | defer func() { 42 | dingCacheManager.ReleasedDingFile(blobsFile) 43 | }() 44 | 45 | tasks := getContiguousRanges(ctx, dingFile, startPos, endPos) 46 | taskSize := len(tasks) 47 | for i := 0; i < taskSize; i++ { 48 | if ctx.Err() != nil { 49 | zap.S().Errorf("FileDownload cancelled: %v", ctx.Err()) 50 | return 51 | } 52 | task := tasks[i] 53 | if remote, ok := task.(*RemoteFileTask); ok { 54 | remote.Context = ctx 55 | remote.DingFile = dingFile 56 | remote.authorization = authorization 57 | remote.hfUrl = hfUrl 58 | remote.Queue = make(chan []byte, getQueueSize(remote.RangeStartPos, remote.RangeEndPos)) 59 | remote.ResponseChan = responseChan 60 | remote.TaskSize = taskSize 61 | remote.FileName = fileName 62 | remote.blobsFile = blobsFile 63 | remote.filesPath = filesPath 64 | remote.orgRepo = orgRepo 65 | remoteTasks = append(remoteTasks, remote) 66 | } else if cache, ok := task.(*CacheFileTask); ok { 67 | cache.Context = ctx 68 | cache.DingFile = dingFile 69 | cache.TaskSize = taskSize 70 | cache.FileName = fileName 71 | cache.blobsFile = blobsFile 72 | cache.filesPath = filesPath 73 | cache.orgRepo = orgRepo 74 | cache.ResponseChan = responseChan 75 | } 76 | } 77 | wg.Add(1) 78 | go func() { 79 | defer func() { 80 | wg.Done() 81 | }() 82 | for i := 0; i < len(tasks); i++ { 83 | if ctx.Err() != nil { 84 | break 85 | } 86 | task := tasks[i] 87 | if i == 0 { 88 | task.GetResponseChan() <- []byte{} // 先建立长连接 89 | } 90 | task.OutResult() 91 | } 92 | }() 93 | if len(remoteTasks) > 0 { 94 | wg.Add(1) 95 | go func() { 96 | defer func() { 97 | wg.Done() 98 | }() 99 | startRemoteDownload(ctx, remoteTasks) 100 | }() 101 | } 102 | wg.Wait() // 等待协程池所有远程下载任务执行完毕 103 | } 104 | 105 | func getQueueSize(rangeStartPos, rangeEndPos int64) int64 { 106 | bufSize := min(config.SysConfig.Download.RemoteFileBufferSize, rangeEndPos-rangeStartPos) 107 | return bufSize/config.SysConfig.Download.RespChunkSize + 1 108 | } 109 | 110 | func startRemoteDownload(ctx context.Context, remoteFileTasks []*RemoteFileTask) { 111 | var pool *common.Pool 112 | taskLen := len(remoteFileTasks) 113 | if taskLen == 0 { 114 | return 115 | } else if taskLen >= config.SysConfig.Download.GoroutineMaxNumPerFile { 116 | pool = common.NewPool(config.SysConfig.Download.GoroutineMaxNumPerFile) 117 | } else { 118 | pool = common.NewPool(taskLen) 119 | } 120 | defer pool.Close() 121 | for i := 0; i < taskLen; i++ { 122 | if ctx.Err() != nil { 123 | return 124 | } 125 | task := remoteFileTasks[i] 126 | if err := pool.Submit(ctx, task); err != nil { 127 | zap.S().Errorf("submit task err.%v", err) 128 | return 129 | } 130 | if config.SysConfig.GetRemoteFileRangeWaitTime() != 0 { 131 | time.Sleep(config.SysConfig.GetRemoteFileRangeWaitTime()) 132 | } 133 | } 134 | } 135 | 136 | // 将文件的偏移量分为cache和remote，对针对remote按照指定的RangeSize做切分 137 | 138 | func getContiguousRanges(ctx context.Context, dingFile *DingCache, startPos, endPos int64) (tasks []common.Task) { 139 | if startPos == 0 && endPos == 0 { 140 | return 141 | } 142 | if startPos < 0 || endPos <= startPos || endPos > dingFile.GetFileSize() { 143 | zap.S().Errorf("Invalid startPos or endPos: startPos=%d, endPos=%d", startPos, endPos) 144 | return 145 | } 146 | startBlock := startPos / dingFile.getBlockSize() 147 | endBlock := (endPos - 1) / dingFile.getBlockSize() 148 | 149 | rangeStartPos, curPos := startPos, startPos 150 | blockExists, err := dingFile.HasBlock(startBlock) 151 | if err != nil { 152 | zap.S().Errorf("Failed to check block existence: %v", err) 153 | return 154 | } 155 | rangeIsRemote := !blockExists // 不存在，从远程获取，为true 156 | taskNo := 0 157 | for curBlock := startBlock; curBlock <= endBlock; curBlock++ { 158 | if ctx.Err() != nil { 159 | return 160 | } 161 | _, _, blockEndPos := getBlockInfo(curPos, dingFile.getBlockSize(), dingFile.GetFileSize()) 162 | blockExists, err = dingFile.HasBlock(curBlock) 163 | if err != nil { 164 | zap.S().Errorf("HasBlock err. curBlock:%d,curPos:%d, %v", curBlock, curPos, err) 165 | return 166 | } 167 | curIsRemote := !blockExists // 不存在，从远程获取，为true，存在为false。 168 | if rangeIsRemote != curIsRemote { 169 | if rangeStartPos < curPos { 170 | if rangeIsRemote { 171 | tasks = splitRemoteRange(tasks, rangeStartPos, curPos, &taskNo) 172 | } else { 173 | c := NewCacheFileTask(taskNo, rangeStartPos, curPos) 174 | tasks = append(tasks, c) 175 | taskNo++ 176 | } 177 | } 178 | rangeStartPos = curPos 179 | rangeIsRemote = curIsRemote 180 | } 181 | curPos = blockEndPos 182 | } 183 | if rangeIsRemote { 184 | tasks = splitRemoteRange(tasks, rangeStartPos, endPos, &taskNo) 185 | } else { 186 | c := NewCacheFileTask(taskNo, rangeStartPos, endPos) 187 | tasks = append(tasks, c) 188 | taskNo++ 189 | } 190 | return 191 | } 192 | 193 | func splitRemoteRange(tasks []common.Task, startPos, endPos int64, taskNo *int) []common.Task { 194 | rangeSize := config.SysConfig.Download.RemoteFileRangeSize 195 | if rangeSize == 0 { 196 | c := NewRemoteFileTask(*taskNo, startPos, endPos) 197 | tasks = append(tasks, c) 198 | return tasks 199 | } 200 | for start := startPos; start < endPos; { 201 | end := start + rangeSize 202 | if end > endPos { 203 | end = endPos 204 | } 205 | c := NewRemoteFileTask(*taskNo, start, end) 206 | tasks = append(tasks, c) 207 | *taskNo++ 208 | start = end 209 | } 210 | return tasks 211 | } 212 | 213 | // get_block_info 函数 214 | func getBlockInfo(pos, blockSize, fileSize int64) (int64, int64, int64) { 215 | curBlock := pos / blockSize 216 | blockStartPos := curBlock * blockSize 217 | blockEndPos := min((curBlock+1)*blockSize, fileSize) 218 | return curBlock, blockStartPos, blockEndPos 219 | } 220 | -------------------------------------------------------------------------------- /internal/downloader/file.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "bytes" 19 | "encoding/binary" 20 | "errors" 21 | "fmt" 22 | "os" 23 | "sync" 24 | 25 | cache "dingospeed/internal/data" 26 | "dingospeed/pkg/config" 27 | "dingospeed/pkg/util" 28 | 29 | "go.uber.org/zap" 30 | ) 31 | 32 | const ( 33 | CURRENT_OLAH_CACHE_VERSION = 8 34 | // DEFAULT_BLOCK_MASK_MAX = 30 35 | DEFAULT_BLOCK_MASK_MAX = 1024 * 1024 36 | 37 | cost = 1 38 | ) 39 | 40 | var magicNumber = [4]byte{'O', 'L', 'A', 'H'} 41 | 42 | // DingCacheHeader 结构体表示 Olah 缓存文件的头部 43 | type DingCacheHeader struct { 44 | MagicNumber [4]byte 45 | Version int64 46 | BlockSize int64 47 | FileSize int64 48 | BlockMaskSize int64 49 | BlockNumber int64 50 | BlockMask *Bitset 51 | } 52 | 53 | // NewDingCacheHeader 创建一个新的 DingCacheHeader 对象 54 | func NewDingCacheHeader(version, blockSize, fileSize int64) *DingCacheHeader { 55 | blockNumber := (fileSize + blockSize - 1) / blockSize 56 | blockMask := NewBitset(int64(DEFAULT_BLOCK_MASK_MAX)) 57 | return &DingCacheHeader{ 58 | MagicNumber: magicNumber, 59 | Version: version, 60 | BlockSize: blockSize, 61 | FileSize: fileSize, 62 | BlockMaskSize: DEFAULT_BLOCK_MASK_MAX, 63 | BlockNumber: blockNumber, 64 | BlockMask: blockMask, 65 | } 66 | } 67 | 68 | // GetHeaderSize 返回头部的大小 69 | func (h *DingCacheHeader) GetHeaderSize() int64 { 70 | return int64(36 + len(h.BlockMask.bits)) 71 | } 72 | 73 | // Read 从文件流中读取头部信息 74 | func (h *DingCacheHeader) Read(f *os.File) error { 75 | magic := make([]byte, 4) 76 | if _, err := f.Read(magic); err != nil { 77 | return errors.New("read magic 4 bytes err") 78 | } 79 | if !bytes.Equal(magic, []byte{'O', 'L', 'A', 'H'}) { 80 | return errors.New("file is not a Olah cache file") 81 | } 82 | h.MagicNumber = magicNumber 83 | if err := binary.Read(f, binary.LittleEndian, &h.Version); err != nil { 84 | return err 85 | } 86 | if err := binary.Read(f, binary.LittleEndian, &h.BlockSize); err != nil { 87 | return err 88 | } 89 | if err := binary.Read(f, binary.LittleEndian, &h.FileSize); err != nil { 90 | return err 91 | } 92 | if err := binary.Read(f, binary.LittleEndian, &h.BlockMaskSize); err != nil { 93 | return err 94 | } 95 | h.BlockNumber = (h.FileSize + h.BlockSize - 1) / h.BlockSize 96 | h.BlockMask = NewBitset(h.BlockMaskSize) 97 | if _, err := f.Read(h.BlockMask.bits); err != nil { 98 | return err 99 | } 100 | return h.ValidHeader() 101 | } 102 | 103 | func (h *DingCacheHeader) ValidHeader() error { 104 | if h.FileSize > h.BlockMaskSize*h.BlockSize { 105 | return fmt.Errorf("the size of file %d is out of the max capability of container (%d * %d)", h.FileSize, h.BlockMaskSize, h.BlockSize) 106 | } 107 | if h.Version < CURRENT_OLAH_CACHE_VERSION { 108 | return fmt.Errorf("the Olah Cache file is created by older version Olah. Please remove cache files and retry") 109 | } 110 | if h.Version > CURRENT_OLAH_CACHE_VERSION { 111 | return fmt.Errorf("the Olah Cache file is created by newer version Olah. Please remove cache files and retry") 112 | } 113 | return nil 114 | } 115 | 116 | // Write 将头部信息写入文件流 117 | func (h *DingCacheHeader) Write(f *os.File) error { 118 | if _, err := f.Write(h.MagicNumber[:]); err != nil { 119 | return err 120 | } 121 | if err := binary.Write(f, binary.LittleEndian, h.Version); err != nil { 122 | return err 123 | } 124 | if err := binary.Write(f, binary.LittleEndian, h.BlockSize); err != nil { 125 | return err 126 | } 127 | if err := binary.Write(f, binary.LittleEndian, h.FileSize); err != nil { 128 | return err 129 | } 130 | if err := binary.Write(f, binary.LittleEndian, h.BlockMaskSize); err != nil { 131 | return err 132 | } 133 | if _, err := f.Write(h.BlockMask.bits); err != nil { 134 | return err 135 | } 136 | return nil 137 | } 138 | 139 | // DingCache 结构体表示 Olah 缓存文件 140 | type DingCache struct { 141 | path string 142 | header *DingCacheHeader 143 | isOpen bool 144 | headerLock sync.RWMutex 145 | fileLock sync.RWMutex 146 | } 147 | 148 | // NewDingCache 创建一个新的 DingCache 对象 149 | func NewDingCache(path string, blockSize int64) (*DingCache, error) { 150 | cache := &DingCache{ 151 | path: path, 152 | header: nil, 153 | isOpen: false, 154 | headerLock: sync.RWMutex{}, 155 | } 156 | if err := cache.Open(path, blockSize); err != nil { 157 | return nil, err 158 | } 159 | return cache, nil 160 | } 161 | 162 | // Open 打开缓存文件 163 | func (c *DingCache) Open(path string, blockSize int64) error { 164 | if c.isOpen { 165 | return errors.New("this file has been open") 166 | } 167 | if _, err := os.Stat(path); err == nil { // 文件存在 168 | c.headerLock.Lock() 169 | defer c.headerLock.Unlock() 170 | f, err := os.OpenFile(path, os.O_RDONLY, 0644) 171 | if err != nil { 172 | return err 173 | } 174 | defer f.Close() 175 | c.header = &DingCacheHeader{} 176 | if err := c.header.Read(f); err != nil { 177 | return err 178 | } 179 | } else { 180 | c.headerLock.Lock() 181 | defer c.headerLock.Unlock() 182 | f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) 183 | if err != nil { 184 | return err 185 | } 186 | defer f.Close() 187 | c.header = NewDingCacheHeader(CURRENT_OLAH_CACHE_VERSION, blockSize, 0) 188 | if err := c.header.Write(f); err != nil { 189 | return err 190 | } 191 | } 192 | c.isOpen = true 193 | return nil 194 | } 195 | 196 | // Close 关闭缓存文件 197 | func (c *DingCache) Close() error { 198 | if !c.isOpen { 199 | return errors.New("This file has been close.") 200 | } 201 | c.fileLock.Lock() 202 | defer c.fileLock.Unlock() 203 | if err := c.flushHeader(); err != nil { 204 | return err 205 | } 206 | c.path = "" 207 | c.header = nil 208 | c.isOpen = false 209 | return nil 210 | } 211 | 212 | func (c *DingCache) flushHeader() error { 213 | // c.headerLock.Lock() 214 | // defer c.headerLock.Unlock() 215 | f, err := os.OpenFile(c.path, os.O_RDWR, 0644) 216 | if err != nil { 217 | return err 218 | } 219 | defer f.Close() 220 | if _, err := f.Seek(0, 0); err != nil { 221 | return err 222 | } 223 | return c.header.Write(f) 224 | } 225 | 226 | // getFileSize 返回文件大小 227 | func (c *DingCache) GetFileSize() int64 { 228 | c.headerLock.RLock() 229 | defer c.headerLock.RUnlock() 230 | return c.header.FileSize 231 | } 232 | 233 | // getBlockNumber 返回块数 234 | func (c *DingCache) getBlockNumber() int64 { 235 | c.headerLock.RLock() 236 | defer c.headerLock.RUnlock() 237 | return c.header.BlockNumber 238 | } 239 | 240 | // getBlockSize 返回块大小 241 | func (c *DingCache) getBlockSize() int64 { 242 | c.headerLock.RLock() 243 | defer c.headerLock.RUnlock() 244 | return c.header.BlockSize 245 | } 246 | 247 | // getHeaderSize 返回头部大小 248 | func (c *DingCache) getHeaderSize() int64 { 249 | c.headerLock.RLock() 250 | defer c.headerLock.RUnlock() 251 | return c.header.GetHeaderSize() 252 | } 253 | 254 | func (c *DingCache) resizeHeader(blockNum, fileSize int64) error { 255 | c.headerLock.RLock() 256 | defer c.headerLock.RUnlock() 257 | c.header.BlockNumber = blockNum 258 | c.header.FileSize = fileSize 259 | return c.header.ValidHeader() 260 | } 261 | 262 | func (c *DingCache) setHeaderBlock(blockIndex int64) error { 263 | // c.headerLock.Lock() 264 | // defer c.headerLock.Unlock() 265 | return c.header.BlockMask.Set(blockIndex) 266 | } 267 | 268 | // testHeaderBlock 测试头部块信息 269 | func (c *DingCache) testHeaderBlock(blockIndex int64) (bool, error) { 270 | c.headerLock.RLock() 271 | defer c.headerLock.RUnlock() 272 | return c.header.BlockMask.Test(blockIndex) 273 | } 274 | 275 | // padBlock 填充块数据 276 | func (c *DingCache) padBlock(rawBlock []byte) []byte { 277 | blockSize := c.getBlockSize() 278 | if int64(len(rawBlock)) < blockSize { 279 | return append(rawBlock, bytes.Repeat([]byte{0}, int(blockSize)-len(rawBlock))...) 280 | } 281 | return rawBlock 282 | } 283 | 284 | // HasBlock 检查块是否存在 285 | func (c *DingCache) HasBlock(blockIndex int64) (bool, error) { 286 | return c.testHeaderBlock(blockIndex) 287 | } 288 | 289 | // ReadBlock 读取块数据，该操作会预先缓存prefetchBlocks个块数据，避免每次读取时对文件做操作。 290 | 291 | func (c *DingCache) ReadBlock(blockIndex int64) ([]byte, error) { 292 | if !c.isOpen { 293 | return nil, errors.New("this file has been closed") 294 | } 295 | if blockIndex >= c.getBlockNumber() { 296 | return nil, errors.New("Invalid block index.") 297 | } 298 | key := c.getBlockKey(blockIndex) 299 | if config.SysConfig.Cache.Enabled { 300 | if blockData, ok := cache.FileBlockCache.Get(key); ok { 301 | // 若当前为最后一个块，则移除最近的16个块 302 | if blockIndex == c.getBlockNumber()-1 { 303 | var cacheFlag bool 304 | for i := config.SysConfig.GetPrefetchBlocks(); i >= 0; i-- { 305 | no := blockIndex - i 306 | if no >= 0 { 307 | oldKey := c.getBlockKey(no) 308 | cache.FileBlockCache.Delete(oldKey) 309 | cacheFlag = true 310 | } 311 | } 312 | if cacheFlag { 313 | cache.FileBlockCache.Wait() 314 | } 315 | } 316 | return blockData, nil 317 | } 318 | } 319 | hasBlock, err := c.HasBlock(blockIndex) 320 | if err != nil { 321 | return nil, err 322 | } 323 | if !hasBlock { 324 | return nil, nil 325 | } 326 | offset := c.getHeaderSize() + blockIndex*c.getBlockSize() 327 | f, err := os.OpenFile(c.path, os.O_RDONLY, 0644) 328 | if err != nil { 329 | return nil, err 330 | } 331 | defer f.Close() 332 | if _, err := f.Seek(offset, 0); err != nil { 333 | return nil, err 334 | } 335 | rawBlock := make([]byte, c.getBlockSize()) // 读取当前块（blockIndex）的数据 336 | if _, err := f.Read(rawBlock); err != nil { 337 | return nil, err 338 | } 339 | if config.SysConfig.Cache.Enabled { 340 | c.readBlockAndCache(f, blockIndex) 341 | } 342 | block := c.padBlock(rawBlock) 343 | return block, nil 344 | } 345 | 346 | func (c *DingCache) readBlockAndCache(f *os.File, blockIndex int64) { 347 | var cacheFlag bool 348 | memoryUsedPercent := config.SystemInfo.MemoryUsedPercent 349 | if memoryUsedPercent != 0 && memoryUsedPercent >= config.SysConfig.GetPrefetchMemoryUsedThreshold() { 350 | return 351 | } 352 | // 读取并缓存当前块后的16个块 353 | for blockOffset := int64(1); blockOffset <= config.SysConfig.GetPrefetchBlocks(); blockOffset++ { 354 | newOffsetBlock := blockIndex + blockOffset 355 | if newOffsetBlock >= c.getBlockNumber() { 356 | break 357 | } 358 | hasNextBlock, err := c.HasBlock(newOffsetBlock) 359 | if err != nil { 360 | zap.S().Errorf("hasBlock err. newOffsetBlock:%d, %v", newOffsetBlock, err) 361 | break 362 | } 363 | if hasNextBlock { 364 | key := c.getBlockKey(newOffsetBlock) 365 | prefetchRawBlock := make([]byte, c.getBlockSize()) 366 | if _, err := f.Read(prefetchRawBlock); err != nil { 367 | zap.S().Errorf("read err. newOffsetBlock:%d, %v", newOffsetBlock, err) 368 | break 369 | } 370 | blockByte := c.padBlock(prefetchRawBlock) 371 | cache.FileBlockCache.Set(key, blockByte) 372 | // 删除上一个缓存周期的内容，释放内存 373 | oldOffsetBlock := newOffsetBlock - config.SysConfig.GetPrefetchBlocks() - 1 374 | if oldOffsetBlock > 0 { 375 | oldKey := c.getBlockKey(newOffsetBlock) 376 | cache.FileBlockCache.Delete(oldKey) 377 | } 378 | cacheFlag = true 379 | } else { 380 | break 381 | } 382 | } 383 | if cacheFlag { 384 | cache.FileBlockCache.Wait() 385 | } 386 | } 387 | 388 | func (c *DingCache) WriteBlock(blockIndex int64, blockBytes []byte) error { 389 | if !c.isOpen { 390 | return errors.New("this file has been closed") 391 | } 392 | if blockIndex >= c.getBlockNumber() { 393 | return errors.New("invalid block index") 394 | } 395 | if int64(len(blockBytes)) != c.getBlockSize() { 396 | return errors.New("block size does not match the cache's block size") 397 | } 398 | offset := c.getHeaderSize() + blockIndex*c.getBlockSize() 399 | f, err := os.OpenFile(c.path, os.O_RDWR, 0644) 400 | if err != nil { 401 | return err 402 | } 403 | defer f.Close() 404 | if _, err := f.Seek(offset, 0); err != nil { 405 | return err 406 | } 407 | if (blockIndex+1)*c.getBlockSize() > c.GetFileSize() { 408 | realBlockBytes := blockBytes[:c.GetFileSize()-blockIndex*c.getBlockSize()] 409 | if _, err = f.Write(realBlockBytes); err != nil { 410 | return err 411 | } 412 | } else { 413 | if _, err = f.Write(blockBytes); err != nil { 414 | return err 415 | } 416 | } 417 | c.fileLock.Lock() 418 | defer c.fileLock.Unlock() 419 | if err = c.setHeaderBlock(blockIndex); err != nil { 420 | return err 421 | } 422 | if err = c.flushHeader(); err != nil { 423 | return err 424 | } 425 | // key := c.getBlockKey(blockIndex) 不需要删除，本来就没有 426 | // cache.FileBlockCache.Del(key) 427 | return nil 428 | } 429 | 430 | // resizeFileSize 调整文件大小 431 | func (c *DingCache) resizeFileSize(fileSize int64) error { 432 | if !c.isOpen { 433 | return errors.New("this file has been closed") 434 | } 435 | if fileSize == c.GetFileSize() { 436 | return nil 437 | } 438 | 439 | if fileSize < c.GetFileSize() { 440 | return errors.New("invalid resize file size. New file size must be greater than the current file size") 441 | } 442 | f, err := os.OpenFile(c.path, os.O_RDWR, 0644) 443 | if err != nil { 444 | return err 445 | } 446 | defer f.Close() 447 | newBinSize := c.getHeaderSize() + fileSize 448 | if _, err = f.Seek(newBinSize-1, 0); err != nil { 449 | return err 450 | } 451 | if _, err = f.Write([]byte{0}); err != nil { 452 | return err 453 | } 454 | if err = f.Truncate(newBinSize); err != nil { 455 | return err 456 | } 457 | return nil 458 | } 459 | 460 | // Resize 调整缓存大小 461 | func (c *DingCache) Resize(fileSize int64) error { 462 | if !c.isOpen { 463 | return errors.New("this file has been closed") 464 | } 465 | bs := c.getBlockSize() 466 | newBlockNum := (fileSize + bs - 1) / bs 467 | c.fileLock.Lock() 468 | defer c.fileLock.Unlock() 469 | if err := c.resizeFileSize(fileSize); err != nil { 470 | return err 471 | } 472 | // 设置块数量、文件大小参数 473 | if err := c.resizeHeader(newBlockNum, fileSize); err != nil { 474 | return err 475 | } 476 | return c.flushHeader() 477 | } 478 | 479 | func (c *DingCache) getBlockKey(blockIndex int64) string { 480 | simpleKey := fmt.Sprintf("%s/%d", c.path, blockIndex) 481 | return util.Md5(simpleKey) 482 | } 483 | -------------------------------------------------------------------------------- /internal/downloader/file_manager.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | 7 | "dingospeed/pkg/common" 8 | "dingospeed/pkg/config" 9 | 10 | "go.uber.org/zap" 11 | ) 12 | 13 | var ( 14 | instance *DingCacheManager 15 | once sync.Once 16 | ) 17 | 18 | func GetInstance() *DingCacheManager { 19 | once.Do(func() { 20 | instance = &DingCacheManager{ 21 | dingCacheMap: common.NewSafeMap[string, *DingCache](), 22 | dingCacheRef: common.NewSafeMap[string, *atomic.Int64](), 23 | } 24 | }) 25 | return instance 26 | } 27 | 28 | type DingCacheManager struct { 29 | dingCacheMap *common.SafeMap[string, *DingCache] 30 | dingCacheRef *common.SafeMap[string, *atomic.Int64] 31 | mu sync.RWMutex 32 | } 33 | 34 | func (f *DingCacheManager) GetDingFile(savePath string, fileSize int64) (*DingCache, error) { 35 | f.mu.Lock() 36 | defer f.mu.Unlock() 37 | var ( 38 | dingFile *DingCache 39 | ok bool 40 | err error 41 | ) 42 | if dingFile, ok = f.dingCacheMap.Get(savePath); ok { 43 | if refCount, ok := f.dingCacheRef.Get(savePath); ok { 44 | refCount.Add(1) 45 | } else { 46 | zap.S().Errorf("dingCacheRef key is not exist.key:%s", savePath) 47 | } 48 | return dingFile, nil 49 | } else { 50 | if dingFile, err = NewDingCache(savePath, config.SysConfig.Download.BlockSize); err != nil { 51 | zap.S().Errorf("NewDingCache err.%v", err) 52 | return nil, err 53 | } 54 | if dingFile.GetFileSize() == 0 { 55 | if err = dingFile.Resize(fileSize); err != nil { 56 | zap.S().Errorf("Resize err.%v", err) 57 | return nil, err 58 | } 59 | } 60 | 61 | f.dingCacheMap.Set(savePath, dingFile) 62 | var counter atomic.Int64 63 | counter.Store(1) 64 | f.dingCacheRef.Set(savePath, &counter) 65 | } 66 | return dingFile, nil 67 | } 68 | 69 | func (f *DingCacheManager) ReleasedDingFile(savePath string) { 70 | f.mu.Lock() 71 | defer f.mu.Unlock() 72 | refCount, ok := f.dingCacheRef.Get(savePath) 73 | if !ok { 74 | return 75 | } 76 | refCount.Add(-1) 77 | zap.S().Debugf("ReleasedDingFile:%s, refcount:%d", savePath, refCount.Load()) 78 | if refCount.Load() <= 0 { 79 | if dingFile, ok := f.dingCacheMap.Get(savePath); ok { 80 | dingFile.Close() 81 | } 82 | f.dingCacheMap.Delete(savePath) 83 | f.dingCacheRef.Delete(savePath) 84 | } else { 85 | f.dingCacheRef.Set(savePath, refCount) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /internal/downloader/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "go.uber.org/zap" 22 | ) 23 | 24 | func TestFileWrite(t *testing.T) { 25 | var dingFile *DingCache 26 | var err error 27 | savePath := "cachefile" 28 | fileSize := int64(8388608) 29 | blockSize := int64(8388608) 30 | if dingFile, err = NewDingCache(savePath, blockSize); err != nil { 31 | zap.S().Errorf("NewDingCache err.%v", err) 32 | return 33 | } 34 | if err = dingFile.Resize(fileSize); err != nil { 35 | zap.S().Errorf("Resize err.%v", err) 36 | return 37 | } 38 | } 39 | 40 | func TestFileWrite2(t *testing.T) { 41 | 42 | h := NewDingCacheHeader(1, 1, 1) 43 | fmt.Println(string(h.MagicNumber[:])) 44 | fmt.Println(string(h.MagicNumber[:])) 45 | 46 | fmt.Println(string(h.MagicNumber[:])) 47 | 48 | } 49 | -------------------------------------------------------------------------------- /internal/downloader/remote_task.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "io" 21 | "net/http" 22 | "strconv" 23 | "sync" 24 | 25 | "dingospeed/pkg/config" 26 | "dingospeed/pkg/consts" 27 | "dingospeed/pkg/prom" 28 | "dingospeed/pkg/util" 29 | 30 | "go.uber.org/zap" 31 | ) 32 | 33 | type RemoteFileTask struct { 34 | DownloadTask 35 | authorization string 36 | hfUrl string 37 | Queue chan []byte `json:"-"` 38 | } 39 | 40 | func NewRemoteFileTask(taskNo int, rangeStartPos int64, rangeEndPos int64) *RemoteFileTask { 41 | r := &RemoteFileTask{} 42 | r.TaskNo = taskNo 43 | r.RangeStartPos = rangeStartPos 44 | r.RangeEndPos = rangeEndPos 45 | return r 46 | } 47 | 48 | // 分段下载 49 | func (r RemoteFileTask) DoTask() { 50 | var ( 51 | curBlock int64 52 | wg sync.WaitGroup 53 | ) 54 | contentChan := make(chan []byte, consts.RespChanSize) 55 | rangeStartPos, rangeEndPos := r.RangeStartPos, r.RangeEndPos 56 | zap.S().Infof("remote file download:%s/%s, taskNo:%d, size:%d, startPos:%d, endPos:%d", r.orgRepo, r.FileName, r.TaskNo, r.TaskSize, rangeStartPos, rangeEndPos) 57 | wg.Add(2) 58 | go r.getFileRangeFromRemote(&wg, rangeStartPos, rangeEndPos, contentChan) 59 | curPos := rangeStartPos 60 | streamCache := bytes.Buffer{} 61 | lastBlock, lastBlockStartPos, lastBlockEndPos := getBlockInfo(curPos, r.DingFile.getBlockSize(), r.DingFile.GetFileSize()) // 块编号，开始位置，结束位置 62 | blockNumber := r.DingFile.getBlockNumber() 63 | go func() { 64 | defer func() { 65 | close(r.Queue) 66 | wg.Done() 67 | }() 68 | for { 69 | select { 70 | case chunk, ok := <-contentChan: 71 | { 72 | if !ok { 73 | return 74 | } 75 | // 先写到缓存 76 | select { 77 | case r.Queue <- chunk: 78 | case <-r.Context.Done(): 79 | return 80 | } 81 | 82 | chunkLen := int64(len(chunk)) 83 | curPos += chunkLen 84 | 85 | if config.SysConfig.EnableMetric() { 86 | // 原子性地更新总下载字节数 87 | source := util.Itoa(r.Context.Value(consts.PromSource)) 88 | prom.PromRequestByteCounter(prom.RequestRemoteByte, source, chunkLen) 89 | } 90 | 91 | if len(chunk) != 0 { 92 | streamCache.Write(chunk) 93 | } 94 | curBlock = curPos / r.DingFile.getBlockSize() 95 | // 若是一个新的数据块，则将上一个数据块持久化。 96 | if curBlock != lastBlock { 97 | splitPos := lastBlockEndPos - max(lastBlockStartPos, rangeStartPos) 98 | cacheLen := int64(streamCache.Len()) 99 | if splitPos > cacheLen { 100 | // 正常不会出现splitPos>len(streamCacheBytes),若出现只能降级处理。 101 | zap.S().Errorf("splitPos err.%d-%d", splitPos, cacheLen) 102 | splitPos = cacheLen 103 | } 104 | streamCacheBytes := streamCache.Bytes() 105 | rawBlock := streamCacheBytes[:splitPos] // 当前块的数据 106 | if int64(len(rawBlock)) == r.DingFile.getBlockSize() { 107 | hasBlockBool, err := r.DingFile.HasBlock(lastBlock) 108 | if err != nil { 109 | zap.S().Errorf("HasBlock err.%v", err) 110 | } 111 | if err == nil && !hasBlockBool { 112 | if err = r.DingFile.WriteBlock(lastBlock, rawBlock); err != nil { 113 | zap.S().Errorf("writeBlock err.%v", err) 114 | } 115 | zap.S().Debugf("%s/%s, taskNo:%d, block：%d(%d)write done, range：%d-%d.", r.orgRepo, r.FileName, r.TaskNo, lastBlock, blockNumber, lastBlockStartPos, lastBlockEndPos) 116 | } 117 | } 118 | nextBlock := streamCacheBytes[splitPos:] // 下一个块的数据 119 | streamCache.Truncate(0) 120 | streamCache.Write(nextBlock) 121 | lastBlock, lastBlockStartPos, lastBlockEndPos = getBlockInfo(curPos, r.DingFile.getBlockSize(), r.DingFile.GetFileSize()) 122 | } 123 | } 124 | case <-r.Context.Done(): 125 | zap.S().Warnf("file:%s/%s, task %d, ctx done, DoTask exit.", r.orgRepo, r.FileName, r.TaskNo) 126 | return 127 | } 128 | } 129 | }() 130 | wg.Wait() 131 | rawBlock := streamCache.Bytes() 132 | if curBlock == r.DingFile.getBlockNumber()-1 { 133 | // 对不足一个block的数据做补全 134 | if int64(len(rawBlock)) == r.DingFile.GetFileSize()%r.DingFile.getBlockSize() { 135 | padding := bytes.Repeat([]byte{0}, int(r.DingFile.getBlockSize())-len(rawBlock)) 136 | rawBlock = append(rawBlock, padding...) 137 | } 138 | lastBlock = curBlock 139 | } 140 | if int64(len(rawBlock)) == r.DingFile.getBlockSize() { 141 | hasBlockBool, err := r.DingFile.HasBlock(lastBlock) 142 | if err != nil { 143 | zap.S().Errorf("HasBlock err.%v", err) 144 | return 145 | } 146 | if !hasBlockBool { 147 | if err = r.DingFile.WriteBlock(lastBlock, rawBlock); err != nil { 148 | zap.S().Errorf("last writeBlock err.%v", err) 149 | } 150 | zap.S().Debugf("file:%s/%s, taskNo:%d, last block：%d(%d)write done, range：%d-%d.", r.orgRepo, r.FileName, r.TaskNo, lastBlock, blockNumber, lastBlockStartPos, lastBlockEndPos) 151 | if err = util.CreateSymlinkIfNotExists(r.blobsFile, r.filesPath); err != nil { 152 | zap.S().Errorf("filesPath:%s is not link", r.filesPath) 153 | } 154 | } 155 | } 156 | if curPos != rangeEndPos { 157 | zap.S().Warnf("file:%s, taskNo:%d, remote range (%d) is different from sent size (%d).", r.FileName, r.TaskNo, rangeEndPos-rangeStartPos, curPos-rangeStartPos) 158 | } 159 | } 160 | 161 | func (r RemoteFileTask) OutResult() { 162 | for { 163 | select { 164 | case data, ok := <-r.Queue: 165 | if !ok { 166 | zap.S().Debugf("OutResult r.Queue close %s/%s", r.orgRepo, r.FileName) 167 | return 168 | } 169 | select { 170 | case r.ResponseChan <- data: 171 | case <-r.Context.Done(): 172 | zap.S().Debugf("OutResult remote Context.Done() %s/%s", r.orgRepo, r.FileName) 173 | return 174 | } 175 | case <-r.Context.Done(): 176 | zap.S().Debugf("OutResult remote ctx err, fileName:%s/%s,err:%v", r.orgRepo, r.FileName, r.Context.Err()) 177 | return 178 | } 179 | } 180 | } 181 | 182 | func (r RemoteFileTask) GetResponseChan() chan []byte { 183 | return r.ResponseChan 184 | } 185 | 186 | func (r RemoteFileTask) getFileRangeFromRemote(wg *sync.WaitGroup, startPos, endPos int64, contentChan chan<- []byte) { 187 | headers := make(map[string]string) 188 | if r.authorization != "" { 189 | headers["authorization"] = r.authorization 190 | } 191 | headers["range"] = fmt.Sprintf("bytes=%d-%d", startPos, endPos-1) 192 | defer func() { 193 | close(contentChan) 194 | wg.Done() 195 | }() 196 | var rawData []byte 197 | chunkByteLen := 0 198 | var contentEncoding, contentLengthStr = "", "" 199 | 200 | if err := util.GetStream(r.hfUrl, headers, config.SysConfig.GetReqTimeOut(), func(resp *http.Response) { 201 | contentEncoding = resp.Header.Get("content-encoding") 202 | contentLengthStr = resp.Header.Get("content-length") 203 | for { 204 | select { 205 | case <-r.Context.Done(): 206 | zap.S().Warnf("getFileRangeFromRemote Context.Done err :%s", r.FileName) 207 | return 208 | default: 209 | chunk := make([]byte, config.SysConfig.Download.RespChunkSize) 210 | n, err := resp.Body.Read(chunk) 211 | if n > 0 { 212 | if contentEncoding != "" { // 数据有编码，先收集，后面解码 213 | rawData = append(rawData, chunk[:n]...) 214 | } else { 215 | select { 216 | case contentChan <- chunk[:n]: 217 | case <-r.Context.Done(): 218 | return 219 | } 220 | } 221 | chunkByteLen += n // 原始数量 222 | } 223 | if err != nil { 224 | if err == io.EOF { 225 | return 226 | } 227 | zap.S().Errorf("file:%s, task %d, req remote err.%v", r.FileName, r.TaskNo, err) 228 | return 229 | } 230 | } 231 | } 232 | }); err != nil { 233 | zap.S().Errorf("GetStream err.%v", err) 234 | return 235 | } 236 | if contentEncoding != "" { 237 | // 这里需要实现解压缩逻辑 238 | finalData, err := util.DecompressData(rawData, contentEncoding) 239 | if err != nil { 240 | zap.S().Errorf("DecompressData err.%v", err) 241 | return 242 | } 243 | contentChan <- finalData // 返回解码后的数据流 244 | chunkByteLen = len(finalData) // 将解码后的长度复制为原理的chunkBytes 245 | } 246 | if contentLengthStr != "" { 247 | contentLength, err := strconv.Atoi(contentLengthStr) // 原始数据长度 248 | if err != nil { 249 | zap.S().Errorf("contentLengthStr conv err.%s", contentLengthStr) 250 | return 251 | } 252 | if contentEncoding != "" { 253 | contentLength = chunkByteLen 254 | } 255 | if endPos-startPos != int64(contentLength) { 256 | zap.S().Errorf("file:%s, taskNo:%d,The content of the response is incomplete. Expected-%d. Accepted-%d", r.FileName, r.TaskNo, endPos-startPos, contentLength) 257 | return 258 | } 259 | } 260 | if endPos-startPos != int64(chunkByteLen) { 261 | zap.S().Warnf("file:%s, taskNo:%d,The block is incomplete. Expected-%d. Accepted-%d", r.FileName, r.TaskNo, endPos-startPos, chunkByteLen) 262 | return 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /internal/downloader/resp_notice.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package downloader 16 | 17 | import "dingospeed/pkg/common" 18 | 19 | var RespNoticeMap = common.NewSafeMap[string, *Broadcaster]() 20 | -------------------------------------------------------------------------------- /internal/handler/file_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package handler 16 | 17 | import ( 18 | "fmt" 19 | "net/url" 20 | 21 | "dingospeed/internal/service" 22 | "dingospeed/pkg/config" 23 | "dingospeed/pkg/consts" 24 | "dingospeed/pkg/prom" 25 | "dingospeed/pkg/util" 26 | 27 | "github.com/labstack/echo/v4" 28 | "github.com/prometheus/client_golang/prometheus" 29 | "go.uber.org/zap" 30 | ) 31 | 32 | type FileHandler struct { 33 | fileService *service.FileService 34 | sysService *service.SysService 35 | } 36 | 37 | func NewFileHandler(fileService *service.FileService, sysService *service.SysService) *FileHandler { 38 | return &FileHandler{ 39 | fileService: fileService, 40 | sysService: sysService, 41 | } 42 | } 43 | 44 | func (handler *FileHandler) HeadFileHandler1(c echo.Context) error { 45 | repoType, org, repo, commit, filePath, err := paramProcess(c, 1) 46 | if err != nil { 47 | zap.S().Error("解码出错:%v", err) 48 | return util.ErrorRequestParam(c) 49 | } 50 | return handler.fileService.FileHeadCommon(c, repoType, org, repo, commit, filePath) 51 | } 52 | 53 | func (handler *FileHandler) HeadFileHandler2(c echo.Context) error { 54 | repoType, org, repo, commit, filePath, err := paramProcess(c, 2) 55 | if err != nil { 56 | zap.S().Error("解码出错:%v", err) 57 | return util.ErrorRequestParam(c) 58 | } 59 | return handler.fileService.FileHeadCommon(c, repoType, org, repo, commit, filePath) 60 | } 61 | 62 | func (handler *FileHandler) HeadFileHandler3(c echo.Context) error { 63 | repoType, org, repo, commit, filePath, err := paramProcess(c, 3) 64 | if err != nil { 65 | zap.S().Error("解码出错:%v", err) 66 | return util.ErrorRequestParam(c) 67 | } 68 | return handler.fileService.FileHeadCommon(c, repoType, org, repo, commit, filePath) 69 | } 70 | 71 | func paramProcess(c echo.Context, processMode int) (string, string, string, string, string, error) { 72 | var ( 73 | repoType string 74 | org string 75 | repo string 76 | commit string 77 | filePath string 78 | ) 79 | if processMode == 1 { 80 | repoType = c.Param("repoType") 81 | org = c.Param("org") 82 | repo = c.Param("repo") 83 | commit = c.Param("commit") 84 | filePath = c.Param("filePath") 85 | } else if processMode == 2 { 86 | orgOrRepoType := c.Param("orgOrRepoType") 87 | repo = c.Param("repo") 88 | commit = c.Param("commit") 89 | filePath = c.Param("filePath") 90 | if _, ok := consts.RepoTypesMapping[orgOrRepoType]; ok { 91 | repoType = orgOrRepoType 92 | org = "" 93 | } else { 94 | repoType = "models" 95 | org = orgOrRepoType 96 | } 97 | } else if processMode == 3 { 98 | repo = c.Param("repo") 99 | commit = c.Param("commit") 100 | filePath = c.Param("filePath") 101 | repoType = "models" 102 | } else { 103 | panic("param process error.") 104 | } 105 | filePath, err := url.QueryUnescape(filePath) 106 | return repoType, org, repo, commit, filePath, err 107 | } 108 | 109 | func (handler *FileHandler) GetFileHandler1(c echo.Context) error { 110 | repoType, org, repo, commit, filePath, err := paramProcess(c, 1) 111 | if err != nil { 112 | zap.S().Error("解码出错:%v", err) 113 | return util.ErrorRequestParam(c) 114 | } 115 | return handler.fileGetCommon(c, repoType, org, repo, commit, filePath) 116 | } 117 | 118 | func (handler *FileHandler) GetFileHandler2(c echo.Context) error { 119 | repoType, org, repo, commit, filePath, err := paramProcess(c, 2) 120 | if err != nil { 121 | zap.S().Error("解码出错:%v", err) 122 | return util.ErrorRequestParam(c) 123 | } 124 | return handler.fileGetCommon(c, repoType, org, repo, commit, filePath) 125 | } 126 | 127 | func (handler *FileHandler) GetFileHandler3(c echo.Context) error { 128 | repoType, org, repo, commit, filePath, err := paramProcess(c, 3) 129 | if err != nil { 130 | zap.S().Error("解码出错:%v", err) 131 | return util.ErrorRequestParam(c) 132 | } 133 | return handler.fileGetCommon(c, repoType, org, repo, commit, filePath) 134 | } 135 | 136 | func (handler *FileHandler) fileGetCommon(c echo.Context, repoType, org, repo, commit, filePath string) error { 137 | if config.SysConfig.EnableMetric() { 138 | labels := prometheus.Labels{} 139 | labels[repoType] = fmt.Sprintf("%s/%s", org, repo) 140 | source := util.Itoa(c.Get(consts.PromSource)) 141 | if _, ok := consts.RepoTypesMapping[repoType]; ok { 142 | labels["source"] = source 143 | if repoType == "models" { 144 | prom.RequestModelCnt.With(labels).Inc() 145 | } else if repoType == "datasets" { 146 | prom.RequestDataSetCnt.With(labels).Inc() 147 | } 148 | } 149 | err := handler.fileService.FileGetCommon(c, repoType, org, repo, commit, filePath) 150 | if _, ok := consts.RepoTypesMapping[repoType]; ok { 151 | labels["source"] = source 152 | if repoType == "models" { 153 | prom.RequestModelCnt.With(labels).Dec() 154 | } else if repoType == "datasets" { 155 | prom.RequestDataSetCnt.With(labels).Dec() 156 | } 157 | } 158 | return err 159 | } else { 160 | return handler.fileService.FileGetCommon(c, repoType, org, repo, commit, filePath) 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /internal/handler/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package handler 16 | 17 | import ( 18 | "github.com/google/wire" 19 | ) 20 | 21 | var HandlerProvider = wire.NewSet(NewFileHandler, NewMetaHandler, NewSysHandler) 22 | -------------------------------------------------------------------------------- /internal/handler/meta_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package handler 16 | 17 | import ( 18 | "strings" 19 | 20 | "dingospeed/internal/service" 21 | 22 | "github.com/labstack/echo/v4" 23 | ) 24 | 25 | type MetaHandler struct { 26 | metaService *service.MetaService 27 | } 28 | 29 | func NewMetaHandler(fileService *service.MetaService) *MetaHandler { 30 | return &MetaHandler{ 31 | metaService: fileService, 32 | } 33 | } 34 | 35 | func (handler *MetaHandler) MetaProxyCommonHandler(c echo.Context) error { 36 | repoType := c.Param("repoType") 37 | org := c.Param("org") 38 | repo := c.Param("repo") 39 | commit := c.Param("commit") 40 | method := strings.ToLower(c.Request().Method) 41 | return handler.metaService.MetaProxyCommon(c, repoType, org, repo, commit, method) 42 | } 43 | 44 | func (handler *MetaHandler) WhoamiV2Handler(c echo.Context) error { 45 | return handler.metaService.WhoamiV2(c) 46 | } 47 | 48 | func (handler *MetaHandler) ReposHandler(c echo.Context) error { 49 | return handler.metaService.Repos(c) 50 | } 51 | -------------------------------------------------------------------------------- /internal/handler/sys_handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "dingospeed/internal/model" 5 | "dingospeed/internal/service" 6 | "dingospeed/pkg/app" 7 | "dingospeed/pkg/config" 8 | "dingospeed/pkg/util" 9 | 10 | "github.com/labstack/echo/v4" 11 | ) 12 | 13 | type SysHandler struct { 14 | sysService *service.SysService 15 | } 16 | 17 | func NewSysHandler(sysService *service.SysService) *SysHandler { 18 | return &SysHandler{ 19 | sysService: sysService, 20 | } 21 | } 22 | 23 | func (s *SysHandler) Info(c echo.Context) error { 24 | info := &model.SystemInfo{} 25 | if appInfo, ok := app.FromContext(c.Request().Context()); ok { 26 | info.Id = appInfo.ID() 27 | info.Name = appInfo.Name() 28 | info.Version = appInfo.Version() 29 | info.StartTime = appInfo.StartTime() 30 | } 31 | info.HfNetLoc = config.SysConfig.GetHfNetLoc() 32 | return util.ResponseData(c, info) 33 | } 34 | -------------------------------------------------------------------------------- /internal/model/sysinfo.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type SystemInfo struct { 4 | Id string `json:"id"` 5 | Name string `json:"name"` 6 | Version string `json:"version"` 7 | StartTime string `json:"startTime"` 8 | HfNetLoc string `json:"hfNetLoc"` 9 | CollectTime int64 `json:"-"` 10 | MemoryUsedPercent float64 `json:"-"` 11 | } 12 | 13 | func (s *SystemInfo) SetMemoryUsed(collectTime int64, usedPercent float64) { 14 | s.CollectTime = collectTime 15 | s.MemoryUsedPercent = usedPercent 16 | } 17 | -------------------------------------------------------------------------------- /internal/router/http_router.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package router 16 | 17 | import ( 18 | "dingospeed/internal/handler" 19 | "dingospeed/pkg/config" 20 | 21 | "github.com/labstack/echo/v4" 22 | "github.com/prometheus/client_golang/prometheus/promhttp" 23 | ) 24 | 25 | type HttpRouter struct { 26 | echo *echo.Echo 27 | fileHandler *handler.FileHandler 28 | metaHandler *handler.MetaHandler 29 | sysHandler *handler.SysHandler 30 | } 31 | 32 | func NewHttpRouter(echo *echo.Echo, fileHandler *handler.FileHandler, metaHandler *handler.MetaHandler, sysHandler *handler.SysHandler) *HttpRouter { 33 | r := &HttpRouter{ 34 | echo: echo, 35 | fileHandler: fileHandler, 36 | metaHandler: metaHandler, 37 | sysHandler: sysHandler, 38 | } 39 | r.initRouter() 40 | return r 41 | } 42 | 43 | func (r *HttpRouter) initRouter() { 44 | // 系统信息 45 | r.echo.GET("/info", r.sysHandler.Info) 46 | if config.SysConfig.EnableMetric() { 47 | r.echo.GET("/metrics", echo.WrapHandler(promhttp.Handler())) 48 | } 49 | 50 | // 单个文件 51 | r.echo.HEAD("/:repoType/:org/:repo/resolve/:commit/:filePath", r.fileHandler.HeadFileHandler1) 52 | r.echo.HEAD("/:orgOrRepoType/:repo/resolve/:commit/:filePath", r.fileHandler.HeadFileHandler2) 53 | r.echo.HEAD("/:repo/resolve/:commit/:filePath", r.fileHandler.HeadFileHandler3) 54 | 55 | r.echo.GET("/:repoType/:org/:repo/resolve/:commit/:filePath", r.fileHandler.GetFileHandler1) 56 | r.echo.GET("/:orgOrRepoType/:repo/resolve/:commit/:filePath", r.fileHandler.GetFileHandler2) 57 | r.echo.GET("/:repo/resolve/:commit/:filePath", r.fileHandler.GetFileHandler3) 58 | 59 | // 模型 60 | r.echo.HEAD("/api/:repoType/:org/:repo/revision/:commit", r.metaHandler.MetaProxyCommonHandler) 61 | r.echo.GET("/api/:repoType/:org/:repo/revision/:commit", r.metaHandler.MetaProxyCommonHandler) 62 | 63 | r.echo.GET("/api/whoami-v2", r.metaHandler.WhoamiV2Handler) 64 | r.echo.GET("/repos", r.metaHandler.ReposHandler) 65 | 66 | } 67 | -------------------------------------------------------------------------------- /internal/router/router.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package router 16 | 17 | import ( 18 | "github.com/google/wire" 19 | ) 20 | 21 | var RouterProvider = wire.NewSet(NewHttpRouter) 22 | -------------------------------------------------------------------------------- /internal/server/http.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "errors" 7 | "fmt" 8 | "html/template" 9 | "io" 10 | "net" 11 | "net/http" 12 | 13 | "go.uber.org/zap" 14 | 15 | "dingospeed/internal/router" 16 | "dingospeed/pkg/config" 17 | "dingospeed/pkg/middleware" 18 | 19 | "github.com/labstack/echo/v4" 20 | ) 21 | 22 | type HTTPServer struct { 23 | *http.Server 24 | lis net.Listener 25 | err error 26 | network string 27 | address string 28 | http *router.HttpRouter 29 | } 30 | 31 | //go:embed "templates/*.html" 32 | var templatesFS embed.FS 33 | 34 | type Template struct { 35 | templates *template.Template 36 | } 37 | 38 | func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { 39 | return t.templates.ExecuteTemplate(w, name, data) 40 | } 41 | 42 | func NewServer(config *config.Config, echo *echo.Echo, httpr *router.HttpRouter) *HTTPServer { 43 | s := &HTTPServer{ 44 | network: "tcp", 45 | address: fmt.Sprintf("%s:%d", config.Server.Host, config.Server.Port), 46 | http: httpr, 47 | } 48 | s.Server = &http.Server{ 49 | Handler: echo, 50 | ReadTimeout: 0, 51 | WriteTimeout: 0, // 对用户侧的响应设置永不超时 52 | MaxHeaderBytes: 1 << 20, 53 | } 54 | return s 55 | } 56 | 57 | func (s *HTTPServer) Start(ctx context.Context) error { 58 | lis, err := net.Listen(s.network, s.address) 59 | if err != nil { 60 | s.err = err 61 | return err 62 | } 63 | s.lis = lis 64 | s.BaseContext = func(net.Listener) context.Context { 65 | return ctx 66 | } 67 | zap.S().Infof("[HTTP] server listening on: %s", s.lis.Addr().String()) 68 | if err := s.Serve(s.lis); !errors.Is(err, http.ErrServerClosed) { 69 | return err 70 | } 71 | return nil 72 | } 73 | 74 | func (s *HTTPServer) Stop(ctx context.Context) error { 75 | zap.S().Infof("[HTTP] server shutdown.") 76 | return s.Shutdown(ctx) 77 | } 78 | 79 | func NewEngine() *echo.Echo { 80 | r := echo.New() 81 | middleware.InitMiddlewareConfig() 82 | r.Use(middleware.QueueLimitMiddleware) 83 | 84 | t := &Template{ 85 | templates: template.Must(template.ParseFS(templatesFS, "templates/*.html")), 86 | } 87 | r.Renderer = t 88 | return r 89 | } 90 | -------------------------------------------------------------------------------- /internal/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import "github.com/google/wire" 4 | 5 | var ServerProvider = wire.NewSet(NewServer, NewEngine) 6 | -------------------------------------------------------------------------------- /internal/server/templates/repos.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 18 | 19 | 20 | 21 | 22 | 24 | 41 | 42 | 43 | 44 |
45 |

Huggingface Mirror Repositories

46 | 47 |
48 |

Data Sets

49 |
50 | {{range .datasets_repos}} 51 |
52 |
53 |
{{.}}
54 |
55 |
56 | {{end}} 57 |
58 |
59 | 60 |
61 |

Models

62 |
63 | {{range .models_repos}} 64 |
65 |
66 |
{{.}}
67 |
68 |
69 | {{end}} 70 |
71 |
72 | 73 |
74 |

Spaces

75 |
76 | {{range .spaces_repos}} 77 |
78 |
79 |
{{.}}
80 |
81 |
82 | {{end}} 83 |
84 |
85 |
86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /internal/service/file_service.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "dingospeed/internal/dao" 19 | "dingospeed/pkg/config" 20 | "dingospeed/pkg/consts" 21 | "dingospeed/pkg/util" 22 | 23 | "github.com/labstack/echo/v4" 24 | "go.uber.org/zap" 25 | ) 26 | 27 | type FileService struct { 28 | fileDao *dao.FileDao 29 | } 30 | 31 | func NewFileService(fileDao *dao.FileDao) *FileService { 32 | return &FileService{ 33 | fileDao: fileDao, 34 | } 35 | } 36 | 37 | func (d *FileService) FileHeadCommon(c echo.Context, repoType, org, repo, commit, filePath string) error { 38 | commitSha, err := d.getFileCommitSha(c, repoType, org, repo, commit) 39 | if err != nil { 40 | return err 41 | } 42 | return d.fileDao.FileGetGenerator(c, repoType, org, repo, commitSha, filePath, consts.RequestTypeHead) 43 | } 44 | 45 | func (d *FileService) FileGetCommon(c echo.Context, repoType, org, repo, commit, filePath string) error { 46 | zap.S().Infof("exec file get:%s/%s/%s/%s/%s, remoteAdd:%s", repoType, org, repo, commit, filePath, c.Request().RemoteAddr) 47 | commitSha, err := d.getFileCommitSha(c, repoType, org, repo, commit) 48 | if err != nil { 49 | return err 50 | } 51 | return d.fileDao.FileGetGenerator(c, repoType, org, repo, commitSha, filePath, consts.RequestTypeGet) 52 | } 53 | 54 | func (d *FileService) getFileCommitSha(c echo.Context, repoType, org, repo, commit string) (string, error) { 55 | if _, ok := consts.RepoTypesMapping[repoType]; !ok { 56 | zap.S().Errorf("FileGetCommon repoType:%s is not exist RepoTypesMapping", repoType) 57 | return "", util.ErrorPageNotFound(c) 58 | } 59 | if org == "" && repo == "" { 60 | zap.S().Errorf("FileGetCommon or and repo is null") 61 | return "", util.ErrorRepoNotFound(c) 62 | } 63 | authorization := c.Request().Header.Get("authorization") 64 | if config.SysConfig.Online() { 65 | if code, err := d.fileDao.CheckCommitHf(repoType, org, repo, commit, authorization); err != nil { // 若请求找不到，直接返回仓库不存在。 66 | zap.S().Errorf("FileGetCommon CheckCommitHf is false, commit:%s", commit) 67 | return "", util.ErrorEntryUnknown(c, code, err.Error()) 68 | } 69 | } 70 | commitSha, err := d.fileDao.GetCommitHf(repoType, org, repo, commit, authorization) 71 | if err != nil { 72 | zap.S().Errorf(" getFileCommitSha GetCommitHf err.%v", err) 73 | return "", util.ErrorRepoNotFound(c) 74 | } 75 | return commitSha, nil 76 | } 77 | -------------------------------------------------------------------------------- /internal/service/meta_service.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import ( 18 | "dingospeed/internal/dao" 19 | "dingospeed/pkg/config" 20 | "dingospeed/pkg/consts" 21 | "dingospeed/pkg/util" 22 | 23 | "github.com/labstack/echo/v4" 24 | "go.uber.org/zap" 25 | ) 26 | 27 | type MetaService struct { 28 | fileDao *dao.FileDao 29 | metaDao *dao.MetaDao 30 | } 31 | 32 | func NewMetaService(fileDao *dao.FileDao, metaDao *dao.MetaDao) *MetaService { 33 | return &MetaService{ 34 | fileDao: fileDao, 35 | metaDao: metaDao, 36 | } 37 | } 38 | 39 | func (d *MetaService) MetaProxyCommon(c echo.Context, repoType, org, repo, commit, method string) error { 40 | zap.S().Debugf("MetaProxyCommon:%s/%s/%s/%s/%s", repoType, org, repo, commit, method) 41 | if _, ok := consts.RepoTypesMapping[repoType]; !ok { 42 | zap.S().Errorf("MetaProxyCommon repoType:%s is not exist RepoTypesMapping", repoType) 43 | return util.ErrorPageNotFound(c) 44 | } 45 | if org == "" && repo == "" { 46 | zap.S().Errorf("MetaProxyCommon or and repo is null") 47 | return util.ErrorRepoNotFound(c) 48 | } 49 | authorization := c.Request().Header.Get("authorization") 50 | if config.SysConfig.Online() { 51 | // check repo 52 | if code, err := d.fileDao.CheckCommitHf(repoType, org, repo, "", authorization); err != nil { 53 | zap.S().Errorf("MetaProxyCommon CheckCommitHf is false, commit is null") 54 | return util.ErrorEntryUnknown(c, code, err.Error()) 55 | } 56 | // check repo commit 57 | if code, err := d.fileDao.CheckCommitHf(repoType, org, repo, commit, authorization); err != nil { 58 | zap.S().Errorf("MetaProxyCommon CheckCommitHf is false, commit:%s", commit) 59 | return util.ErrorEntryUnknown(c, code, err.Error()) 60 | } 61 | } 62 | commitSha, err := d.fileDao.GetCommitHf(repoType, org, repo, commit, authorization) 63 | if err != nil { 64 | zap.S().Errorf("MetaProxyCommon GetCommitHf err.%v", err) 65 | return util.ErrorRepoNotFound(c) 66 | } 67 | if config.SysConfig.Online() && commitSha != commit { 68 | _ = d.metaDao.MetaGetGenerator(c, repoType, org, repo, commit, method, false) 69 | return d.metaDao.MetaGetGenerator(c, repoType, org, repo, commitSha, method, true) 70 | } else { 71 | return d.metaDao.MetaGetGenerator(c, repoType, org, repo, commitSha, method, true) 72 | } 73 | } 74 | 75 | func (d *MetaService) WhoamiV2(c echo.Context) error { 76 | err := d.fileDao.WhoamiV2Generator(c) 77 | return err 78 | } 79 | 80 | func (d *MetaService) Repos(c echo.Context) error { 81 | err := d.fileDao.ReposGenerator(c) 82 | return err 83 | } 84 | -------------------------------------------------------------------------------- /internal/service/service.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package service 16 | 17 | import "github.com/google/wire" 18 | 19 | var ServiceProvider = wire.NewSet(NewFileService, NewMetaService, NewSysService) 20 | -------------------------------------------------------------------------------- /internal/service/sys_service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | "time" 9 | 10 | "dingospeed/pkg/config" 11 | "dingospeed/pkg/util" 12 | 13 | "github.com/shirou/gopsutil/mem" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | var once sync.Once 18 | 19 | type SysService struct { 20 | } 21 | 22 | func NewSysService() *SysService { 23 | sysSvc := &SysService{} 24 | once.Do( 25 | func() { 26 | if config.SysConfig.Cache.Enabled { 27 | go sysSvc.MemoryUsed() 28 | } 29 | 30 | if config.SysConfig.DiskClean.Enabled { 31 | go sysSvc.cycleCheckDiskUsage() 32 | } 33 | }) 34 | return sysSvc 35 | } 36 | 37 | func (s SysService) MemoryUsed() { 38 | ticker := time.NewTicker(config.SysConfig.GetCollectTimePeriod()) 39 | defer ticker.Stop() 40 | for { 41 | select { 42 | case <-ticker.C: 43 | memoryInfo, err := mem.VirtualMemory() 44 | if err != nil { 45 | fmt.Printf("获取内存信息时出错: %v\n", err) 46 | continue 47 | } 48 | config.SystemInfo.SetMemoryUsed(time.Now().Unix(), memoryInfo.UsedPercent) 49 | } 50 | } 51 | } 52 | 53 | func (s SysService) cycleCheckDiskUsage() { 54 | ticker := time.NewTicker(config.SysConfig.GetDiskCollectTimePeriod()) 55 | defer ticker.Stop() 56 | for { 57 | select { 58 | case <-ticker.C: 59 | checkDiskUsage() 60 | } 61 | } 62 | } 63 | 64 | // 检查磁盘使用情况 65 | func checkDiskUsage() { 66 | if !config.SysConfig.Online() { 67 | return 68 | } 69 | if !config.SysConfig.DiskClean.Enabled { 70 | return 71 | } 72 | 73 | currentSize, err := util.GetFolderSize(config.SysConfig.Repos()) 74 | if err != nil { 75 | zap.S().Errorf("Error getting folder size: %v", err) 76 | return 77 | } 78 | 79 | limitSize := config.SysConfig.DiskClean.CacheSizeLimit 80 | limitSizeH := util.ConvertBytesToHumanReadable(limitSize) 81 | currentSizeH := util.ConvertBytesToHumanReadable(currentSize) 82 | 83 | if currentSize < limitSize { 84 | return 85 | } 86 | 87 | zap.S().Infof("Cache size exceeded! Limit: %s, Current: %s.\n", limitSizeH, currentSizeH) 88 | zap.S().Infof("Cleaning...") 89 | 90 | filesPath := filepath.Join(config.SysConfig.Repos(), "files") 91 | 92 | var allFiles []util.FileWithPath 93 | switch config.SysConfig.CacheCleanStrategy() { 94 | case "LRU": 95 | allFiles, err = util.SortFilesByAccessTime(filesPath) 96 | if err != nil { 97 | zap.S().Errorf("Error sorting files by access time in %s: %v\n", filesPath, err) 98 | return 99 | } 100 | case "FIFO": 101 | allFiles, err = util.SortFilesByModifyTime(filesPath) 102 | if err != nil { 103 | zap.S().Errorf("Error sorting files by modify time in %s: %v\n", filesPath, err) 104 | return 105 | } 106 | case "LARGE_FIRST": 107 | allFiles, err = util.SortFilesBySize(filesPath) 108 | if err != nil { 109 | zap.S().Errorf("Error sorting files by size in %s: %v\n", filesPath, err) 110 | return 111 | } 112 | default: 113 | zap.S().Errorf("Unknown cache clean strategy: %s\n", config.SysConfig.CacheCleanStrategy()) 114 | return 115 | } 116 | 117 | for _, file := range allFiles { 118 | if currentSize < limitSize { 119 | break 120 | } 121 | filePath := file.Path 122 | fileSize := file.Info.Size() 123 | err := os.Remove(filePath) 124 | if err != nil { 125 | zap.S().Errorf("Error removing file %s: %v\n", filePath, err) 126 | continue 127 | } 128 | currentSize -= fileSize 129 | zap.S().Infof("Remove file: %s. File Size: %s\n", filePath, util.ConvertBytesToHumanReadable(fileSize)) 130 | } 131 | 132 | currentSize, err = util.GetFolderSize(config.SysConfig.Repos()) 133 | if err != nil { 134 | zap.S().Errorf("Error getting folder size after cleaning: %v\n", err) 135 | return 136 | } 137 | currentSizeH = util.ConvertBytesToHumanReadable(currentSize) 138 | zap.S().Infof("Cleaning finished. Limit: %s, Current: %s.\n", limitSizeH, currentSizeH) 139 | } 140 | -------------------------------------------------------------------------------- /pkg/app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "sync" 10 | "syscall" 11 | "time" 12 | 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | type AppInfo interface { 17 | ID() string 18 | Name() string 19 | Version() string 20 | StartTime() string 21 | } 22 | 23 | type App struct { 24 | opts options 25 | ctx context.Context 26 | cancel func() 27 | } 28 | 29 | func New(opts ...Option) *App { 30 | o := options{ 31 | ctx: context.Background(), // 全局最基础的ctx 32 | sigs: []os.Signal{syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT}, 33 | stopTimeout: 10 * time.Second, 34 | startTime: time.Now().Format(time.DateTime), 35 | } 36 | for _, opt := range opts { 37 | opt(&o) 38 | } 39 | ctx, cancel := context.WithCancel(o.ctx) 40 | return &App{ 41 | opts: o, 42 | ctx: ctx, 43 | cancel: cancel, 44 | } 45 | } 46 | 47 | func (a *App) ID() string { return a.opts.id } 48 | 49 | func (a *App) Name() string { return a.opts.name } 50 | 51 | func (a *App) Version() string { return a.opts.version } 52 | 53 | func (a *App) StartTime() string { return a.opts.startTime } 54 | 55 | func (a *App) Stop() (err error) { 56 | if a.cancel != nil { 57 | a.cancel() 58 | } 59 | return err 60 | } 61 | 62 | func (a *App) Run() error { 63 | stx := NewContext(a.ctx, a) 64 | eg, ctx := errgroup.WithContext(stx) 65 | wg := sync.WaitGroup{} 66 | for _, srv := range a.opts.servers { 67 | srv := srv 68 | eg.Go(func() error { 69 | <-ctx.Done() // wait for stop signal 70 | stopCtx, cancel := context.WithTimeout(NewContext(a.opts.ctx, a), a.opts.stopTimeout) 71 | defer cancel() 72 | return srv.Stop(stopCtx) 73 | }) 74 | wg.Add(1) 75 | eg.Go(func() error { 76 | wg.Done() // here is to ensure server start has begun running before register, so defer is not needed 77 | return srv.Start(stx) 78 | }) 79 | } 80 | wg.Wait() 81 | 82 | c := make(chan os.Signal, 1) 83 | signal.Notify(c, a.opts.sigs...) 84 | eg.Go(func() error { 85 | select { 86 | case <-ctx.Done(): 87 | return nil 88 | case <-c: 89 | return a.Stop() 90 | } 91 | }) 92 | 93 | if err := eg.Wait(); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, http.ErrServerClosed) { 94 | return err 95 | } 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /pkg/app/context.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import "context" 4 | 5 | type appKey struct{} 6 | 7 | // NewContext returns a new Context that carries value. 8 | func NewContext(ctx context.Context, s AppInfo) context.Context { 9 | return context.WithValue(ctx, appKey{}, s) 10 | } 11 | 12 | // FromContext returns the Transport value stored in ctx, if any. 13 | func FromContext(ctx context.Context) (s AppInfo, ok bool) { 14 | s, ok = ctx.Value(appKey{}).(AppInfo) 15 | return 16 | } 17 | -------------------------------------------------------------------------------- /pkg/app/options.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "time" 7 | 8 | "dingospeed/pkg/server" 9 | ) 10 | 11 | type Option func(o *options) 12 | 13 | type options struct { 14 | id string 15 | name string 16 | version string 17 | startTime string 18 | ctx context.Context 19 | sigs []os.Signal 20 | stopTimeout time.Duration 21 | servers []server.Server 22 | } 23 | 24 | func ID(id string) Option { 25 | return func(o *options) { o.id = id } 26 | } 27 | 28 | func Name(name string) Option { 29 | return func(o *options) { o.name = name } 30 | } 31 | 32 | func Version(version string) Option { 33 | return func(o *options) { 34 | o.version = version 35 | } 36 | } 37 | 38 | func Server(srv ...server.Server) Option { 39 | return func(o *options) { o.servers = srv } 40 | } 41 | 42 | func Context(ctx context.Context) Option { 43 | return func(o *options) { o.ctx = ctx } 44 | } 45 | 46 | func StopTimeout(t time.Duration) Option { 47 | return func(o *options) { o.stopTimeout = t } 48 | } 49 | 50 | func Signal(sigs ...os.Signal) Option { 51 | return func(o *options) { o.sigs = sigs } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/common/common.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "strings" 19 | "time" 20 | ) 21 | 22 | // FileInfo 列出文件元信息 23 | type FileInfo struct { 24 | Filename string // 文件名 25 | Filesize int64 // 文件大小 26 | Filetype string // 文件类型（目前有普通文件和切片文件两种） 27 | } 28 | 29 | // ListFileInfos 文件列表结构 30 | type ListFileInfos struct { 31 | Files []FileInfo 32 | } 33 | 34 | type Segment struct { 35 | Index int 36 | Start int64 37 | End int64 38 | } 39 | 40 | // FileMetadata 文件片元数据 41 | type FileMetadata struct { 42 | Fid string // 操作文件ID，随机生成的UUID 43 | Filesize int64 // 文件大小（字节单位） 44 | Filename string // 文件名称 45 | SliceNum int // 切片数量 46 | Md5sum string // 文件md5值 47 | ModifyTime time.Time // 文件修改时间 48 | Segments []*Segment 49 | } 50 | 51 | type Response struct { 52 | StatusCode int 53 | Headers map[string]interface{} 54 | Body []byte 55 | BodyChan chan []byte 56 | } 57 | 58 | func (r Response) GetKey(key string) string { 59 | if v, ok := r.Headers[key]; ok { 60 | if strSlice, ok := v.([]string); ok { 61 | if len(strSlice) > 0 { 62 | return strSlice[0] 63 | } 64 | } 65 | } 66 | return "" 67 | } 68 | 69 | func (r Response) ExtractHeaders(headers map[string]interface{}) map[string]string { 70 | lowerCaseHeaders := make(map[string]string) 71 | for k, v := range headers { 72 | if strSlice, ok := v.([]string); ok { 73 | if len(strSlice) > 0 { 74 | lowerCaseHeaders[strings.ToLower(k)] = strSlice[0] 75 | } 76 | } else { 77 | lowerCaseHeaders[strings.ToLower(k)] = "" 78 | } 79 | } 80 | return lowerCaseHeaders 81 | } 82 | 83 | type PathsInfo struct { 84 | Type string `json:"type"` 85 | Oid string `json:"oid"` 86 | Size int64 `json:"size"` 87 | Lfs Lfs `json:"lfs"` 88 | Path string `json:"path"` 89 | } 90 | 91 | type Lfs struct { 92 | Oid string `json:"oid"` 93 | Size int64 `json:"size"` 94 | PointerSize int64 `json:"pointerSize"` 95 | } 96 | 97 | type CacheContent struct { 98 | StatusCode int `json:"status_code"` // json格式要个之前的版本做兼容 99 | Headers map[string]string `json:"headers"` 100 | Content string `json:"content"` 101 | OriginContent []byte `json:"-"` 102 | } 103 | 104 | type ErrorResp struct { 105 | Error string `json:"error"` 106 | } 107 | -------------------------------------------------------------------------------- /pkg/common/pool.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "sync" 21 | ) 22 | 23 | // Task 定义任务类型 24 | type Task interface { 25 | DoTask() 26 | OutResult() 27 | GetResponseChan() chan []byte 28 | } 29 | 30 | // Pool 协程池结构体 31 | type Pool struct { 32 | taskChan chan Task 33 | wg sync.WaitGroup 34 | size int 35 | } 36 | 37 | // NewPool 创建新协程池 38 | func NewPool(size int) *Pool { 39 | p := &Pool{ 40 | taskChan: make(chan Task), 41 | size: size, 42 | } 43 | 44 | for i := 0; i < size; i++ { 45 | p.wg.Add(1) 46 | go p.worker() 47 | } 48 | 49 | return p 50 | } 51 | 52 | // worker 工作协程 53 | func (p *Pool) worker() { 54 | defer p.wg.Done() 55 | for { 56 | select { 57 | case task, ok := <-p.taskChan: 58 | if !ok { 59 | return 60 | } 61 | task.DoTask() 62 | } 63 | } 64 | } 65 | 66 | // Submit 提交任务 67 | func (p *Pool) Submit(ctx context.Context, task Task) error { 68 | select { 69 | case p.taskChan <- task: 70 | return nil 71 | case <-ctx.Done(): 72 | return errors.New("submit task fail") 73 | } 74 | } 75 | 76 | // Close 关闭协程池（安全关闭） 77 | func (p *Pool) Close() { 78 | close(p.taskChan) 79 | p.wg.Wait() 80 | } 81 | -------------------------------------------------------------------------------- /pkg/common/safe_map.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import "sync" 18 | 19 | type SafeMap[K comparable, V any] struct { 20 | m map[K]V 21 | mu sync.RWMutex 22 | } 23 | 24 | func NewSafeMap[K comparable, V any]() *SafeMap[K, V] { 25 | return &SafeMap[K, V]{ 26 | m: make(map[K]V), 27 | } 28 | } 29 | 30 | func (sm *SafeMap[K, V]) Set(key K, value V) { 31 | sm.mu.Lock() 32 | defer sm.mu.Unlock() 33 | sm.m[key] = value 34 | } 35 | 36 | func (sm *SafeMap[K, V]) Get(key K) (V, bool) { 37 | sm.mu.RLock() 38 | defer sm.mu.RUnlock() 39 | value, exists := sm.m[key] 40 | return value, exists 41 | } 42 | 43 | func (sm *SafeMap[K, V]) Delete(key K) { 44 | sm.mu.Lock() 45 | defer sm.mu.Unlock() 46 | delete(sm.m, key) 47 | } 48 | 49 | func (sm *SafeMap[K, V]) Len() int { 50 | sm.mu.RLock() 51 | defer sm.mu.RUnlock() 52 | return len(sm.m) 53 | } 54 | 55 | func (f *SafeMap[K, V]) Wait() { 56 | } 57 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package config 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "dingospeed/internal/model" 24 | myerr "dingospeed/pkg/error" 25 | 26 | "github.com/go-playground/validator/v10" 27 | "github.com/labstack/gommon/log" 28 | "go.uber.org/zap" 29 | "gopkg.in/yaml.v3" 30 | ) 31 | 32 | var SysConfig *Config 33 | var SystemInfo *model.SystemInfo 34 | 35 | type Config struct { 36 | Server ServerConfig `json:"server" yaml:"server"` 37 | Download Download `json:"download" yaml:"download"` 38 | Cache Cache `json:"cache" yaml:"cache"` 39 | Log LogConfig `json:"log" yaml:"log"` 40 | Retry Retry `json:"retry" yaml:"retry"` 41 | TokenBucketLimit TokenBucketLimit `json:"tokenBucketLimit" yaml:"tokenBucketLimit"` 42 | DiskClean DiskClean `json:"diskClean" yaml:"diskClean"` 43 | } 44 | 45 | type ServerConfig struct { 46 | Mode string `json:"mode" yaml:"mode"` 47 | Host string `json:"host" yaml:"host"` 48 | Port int `json:"port" yaml:"port"` 49 | PProf bool `json:"pprof" yaml:"pprof"` 50 | PProfPort int `json:"pprofPort" yaml:"pprofPort"` 51 | Metrics bool `json:"metrics" yaml:"metrics"` 52 | Online bool `json:"online" yaml:"online"` 53 | Repos string `json:"repos" yaml:"repos"` 54 | HfNetLoc string `json:"hfNetLoc" yaml:"hfNetLoc"` 55 | HfScheme string `json:"hfScheme" yaml:"hfScheme" validate:"oneof=https http"` 56 | } 57 | 58 | type Download struct { 59 | RetryChannelNum int `json:"retryChannelNum" yaml:"retryChannelNum"` 60 | GoroutineMaxNumPerFile int `json:"goroutineMaxNumPerFile" yaml:"goroutineMaxNumPerFile" validate:"min=1,max=8"` 61 | BlockSize int64 `json:"blockSize" yaml:"blockSize" validate:"min=1048576,max=134217728"` 62 | ReqTimeout int64 `json:"reqTimeout" yaml:"reqTimeout"` 63 | RespChunkSize int64 `json:"respChunkSize" yaml:"respChunkSize" validate:"min=4096,max=8388608"` 64 | RespChanSize int64 `json:"respChanSize" yaml:"respChanSize"` 65 | RemoteFileRangeSize int64 `json:"remoteFileRangeSize" yaml:"remoteFileRangeSize" validate:"min=0,max=1073741824"` 66 | RemoteFileRangeWaitTime int64 `json:"remoteFileRangeWaitTime" yaml:"remoteFileRangeWaitTime" validate:"min=1,max=10"` 67 | RemoteFileBufferSize int64 `json:"remoteFileBufferSize" yaml:"remoteFileBufferSize" validate:"min=0,max=134217728"` 68 | } 69 | 70 | type Cache struct { 71 | Enabled bool `json:"enabled" yaml:"enabled"` 72 | Type int `json:"type" yaml:"type"` 73 | CollectTimePeriod int `json:"collectTimePeriod" yaml:"collectTimePeriod" validate:"min=1,max=600"` // 周期采集内存使用量，单位秒 74 | PrefetchMemoryUsedThreshold float64 `json:"prefetchMemoryUsedThreshold" yaml:"prefetchMemoryUsedThreshold" validate:"min=50,max=99"` 75 | PrefetchBlocks int64 `json:"prefetchBlocks" yaml:"prefetchBlocks" validate:"min=1,max=32"` // 读取块数据，预先缓存的块数据数量 76 | PrefetchBlockTTL int64 `json:"prefetchBlockTTL" yaml:"prefetchBlockTTL" validate:"min=1,max=120"` // 读取块数据，预先缓存的块数据数量 77 | } 78 | 79 | type Retry struct { 80 | Delay int `json:"delay" yaml:"delay" validate:"min=0,max=60"` 81 | Attempts uint `json:"attempts" yaml:"attempts" validate:"min=1,max=5"` 82 | } 83 | 84 | type LogConfig struct { 85 | MaxSize int `json:"maxSize" yaml:"maxSize"` 86 | MaxBackups int `json:"maxBackups" yaml:"maxBackups"` 87 | MaxAge int `json:"maxAge" yaml:"maxAge"` 88 | } 89 | 90 | type TokenBucketLimit struct { 91 | Capacity int `json:"capacity" yaml:"capacity"` 92 | Rate int `json:"rate" yaml:"rate"` 93 | HandlerCapacity int `json:"handlerCapacity" yaml:"handlerCapacity"` 94 | } 95 | 96 | type DiskClean struct { 97 | Enabled bool `json:"enabled" yaml:"enabled"` 98 | CacheSizeLimit int64 `json:"cacheSizeLimit" yaml:"cacheSizeLimit"` 99 | CacheCleanStrategy string `json:"cacheCleanStrategy" yaml:"cacheCleanStrategy"` 100 | CollectTimePeriod int `json:"collectTimePeriod" yaml:"collectTimePeriod" validate:"min=1,max=600"` // 周期采集内存使用量，单位秒 101 | } 102 | 103 | func (c *Config) GetHFURLBase() string { 104 | return fmt.Sprintf("%s://%s", c.GetHfScheme(), c.GetHfNetLoc()) 105 | } 106 | 107 | func (c *Config) Online() bool { 108 | return c.Server.Online 109 | } 110 | 111 | func (c *Config) Repos() string { 112 | return c.Server.Repos 113 | } 114 | 115 | func (c *Config) GetHost() string { 116 | return c.Server.Host 117 | } 118 | 119 | func (c *Config) GetHfNetLoc() string { 120 | return c.Server.HfNetLoc 121 | } 122 | 123 | func (c *Config) GetCapacity() int { 124 | return c.TokenBucketLimit.Capacity 125 | } 126 | 127 | func (c *Config) GetRate() int { 128 | return c.TokenBucketLimit.Rate 129 | } 130 | 131 | func (c *Config) GetHfScheme() string { 132 | return c.Server.HfScheme 133 | } 134 | 135 | func (c *Config) GetReqTimeOut() time.Duration { 136 | return time.Duration(c.Download.ReqTimeout) * time.Second 137 | } 138 | 139 | func (c *Config) GetCollectTimePeriod() time.Duration { 140 | return time.Duration(c.Cache.CollectTimePeriod) * time.Second 141 | } 142 | 143 | func (c *Config) GetPrefetchMemoryUsedThreshold() float64 { 144 | return c.Cache.PrefetchMemoryUsedThreshold 145 | } 146 | 147 | func (c *Config) GetRemoteFileRangeWaitTime() time.Duration { 148 | return time.Duration(c.Download.RemoteFileRangeWaitTime) * time.Millisecond 149 | } 150 | 151 | func (c *Config) GetPrefetchBlocks() int64 { 152 | return c.Cache.PrefetchBlocks 153 | } 154 | 155 | func (c *Config) GetPrefetchBlockTTL() time.Duration { 156 | return time.Duration(c.Cache.PrefetchBlockTTL) * time.Second 157 | } 158 | 159 | func (c *Config) GetDiskCollectTimePeriod() time.Duration { 160 | return time.Duration(c.DiskClean.CollectTimePeriod) * time.Hour 161 | } 162 | 163 | func (c *Config) EnableMetric() bool { 164 | return c.Server.Metrics 165 | } 166 | 167 | func (c *Config) CacheCleanStrategy() string { 168 | return c.DiskClean.CacheCleanStrategy 169 | } 170 | 171 | func (c *Config) SetDefaults() { 172 | if c.Server.Port == 0 { 173 | c.Server.Port = 8090 174 | } 175 | if c.Server.Host == "" { 176 | c.Server.Host = "localhost" 177 | } 178 | if c.Server.PProfPort == 0 { 179 | c.Server.PProfPort = 6060 180 | } 181 | if c.Download.GoroutineMaxNumPerFile == 0 { 182 | c.Download.GoroutineMaxNumPerFile = 8 183 | } 184 | if c.Download.BlockSize == 0 { 185 | c.Download.BlockSize = 8388608 186 | } 187 | if c.Download.RespChunkSize == 0 { 188 | c.Download.RespChunkSize = 8192 189 | } 190 | if c.Download.RemoteFileRangeWaitTime == 0 { 191 | c.Download.RemoteFileRangeWaitTime = 1 192 | } 193 | if c.Cache.PrefetchBlocks == 0 { 194 | c.Cache.PrefetchBlocks = 16 195 | } 196 | if c.Cache.PrefetchBlockTTL == 0 { 197 | c.Cache.PrefetchBlockTTL = 30 198 | } 199 | if c.Cache.CollectTimePeriod == 0 { 200 | c.Cache.CollectTimePeriod = 5 201 | } 202 | if c.Download.RespChanSize == 0 { 203 | c.Download.RespChanSize = 30 204 | } 205 | if c.DiskClean.CollectTimePeriod == 0 { 206 | c.DiskClean.CollectTimePeriod = 1 207 | } 208 | } 209 | 210 | func Scan(path string) (*Config, error) { 211 | b, err := os.ReadFile(path) 212 | if err != nil { 213 | return nil, err 214 | } 215 | 216 | var c Config 217 | err = yaml.Unmarshal(b, &c) 218 | if err != nil { 219 | return nil, err 220 | } 221 | c.SetDefaults() 222 | 223 | if c.Download.RemoteFileRangeSize%c.Download.BlockSize != 0 { 224 | return nil, myerr.New("RemoteFileRangeSize must be a multiple of BlockSize") 225 | } 226 | 227 | validate := validator.New() 228 | err = validate.Struct(&c) 229 | if err != nil { 230 | var invalidValidationError *validator.InvalidValidationError 231 | if errors.As(err, &invalidValidationError) { 232 | zap.S().Errorf("Invalid validation error: %v\n", err) 233 | } 234 | return nil, err 235 | } 236 | SysConfig = &c // 设置全局配置变量 237 | 238 | marshal, err := yaml.Marshal(c) 239 | if err != nil { 240 | return nil, err 241 | } 242 | log.Info(string(marshal)) 243 | SystemInfo = &model.SystemInfo{} 244 | return &c, nil 245 | } 246 | -------------------------------------------------------------------------------- /pkg/consts/const.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package consts 16 | 17 | var RepoTypesMapping = map[string]RepoType{ 18 | "models": RepoTypeModel, 19 | "spaces": RepoTypeSpace, 20 | "datasets": RepoTypeDataset, 21 | } 22 | 23 | // repo类型 24 | type RepoType string 25 | 26 | const ( 27 | RepoTypeModel RepoType = RepoType("models") 28 | RepoTypeSpace = RepoType("spaces") 29 | RepoTypeDataset = RepoType("datasets") 30 | ) 31 | 32 | func (a RepoType) Value() string { 33 | return string(a) 34 | } 35 | 36 | const HUGGINGFACE_HEADER_X_REPO_COMMIT = "X-Repo-Commit" 37 | 38 | const ( 39 | RequestTypeHead = "head" 40 | RequestTypeGet = "get" 41 | ) 42 | 43 | const RespChanSize = 100 44 | const PromSource = "source" 45 | -------------------------------------------------------------------------------- /pkg/error/error.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package myerr 16 | 17 | type Error struct { 18 | statusCode int 19 | msg string 20 | err error 21 | } 22 | 23 | func (e Error) Error() string { 24 | return e.msg 25 | } 26 | 27 | func (e Error) StatusCode() int { 28 | return e.statusCode 29 | } 30 | func (e Error) Cause(err error) { 31 | e.err = err 32 | } 33 | 34 | func (e Error) Unwrap() error { 35 | return e.err 36 | } 37 | 38 | func New(msg string) Error { 39 | return Error{msg: msg} 40 | } 41 | 42 | func NewAppendCode(code int, msg string) Error { 43 | return Error{msg: msg, statusCode: code} 44 | } 45 | 46 | func Wrap(msg string, err error) Error { 47 | return Error{msg: msg, err: err} 48 | } 49 | -------------------------------------------------------------------------------- /pkg/logger/log.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package log 16 | 17 | import ( 18 | "os" 19 | "path/filepath" 20 | "time" 21 | 22 | "dingospeed/pkg/config" 23 | 24 | "go.uber.org/zap" 25 | "go.uber.org/zap/zapcore" 26 | "gopkg.in/natefinch/lumberjack.v2" 27 | ) 28 | 29 | func InitLogger() { 30 | logMode := zapcore.InfoLevel 31 | if config.SysConfig.Server.Mode == "debug" { 32 | logMode = zapcore.DebugLevel 33 | } 34 | core := zapcore.NewCore(getEncoder(), zapcore.NewMultiWriteSyncer(getWriteSyncer(), zapcore.AddSync(os.Stdout)), logMode) 35 | logger := zap.New(core, zap.AddCaller()) 36 | zap.ReplaceGlobals(logger) 37 | } 38 | 39 | func getEncoder() zapcore.Encoder { 40 | encoder := zap.NewProductionEncoderConfig() 41 | encoder.TimeKey = "time" 42 | encoder.EncodeLevel = zapcore.CapitalLevelEncoder 43 | encoder.EncodeTime = func(t time.Time, encoder zapcore.PrimitiveArrayEncoder) { 44 | encoder.AppendString(t.Local().Format(time.DateTime)) 45 | } 46 | encoder.CallerKey = "caller" 47 | encoder.EncodeCaller = zapcore.ShortCallerEncoder 48 | return zapcore.NewJSONEncoder(encoder) 49 | } 50 | 51 | func getWriteSyncer() zapcore.WriteSyncer { 52 | stSeparator := string(filepath.Separator) 53 | stRootDir, _ := os.Getwd() 54 | stLogFilePath := stRootDir + stSeparator + "log" + stSeparator + time.Now().Format(time.DateOnly) + ".log" 55 | 56 | l := &lumberjack.Logger{ 57 | Filename: stLogFilePath, 58 | MaxSize: config.SysConfig.Log.MaxSize, // megabytes 59 | MaxBackups: config.SysConfig.Log.MaxBackups, 60 | MaxAge: config.SysConfig.Log.MaxAge, // days 61 | Compress: false, // disabled by default 62 | } 63 | return zapcore.AddSync(l) 64 | } 65 | -------------------------------------------------------------------------------- /pkg/middleware/queue_limit.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | 7 | "dingospeed/pkg/config" 8 | "dingospeed/pkg/consts" 9 | "dingospeed/pkg/prom" 10 | "dingospeed/pkg/util" 11 | 12 | "github.com/labstack/echo/v4" 13 | ) 14 | 15 | var requestQueue chan struct{} 16 | 17 | func InitMiddlewareConfig() { 18 | requestQueue = make(chan struct{}, config.SysConfig.TokenBucketLimit.HandlerCapacity) 19 | } 20 | 21 | func QueueLimitMiddleware(next echo.HandlerFunc) echo.HandlerFunc { 22 | return func(c echo.Context) error { 23 | url := c.Request().URL.String() 24 | remoteAddr := c.Request().RemoteAddr 25 | source, _, err := net.SplitHostPort(remoteAddr) 26 | if err != nil { 27 | return err 28 | } 29 | c.Set(consts.PromSource, source) 30 | if config.SysConfig.EnableMetric() { 31 | metrics := strings.Contains(url, "metrics") 32 | if metrics { 33 | return next(c) 34 | } 35 | promFlag := strings.Contains(url, "resolve") || strings.Contains(url, "revision") 36 | if promFlag { 37 | prom.PromSourceCounter(prom.RequestTotalCnt, source) 38 | select { 39 | case requestQueue <- struct{}{}: 40 | defer func() { 41 | <-requestQueue 42 | }() 43 | if err = next(c); err != nil { 44 | prom.PromSourceCounter(prom.RequestFailCnt, source) 45 | return err 46 | } else { 47 | prom.PromSourceCounter(prom.RequestSuccessCnt, source) 48 | return nil 49 | } 50 | default: 51 | prom.PromSourceCounter(prom.RequestTooManyCnt, source) 52 | return util.ErrorTooManyRequest(c) 53 | } 54 | } else { 55 | return nextRequest(c, next) 56 | } 57 | } else { 58 | return nextRequest(c, next) 59 | } 60 | } 61 | } 62 | 63 | func nextRequest(c echo.Context, next echo.HandlerFunc) error { 64 | select { 65 | case requestQueue <- struct{}{}: 66 | defer func() { 67 | <-requestQueue 68 | }() 69 | return next(c) 70 | default: 71 | return util.ErrorTooManyRequest(c) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /pkg/prom/prometheus.go: -------------------------------------------------------------------------------- 1 | package prom 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | // 请求总数 10 | 11 | RequestTotalCnt = promauto.NewGaugeVec(prometheus.GaugeOpts{ 12 | Name: "request_total_cnt", 13 | Help: "Total number of request total", 14 | }, []string{"source"}) 15 | // 请求成功数 16 | 17 | RequestSuccessCnt = promauto.NewGaugeVec(prometheus.GaugeOpts{ 18 | Name: "request_success_cnt", 19 | Help: "Total number of request done", 20 | }, []string{"source"}) 21 | // 请求失败数，客户端或服务端主动端口或程序异常 22 | 23 | RequestFailCnt = promauto.NewGaugeVec(prometheus.GaugeOpts{ 24 | Name: "request_fail_cnt", 25 | Help: "Total number of request fail", 26 | }, []string{"source"}) 27 | 28 | // 请求过多，被拒绝 29 | 30 | RequestTooManyCnt = promauto.NewGaugeVec(prometheus.GaugeOpts{ 31 | Name: "request_too_many_cnt", 32 | Help: "Total number of request too many", 33 | }, []string{"source"}) 34 | 35 | // 模型统计 36 | 37 | RequestModelCnt = promauto.NewGaugeVec(prometheus.GaugeOpts{ 38 | Name: "request_model_cnt", 39 | Help: "Total number of request model", 40 | }, []string{"models", "source"}) 41 | 42 | RequestDataSetCnt = promauto.NewGaugeVec(prometheus.GaugeOpts{ 43 | Name: "request_dataset_cnt", 44 | Help: "Total number of request dataset", 45 | }, []string{"datasets", "source"}) 46 | 47 | // 流量统计 48 | 49 | RequestRemoteByte = promauto.NewCounterVec(prometheus.CounterOpts{ 50 | Name: "request_remote_byte", 51 | Help: "Total number of request remote byte", 52 | }, []string{"source"}) 53 | 54 | RequestResponseByte = promauto.NewCounterVec(prometheus.CounterOpts{ 55 | Name: "request_response_byte", 56 | Help: "Total number of request response byte", 57 | }, []string{"source"}) 58 | ) 59 | 60 | func PromSourceCounter(vec *prometheus.GaugeVec, source string) { 61 | labels := prometheus.Labels{} 62 | labels["source"] = source 63 | vec.With(labels).Inc() 64 | } 65 | 66 | func PromRequestByteCounter(vec *prometheus.CounterVec, source string, len int64) { 67 | labels := prometheus.Labels{} 68 | labels["source"] = source 69 | vec.With(labels).Add(float64(len)) 70 | } 71 | -------------------------------------------------------------------------------- /pkg/server/server.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "context" 19 | ) 20 | 21 | type Server interface { 22 | Start(context.Context) error 23 | Stop(context.Context) error 24 | } 25 | -------------------------------------------------------------------------------- /pkg/util/compress.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "bytes" 19 | "compress/gzip" 20 | "compress/zlib" 21 | "fmt" 22 | "io" 23 | "strings" 24 | 25 | "github.com/andybalholm/brotli" 26 | "github.com/klauspost/compress/zstd" 27 | ) 28 | 29 | // DecompressData 对压缩的数据进行解压缩 30 | func DecompressData(rawData []byte, contentEncoding string) ([]byte, error) { 31 | fmt.Printf("decompress_data parameters:\n- content_encoding: %s\n", contentEncoding) 32 | if contentEncoding == "" { 33 | return rawData, nil 34 | } 35 | algorithms := strings.Split(contentEncoding, ",") 36 | finalData := rawData 37 | for _, algo := range algorithms { 38 | algo = strings.TrimSpace(strings.ToLower(algo)) 39 | var err error 40 | switch algo { 41 | case "gzip": 42 | finalData, err = decompressGzip(finalData) 43 | if err != nil { 44 | return nil, fmt.Errorf("error decompressing gzip data: %w", err) 45 | } 46 | case "deflate": 47 | finalData, err = decompressDeflate(finalData) 48 | if err != nil { 49 | return nil, fmt.Errorf("error decompressing deflate data: %w", err) 50 | } 51 | case "br": 52 | finalData, err = decompressBrotli(finalData) 53 | if err != nil { 54 | return nil, fmt.Errorf("error decompressing Brotli data: %w", err) 55 | } 56 | case "zstd": 57 | finalData, err = decompressZstd(finalData) 58 | if err != nil { 59 | return nil, fmt.Errorf("error decompressing Zstandard data: %w", err) 60 | } 61 | case "compress": 62 | return nil, fmt.Errorf("unsupported decompression algorithm: %s", algo) 63 | default: 64 | return nil, fmt.Errorf("unsupported compression algorithm: %s", algo) 65 | } 66 | } 67 | return finalData, nil 68 | } 69 | 70 | // decompressGzip 解压缩 gzip 数据 71 | func decompressGzip(data []byte) ([]byte, error) { 72 | buf := bytes.NewBuffer(data) 73 | gzr, err := gzip.NewReader(buf) 74 | if err != nil { 75 | return nil, err 76 | } 77 | defer gzr.Close() 78 | return io.ReadAll(gzr) 79 | } 80 | 81 | // decompressDeflate 解压缩 deflate 数据 82 | func decompressDeflate(data []byte) ([]byte, error) { 83 | buf := bytes.NewBuffer(data) 84 | r, err := zlib.NewReader(buf) 85 | if err != nil { 86 | return nil, err 87 | } 88 | defer r.Close() 89 | return io.ReadAll(r) 90 | } 91 | 92 | // decompressBrotli 解压缩 Brotli 数据 93 | func decompressBrotli(data []byte) ([]byte, error) { 94 | buf := bytes.NewBuffer(data) 95 | br := brotli.NewReader(buf) 96 | return io.ReadAll(br) 97 | } 98 | 99 | // decompressZstd 解压缩 Zstandard 数据 100 | func decompressZstd(data []byte) ([]byte, error) { 101 | decoder, err := zstd.NewReader(nil) 102 | if err != nil { 103 | return nil, err 104 | } 105 | defer decoder.Close() 106 | decompressed, err := decoder.DecodeAll(data, nil) 107 | if err != nil { 108 | return nil, err 109 | } 110 | return decompressed, nil 111 | } 112 | -------------------------------------------------------------------------------- /pkg/util/http_util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "bytes" 19 | "io" 20 | "net/http" 21 | "net/url" 22 | "time" 23 | 24 | "dingospeed/pkg/common" 25 | "dingospeed/pkg/config" 26 | "dingospeed/pkg/consts" 27 | "dingospeed/pkg/prom" 28 | 29 | "github.com/avast/retry-go" 30 | "github.com/labstack/echo/v4" 31 | "go.uber.org/zap" 32 | ) 33 | 34 | func RetryRequest(f func() (*common.Response, error)) (*common.Response, error) { 35 | var resp *common.Response 36 | err := retry.Do( 37 | func() error { 38 | var err error 39 | resp, err = f() 40 | return err 41 | }, 42 | retry.Delay(time.Duration(config.SysConfig.Retry.Delay)*time.Second), 43 | retry.Attempts(config.SysConfig.Retry.Attempts), 44 | retry.DelayType(retry.FixedDelay), 45 | ) 46 | return resp, err 47 | } 48 | 49 | // Head 方法用于发送带请求头的 HEAD 请求 50 | func Head(url string, headers map[string]string, timeout time.Duration) (*common.Response, error) { 51 | req, err := http.NewRequest("HEAD", url, nil) 52 | if err != nil { 53 | return nil, err 54 | } 55 | for key, value := range headers { 56 | req.Header.Set(key, value) 57 | } 58 | client := &http.Client{} 59 | client.Timeout = timeout 60 | resp, err := client.Do(req) 61 | if err != nil { 62 | return nil, err 63 | } 64 | respHeaders := make(map[string]interface{}) 65 | for key, value := range resp.Header { 66 | respHeaders[key] = value 67 | } 68 | return &common.Response{ 69 | StatusCode: resp.StatusCode, 70 | Headers: respHeaders, 71 | }, nil 72 | } 73 | 74 | // Get 方法用于发送带请求头的 GET 请求 75 | func Get(url string, headers map[string]string, timeout time.Duration) (*common.Response, error) { 76 | req, err := http.NewRequest("GET", url, nil) 77 | if err != nil { 78 | return nil, err 79 | } 80 | for key, value := range headers { 81 | req.Header.Set(key, value) 82 | } 83 | client := &http.Client{} 84 | client.Timeout = timeout 85 | resp, err := client.Do(req) 86 | if err != nil { 87 | return nil, err 88 | } 89 | defer resp.Body.Close() 90 | body, err := io.ReadAll(resp.Body) 91 | if err != nil { 92 | return nil, err 93 | } 94 | respHeaders := make(map[string]interface{}) 95 | for key, value := range resp.Header { 96 | respHeaders[key] = value 97 | } 98 | return &common.Response{ 99 | StatusCode: resp.StatusCode, 100 | Headers: respHeaders, 101 | Body: body, 102 | }, nil 103 | } 104 | 105 | func GetStream(url string, headers map[string]string, timeout time.Duration, f func(r *http.Response)) error { 106 | client := &http.Client{} 107 | client.Timeout = timeout 108 | req, err := http.NewRequest("GET", url, nil) 109 | if err != nil { 110 | return err 111 | } 112 | for key, value := range headers { 113 | req.Header.Set(key, value) 114 | } 115 | resp, err := client.Do(req) 116 | if err != nil { 117 | return err 118 | } 119 | defer resp.Body.Close() 120 | respHeaders := make(map[string]interface{}) 121 | for key, value := range resp.Header { 122 | respHeaders[key] = value 123 | } 124 | f(resp) 125 | return nil 126 | } 127 | 128 | // Post 方法用于发送带请求头的 POST 请求 129 | func Post(url string, contentType string, data []byte, headers map[string]string) (*common.Response, error) { 130 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) 131 | if err != nil { 132 | return nil, err 133 | } 134 | req.Header.Set("Content-Type", contentType) 135 | for key, value := range headers { 136 | req.Header.Set(key, value) 137 | } 138 | client := &http.Client{} 139 | resp, err := client.Do(req) 140 | if err != nil { 141 | return nil, err 142 | } 143 | defer resp.Body.Close() 144 | body, err := io.ReadAll(resp.Body) 145 | if err != nil { 146 | return nil, err 147 | } 148 | respHeaders := make(map[string]interface{}) 149 | for key, value := range resp.Header { 150 | respHeaders[key] = value 151 | } 152 | return &common.Response{ 153 | StatusCode: resp.StatusCode, 154 | Headers: respHeaders, 155 | Body: body, 156 | }, nil 157 | } 158 | 159 | func ResponseStream(c echo.Context, fileName string, headers map[string]string, content <-chan []byte) error { 160 | c.Response().Header().Set("Content-Type", "text/event-stream") 161 | c.Response().Header().Set("Cache-Control", "no-cache") 162 | c.Response().Header().Set("Connection", "keep-alive") 163 | for k, v := range headers { 164 | c.Response().Header().Set(k, v) 165 | } 166 | c.Response().WriteHeader(http.StatusOK) 167 | flusher, ok := c.Response().Writer.(http.Flusher) 168 | if !ok { 169 | return c.String(http.StatusInternalServerError, "Streaming unsupported!") 170 | } 171 | for { 172 | select { 173 | case b, ok := <-content: 174 | if !ok { 175 | zap.S().Infof("ResponseStream complete, %s.", fileName) 176 | return nil 177 | } 178 | if len(b) > 0 { 179 | if _, err := c.Response().Write(b); err != nil { 180 | zap.S().Warnf("ResponseStream write err,file:%s,%v", fileName, err) 181 | return ErrorProxyTimeout(c) 182 | } 183 | if config.SysConfig.EnableMetric() { 184 | // 原子性地更新响应总数 185 | source := Itoa(c.Get(consts.PromSource)) 186 | prom.PromRequestByteCounter(prom.RequestResponseByte, source, int64(len(b))) 187 | } 188 | } 189 | flusher.Flush() 190 | } 191 | } 192 | } 193 | 194 | func GetDomain(hfURL string) (string, error) { 195 | parsedURL, err := url.Parse(hfURL) 196 | if err != nil { 197 | return "", err 198 | } 199 | return parsedURL.Host, nil 200 | } 201 | -------------------------------------------------------------------------------- /pkg/util/repo_util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "encoding/gob" 19 | "fmt" 20 | "os" 21 | "path/filepath" 22 | "sort" 23 | "syscall" 24 | "time" 25 | 26 | "dingospeed/pkg/common" 27 | 28 | "github.com/bytedance/sonic" 29 | ) 30 | 31 | func GetOrgRepo(org, repo string) string { 32 | if org == "" { 33 | return repo 34 | } else { 35 | return fmt.Sprintf("%s/%s", org, repo) 36 | } 37 | } 38 | 39 | // MakeDirs 确保指定路径对应的目录存在 40 | func MakeDirs(path string) error { 41 | fileInfo, err := os.Stat(path) 42 | if err == nil { 43 | if fileInfo.IsDir() { 44 | // 如果路径本身就是目录，直接使用该路径 45 | return nil 46 | } 47 | } 48 | 49 | // 如果路径不是目录，获取其父目录 50 | saveDir := filepath.Dir(path) 51 | // 检查目录是否存在 52 | _, err = os.Stat(saveDir) 53 | if os.IsNotExist(err) { 54 | // 目录不存在，创建目录 55 | err = os.MkdirAll(saveDir, 0755) 56 | if err != nil { 57 | return err 58 | } 59 | } else if err != nil { 60 | // 其他错误 61 | return err 62 | } 63 | return nil 64 | } 65 | 66 | // FileExists 函数用于判断文件是否存在 67 | func FileExists(filename string) bool { 68 | _, err := os.Stat(filename) 69 | if os.IsNotExist(err) { 70 | return false 71 | } 72 | return err == nil 73 | } 74 | 75 | // IsDir 判断所给路径是否为文件夹 76 | func IsDir(path string) bool { 77 | s, err := os.Stat(path) 78 | if err != nil { 79 | return false 80 | } 81 | return s.IsDir() 82 | } 83 | 84 | // IsFile 判断所给文件是否存在 85 | func IsFile(path string) bool { 86 | s, err := os.Stat(path) 87 | if err != nil { 88 | return false 89 | } 90 | return !s.IsDir() 91 | } 92 | 93 | // GetFileSize 获取文件大小 94 | func GetFileSize(path string) int64 { 95 | fh, err := os.Stat(path) 96 | if err != nil { 97 | fmt.Printf("读取文件%s失败, err: %s\n", path, err) 98 | } 99 | return fh.Size() 100 | } 101 | 102 | func ReName(src, dst string) { 103 | dstDir := filepath.Dir(dst) 104 | if err := os.MkdirAll(dstDir, 0755); err != nil { 105 | fmt.Printf("创建目录失败: %v\n", err) 106 | return 107 | } 108 | if err := os.Rename(src, dst); err != nil { 109 | fmt.Printf("移动文件失败: %v\n", err) 110 | return 111 | } 112 | } 113 | 114 | func CreateSymlinkIfNotExists(src, dst string) error { 115 | _, err := os.Lstat(dst) 116 | if os.IsNotExist(err) { 117 | // 获取 dst 所在的目录 118 | dstDir := filepath.Dir(dst) 119 | // 计算 src 相对于 dstDir 的路径 120 | relSrc, err := filepath.Rel(dstDir, src) 121 | if err != nil { 122 | return fmt.Errorf("计算相对路径失败: %v", err) 123 | } 124 | return os.Symlink(relSrc, dst) 125 | } 126 | return err 127 | } 128 | 129 | func ReadFileToBytes(filename string) ([]byte, error) { 130 | return os.ReadFile(filename) 131 | } 132 | 133 | func WriteDataToFile(filename string, data interface{}) error { 134 | jsonData, err := sonic.Marshal(data) 135 | if err != nil { 136 | return fmt.Errorf("JSON 编码出错: %w", err) 137 | } 138 | file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) 139 | if err != nil { 140 | return fmt.Errorf("打开文件出错: %w", err) 141 | } 142 | defer file.Close() 143 | _, err = file.Write(jsonData) 144 | if err != nil { 145 | return fmt.Errorf("写入文件出错: %w", err) 146 | } 147 | return nil 148 | } 149 | 150 | // StoreMetadata 保存文件元数据 151 | func StoreMetadata(filePath string, metadata *common.FileMetadata) error { 152 | // 写入文件 153 | file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666) 154 | if err != nil { 155 | fmt.Printf("写元数据文件%s失败\n", filePath) 156 | return err 157 | } 158 | defer file.Close() 159 | 160 | enc := gob.NewEncoder(file) 161 | err = enc.Encode(metadata) 162 | if err != nil { 163 | fmt.Printf("写元数据文件%s失败\n", filePath) 164 | return err 165 | } 166 | return nil 167 | } 168 | 169 | func SplitFileToSegment(fileSize int64, blockSize int64) (int, []*common.Segment) { 170 | segments := make([]*common.Segment, 0) 171 | start, index := int64(0), 0 172 | for start < fileSize { 173 | end := start + blockSize 174 | if end > fileSize { 175 | end = fileSize 176 | } 177 | segments = append(segments, &common.Segment{Index: index, Start: start, End: end}) 178 | end++ 179 | index++ 180 | start = end 181 | } 182 | return index, segments 183 | } 184 | 185 | func GetFolderSize(path string) (int64, error) { 186 | var size int64 187 | err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { 188 | if err != nil { 189 | return err 190 | } 191 | if !info.IsDir() { 192 | size += info.Size() 193 | } 194 | return nil 195 | }) 196 | return size, err 197 | } 198 | 199 | // FileWithPath 自定义结构体，用于存储文件信息和对应的路径 200 | type FileWithPath struct { 201 | Info os.FileInfo 202 | Path string 203 | } 204 | 205 | // getAccessTime 跨平台获取文件访问时间 206 | func getAccessTime(info os.FileInfo) time.Time { 207 | if stat, ok := info.Sys().(*syscall.Stat_t); ok { 208 | if ts, ok := tryGetAtime(stat); ok { 209 | return time.Unix(ts.Sec, ts.Nsec) 210 | } 211 | } 212 | // 若无法获取访问时间，使用修改时间替代 213 | return info.ModTime() 214 | } 215 | 216 | // tryGetAtime 尝试不同方式获取文件访问时间 217 | func tryGetAtime(stat *syscall.Stat_t) (syscall.Timespec, bool) { 218 | if v, ok := interface{}(stat).(interface{ Atimespec() syscall.Timespec }); ok { 219 | return v.Atimespec(), true 220 | } 221 | if v, ok := interface{}(stat).(interface{ Atim() syscall.Timespec }); ok { 222 | return v.Atim(), true 223 | } 224 | 225 | return syscall.Timespec{}, false 226 | } 227 | 228 | // SortFilesByAccessTime 按文件访问时间对指定路径下的文件进行正序排序 229 | func SortFilesByAccessTime(path string) ([]FileWithPath, error) { 230 | var filesWithPaths []FileWithPath 231 | err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error { 232 | if err != nil { 233 | return err 234 | } 235 | if !info.IsDir() { 236 | filesWithPaths = append(filesWithPaths, FileWithPath{ 237 | Info: info, 238 | Path: p, 239 | }) 240 | } 241 | return nil 242 | }) 243 | if err != nil { 244 | return nil, err 245 | } 246 | 247 | // 按访问时间对文件进行正序排序，秒数相同则比较纳秒 248 | sort.Slice(filesWithPaths, func(i, j int) bool { 249 | timeI := getAccessTime(filesWithPaths[i].Info) 250 | timeJ := getAccessTime(filesWithPaths[j].Info) 251 | if timeI.Unix() == timeJ.Unix() { 252 | return timeI.Nanosecond() < timeJ.Nanosecond() 253 | } 254 | return timeI.Before(timeJ) 255 | }) 256 | 257 | return filesWithPaths, nil 258 | } 259 | 260 | func SortFilesByModifyTime(path string) ([]FileWithPath, error) { 261 | filesWithPaths, err := SortFilesByAccessTime(path) 262 | return filesWithPaths, err 263 | } 264 | 265 | // SortFilesBySize 按文件大小对指定路径下的文件进行降序排序 266 | func SortFilesBySize(path string) ([]FileWithPath, error) { 267 | var filesWithPaths []FileWithPath 268 | // 获取今天的日期 269 | now := time.Now() 270 | year, month, day := now.Date() 271 | today := time.Date(year, month, day, 0, 0, 0, 0, now.Location()) 272 | 273 | err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error { 274 | if err != nil { 275 | return err 276 | } 277 | if !info.IsDir() { 278 | // 获取文件修改时间 279 | modTime := info.ModTime() 280 | // 检查文件修改时间是否不是今天 281 | if modTime.Before(today) { 282 | filesWithPaths = append(filesWithPaths, FileWithPath{ 283 | Info: info, 284 | Path: p, 285 | }) 286 | } 287 | } 288 | return nil 289 | }) 290 | 291 | if err != nil { 292 | return nil, err 293 | } 294 | 295 | sort.Slice(filesWithPaths, func(i, j int) bool { 296 | // 比较文件大小，降序排序 297 | return filesWithPaths[i].Info.Size() > filesWithPaths[j].Info.Size() 298 | }) 299 | 300 | return filesWithPaths, nil 301 | } 302 | 303 | func ConvertBytesToHumanReadable(bytes int64) string { 304 | const unit = 1024 305 | if bytes < unit { 306 | return fmt.Sprintf("%d B", bytes) 307 | } 308 | div, exp := int64(unit), 0 309 | for n := bytes / unit; n >= unit; n /= unit { 310 | div *= unit 311 | exp++ 312 | } 313 | return fmt.Sprintf("%.1f %ciB", float64(bytes)/float64(div), "KMGTPE"[exp]) 314 | } 315 | -------------------------------------------------------------------------------- /pkg/util/response.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "fmt" 19 | "net/http" 20 | 21 | "github.com/labstack/echo/v4" 22 | ) 23 | 24 | func ErrorRepoNotFound(ctx echo.Context) error { 25 | content := map[string]string{ 26 | "error": "Repository not found", 27 | } 28 | headers := map[string]string{ 29 | "x-error-code": "RepoNotFound", 30 | "x-error-message": "Repository not found", 31 | } 32 | return Response(ctx, http.StatusNotFound, headers, content) 33 | } 34 | 35 | func ErrorRequestParam(ctx echo.Context) error { 36 | content := map[string]string{ 37 | "error": "Request param error", 38 | } 39 | headers := map[string]string{ 40 | "x-error-code": "Request param error", 41 | "x-error-message": "Request param error", 42 | } 43 | return Response(ctx, http.StatusBadRequest, headers, content) 44 | } 45 | 46 | func ErrorPageNotFound(ctx echo.Context) error { 47 | content := map[string]string{ 48 | "error": "Sorry, we can't find the page you are looking for.", 49 | } 50 | headers := map[string]string{ 51 | "x-error-code": "RepoNotFound", 52 | "x-error-message": "Sorry, we can't find the page you are looking for.", 53 | } 54 | return Response(ctx, http.StatusNotFound, headers, content) 55 | } 56 | 57 | func ErrorEntryNotFoundBranch(ctx echo.Context, branch, path string) error { 58 | headers := map[string]string{ 59 | "x-error-code": "EntryNotFound", 60 | "x-error-message": fmt.Sprintf("%s does not exist on %s", branch, path), 61 | } 62 | return Response(ctx, http.StatusUnauthorized, headers, nil) 63 | } 64 | 65 | func ErrorEntryUnknown(ctx echo.Context, statusCode int, msg string) error { 66 | content := map[string]string{ 67 | "error": msg, 68 | } 69 | return Response(ctx, statusCode, nil, content) 70 | } 71 | 72 | func ErrorEntryNotFound(ctx echo.Context) error { 73 | headers := map[string]string{ 74 | "x-error-code": "EntryNotFound", 75 | "x-error-message": "Entry not found", 76 | } 77 | return Response(ctx, http.StatusNotFound, headers, nil) 78 | } 79 | 80 | func ErrorRevisionNotFound(ctx echo.Context, revision string) error { 81 | content := map[string]string{ 82 | "error": fmt.Sprintf("Invalid rev id: %s", revision), 83 | } 84 | headers := map[string]string{ 85 | "x-error-code": "RevisionNotFound", 86 | "x-error-message": fmt.Sprintf("Invalid rev id: %s", revision), 87 | } 88 | return Response(ctx, http.StatusNotFound, headers, content) 89 | } 90 | 91 | func ErrorProxyTimeout(ctx echo.Context) error { 92 | headers := map[string]string{ 93 | "x-error-code": "ProxyTimeout", 94 | "x-error-message": "Proxy Timeout", 95 | } 96 | return Response(ctx, http.StatusGatewayTimeout, headers, nil) 97 | } 98 | 99 | func ErrorProxyError(ctx echo.Context) error { 100 | headers := map[string]string{ 101 | "x-error-code": "Proxy error", 102 | "x-error-message": "Proxy error", 103 | } 104 | return Response(ctx, http.StatusInternalServerError, headers, nil) 105 | } 106 | 107 | func ErrorMethodError(ctx echo.Context) error { 108 | content := map[string]string{ 109 | "error": "request method error", 110 | } 111 | headers := map[string]string{ 112 | "x-error-code": "request method error", 113 | "x-error-message": "request method error", 114 | } 115 | return Response(ctx, http.StatusInternalServerError, headers, content) 116 | } 117 | 118 | func ErrorTooManyRequest(ctx echo.Context) error { 119 | content := map[string]string{ 120 | "error": "Too many requests", 121 | } 122 | return Response(ctx, http.StatusTooManyRequests, nil, content) 123 | } 124 | 125 | func ResponseHeaders(ctx echo.Context, headers map[string]string) error { 126 | fullHeaders(ctx, headers) 127 | return ctx.JSON(http.StatusOK, nil) 128 | } 129 | 130 | func Response(ctx echo.Context, httpStatus int, headers map[string]string, data interface{}) error { 131 | fullHeaders(ctx, headers) 132 | return ctx.JSON(httpStatus, data) 133 | } 134 | 135 | func ResponseData(ctx echo.Context, data interface{}) error { 136 | return ctx.JSON(http.StatusOK, data) 137 | } 138 | 139 | func fullHeaders(c echo.Context, headers map[string]string) { 140 | for k, v := range headers { 141 | c.Response().Header().Set(k, v) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package util 16 | 17 | import ( 18 | "crypto/md5" 19 | "encoding/hex" 20 | "encoding/json" 21 | "hash/crc32" 22 | "io" 23 | "math" 24 | "net/http" 25 | "path/filepath" 26 | "reflect" 27 | "strconv" 28 | "strings" 29 | "time" 30 | 31 | myerr "dingospeed/pkg/error" 32 | 33 | "github.com/bytedance/sonic" 34 | "github.com/google/uuid" 35 | ) 36 | 37 | func Max(a, b int) int { 38 | if a > b { 39 | return a 40 | } 41 | return b 42 | } 43 | 44 | func Min(a, b int) int { 45 | if a > b { 46 | return b 47 | } 48 | return a 49 | } 50 | 51 | func Itoa(a interface{}) string { 52 | switch at := a.(type) { 53 | case int, int8, int16, int64, int32: 54 | return strconv.FormatInt(reflect.ValueOf(a).Int(), 10) 55 | case uint, uint8, uint16, uint32, uint64: 56 | return strconv.FormatInt(int64(reflect.ValueOf(a).Uint()), 10) 57 | case float32, float64: 58 | return strconv.FormatFloat(reflect.ValueOf(a).Float(), 'f', -1, 64) 59 | case string: 60 | return at 61 | } 62 | return "" 63 | } 64 | 65 | func Atoi(a string) int { 66 | if a == "" { 67 | return 0 68 | } 69 | r, e := strconv.Atoi(a) 70 | if e == nil { 71 | return r 72 | } 73 | return 0 74 | } 75 | 76 | func Atoi64(a string) int64 { 77 | b, err := strconv.ParseInt(a, 10, 64) 78 | if err != nil { 79 | return 0 80 | } 81 | return b 82 | } 83 | 84 | // 转Int 85 | func AnyToInt(value interface{}) int { 86 | if value == nil { 87 | return 0 88 | } 89 | switch val := value.(type) { 90 | case int: 91 | return val 92 | case int8: 93 | return int(val) 94 | case int16: 95 | return int(val) 96 | case int32: 97 | return int(val) 98 | case int64: 99 | return int(val) 100 | case uint: 101 | return int(val) 102 | case uint8: 103 | return int(val) 104 | case uint16: 105 | return int(val) 106 | case uint32: 107 | return int(val) 108 | case uint64: 109 | return int(val) 110 | case *string: 111 | v, err := strconv.Atoi(*val) 112 | if err != nil { 113 | return 0 114 | } 115 | return v 116 | case string: 117 | v, err := strconv.Atoi(val) 118 | if err != nil { 119 | return 0 120 | } 121 | return v 122 | case float32: 123 | return int(val) 124 | case float64: 125 | return int(val) 126 | case bool: 127 | if val { 128 | return 1 129 | } else { 130 | return 0 131 | } 132 | case json.Number: 133 | v, _ := val.Int64() 134 | return int(v) 135 | } 136 | 137 | return 0 138 | } 139 | 140 | func HashCode(s string) uint32 { 141 | return crc32.ChecksumIEEE([]byte(s)) 142 | } 143 | 144 | // BindJSONWithDisallowUnknownFields 验证传入的请求是否合理 145 | func BindJSONWithDisallowUnknownFields(req *http.Request, obj interface{}) error { 146 | decoder := json.NewDecoder(req.Body) 147 | decoder.DisallowUnknownFields() 148 | err := decoder.Decode(obj) 149 | if err != nil { 150 | return err 151 | } 152 | return nil 153 | } 154 | 155 | func TimeToInt64(time time.Time) int64 { 156 | if time.Unix() > 0 { 157 | return time.Unix() 158 | } 159 | return 0 160 | } 161 | 162 | // 秒级时间戳转time 163 | func UnixSecondToTime(second int64) time.Time { 164 | return time.Unix(second, 0) 165 | } 166 | 167 | // 毫秒级时间戳转time 168 | func UnixMilliToTime(milli int64) time.Time { 169 | return time.Unix(milli/1000, (milli%1000)*(1000*1000)) 170 | } 171 | 172 | // 纳秒级时间戳转time 173 | func UnixNanoToTime(nano int64) time.Time { 174 | return time.Unix(nano/(1000*1000*1000), nano%(1000*1000*1000)) 175 | } 176 | 177 | // 转义百分号 178 | func EscapePercent(s string) string { 179 | return strings.ReplaceAll(s, "%", "\\%") 180 | } 181 | 182 | func StringSliceToInt64Slice(s []string) []int { 183 | r := make([]int, 0, len(s)) 184 | for _, v := range s { 185 | r = append(r, Atoi(v)) 186 | } 187 | return r 188 | } 189 | 190 | func SetValWhenFloatIsNaNOrInf(val float64) float64 { 191 | if math.IsNaN(val) { 192 | return 0.00 193 | } 194 | if math.IsInf(val, 0) { 195 | return 100.00 196 | } 197 | return val 198 | } 199 | 200 | func ReadText(reader io.ReaderAt, i int64) string { 201 | buffer := make([]byte, i) 202 | n, _ := reader.ReadAt(buffer, i) 203 | return string(buffer[:n]) 204 | } 205 | 206 | func CalculatePercentage(a, b float32) (float32, error) { 207 | if b == 0 { 208 | return 0, myerr.New("division by zero is not allowed") 209 | } 210 | percentage := (a / b) * 100 211 | roundedPercentage := float32(int(percentage*100+0.5)) / 100 212 | return roundedPercentage, nil 213 | } 214 | 215 | func UUID() string { 216 | return uuid.New().String() 217 | } 218 | 219 | func Md5(str string) string { 220 | hash := md5.Sum([]byte(str)) 221 | hashString := hex.EncodeToString(hash[:]) 222 | return hashString 223 | } 224 | 225 | func ToJsonString(data interface{}) string { 226 | jsonData, _ := sonic.Marshal(data) 227 | return string(jsonData) 228 | } 229 | 230 | func ProcessPaths(paths []string) []string { 231 | var repos []string 232 | for _, p := range paths { 233 | parts := strings.Split(p, string(filepath.Separator)) 234 | if len(parts) >= 2 { 235 | org := parts[len(parts)-2] 236 | repo := parts[len(parts)-1] 237 | repos = append(repos, org+"/"+repo) 238 | } 239 | } 240 | return repos 241 | } 242 | -------------------------------------------------------------------------------- /png/architecture_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dingodb/dingospeed/10ab1dbc7044efe7ca6646cfed3c8d53899ec711/png/architecture_en.png -------------------------------------------------------------------------------- /png/downloading_models_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dingodb/dingospeed/10ab1dbc7044efe7ca6646cfed3c8d53899ec711/png/downloading_models_en.png -------------------------------------------------------------------------------- /png/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dingodb/dingospeed/10ab1dbc7044efe7ca6646cfed3c8d53899ec711/png/img.png -------------------------------------------------------------------------------- /png/img_download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dingodb/dingospeed/10ab1dbc7044efe7ca6646cfed3c8d53899ec711/png/img_download.png -------------------------------------------------------------------------------- /png/img_store.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dingodb/dingospeed/10ab1dbc7044efe7ca6646cfed3c8d53899ec711/png/img_store.png -------------------------------------------------------------------------------- /png/storing_models_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dingodb/dingospeed/10ab1dbc7044efe7ca6646cfed3c8d53899ec711/png/storing_models_en.png -------------------------------------------------------------------------------- /repair/data_repair.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/hex" 5 | "flag" 6 | "fmt" 7 | "net/http" 8 | "os" 9 | 10 | "dingospeed/pkg/common" 11 | myerr "dingospeed/pkg/error" 12 | "dingospeed/pkg/util" 13 | 14 | "github.com/bytedance/sonic" 15 | "github.com/labstack/gommon/log" 16 | "go.uber.org/zap" 17 | ) 18 | 19 | var ( 20 | repoPathParam string 21 | repoTypeParam string 22 | orgParam string 23 | repoParam string 24 | ) 25 | 26 | type CommitHfSha struct { 27 | Sha string `json:"sha"` 28 | Siblings []struct { 29 | Rfilename string `json:"rfilename"` 30 | } `json:"siblings"` 31 | } 32 | 33 | func init() { 34 | flag.StringVar(&repoPathParam, "repoPath", "./repos", "仓库路径") 35 | flag.StringVar(&repoTypeParam, "repoType", "models", "类型") 36 | flag.StringVar(&orgParam, "org", "", "组织") 37 | flag.StringVar(&repoParam, "repo", "", "仓库") 38 | flag.Parse() 39 | } 40 | 41 | func main() { 42 | fmt.Println("starting data repair....") 43 | if repoPathParam == "" || repoTypeParam == "" { 44 | log.Errorf("repoPath,repoType不能为空") 45 | return 46 | } 47 | if orgParam != "" && repoParam != "" { 48 | repoRepair(repoPathParam, repoTypeParam, orgParam, repoParam) 49 | } else { 50 | if orgParam != "" && repoParam == "" { 51 | orgRepair(repoPathParam, repoTypeParam, orgParam) 52 | } else if orgParam == "" && repoParam == "" { 53 | typePath := fmt.Sprintf("%s/api/%s", repoPathParam, repoTypeParam) 54 | // 读取目录内容 55 | orgEntries, err := os.ReadDir(typePath) 56 | if err != nil { 57 | fmt.Printf("读取目录失败: %v\n", err) 58 | return 59 | } 60 | for _, entry := range orgEntries { 61 | if entry.IsDir() { 62 | orgRepair(repoPathParam, repoTypeParam, entry.Name()) 63 | } 64 | } 65 | } 66 | } 67 | headsPath := fmt.Sprintf("%s/heads", repoPathParam) 68 | if exist := util.FileExists(headsPath); exist { 69 | err := os.RemoveAll(headsPath) 70 | if err != nil { 71 | fmt.Printf("删除目录失败: %v\n", err) 72 | return 73 | } 74 | } 75 | } 76 | 77 | func orgRepair(repoPath, repoType, org string) { 78 | orgPath := fmt.Sprintf("%s/api/%s/%s", repoPath, repoType, org) 79 | // 读取目录内容 80 | repoEntries, err := os.ReadDir(orgPath) 81 | if err != nil { 82 | fmt.Printf("读取目录失败: %v\n", err) 83 | return 84 | } 85 | for _, entry := range repoEntries { 86 | if entry.IsDir() { 87 | repoRepair(repoPath, repoType, org, entry.Name()) 88 | } 89 | } 90 | } 91 | 92 | func repoRepair(repoPath, repoType, org, repo string) { 93 | if repo == "" { 94 | panic("repo is null") 95 | } 96 | filePath := fmt.Sprintf("%s/files/%s/%s/%s", repoPath, repoType, org, repo) 97 | if exist := util.FileExists(filePath); !exist { 98 | return 99 | } 100 | fileBlobs := fmt.Sprintf("%s/blobs", filePath) 101 | if exist := util.FileExists(fileBlobs); exist { 102 | log.Errorf(fmt.Sprintf("该仓库已完成修复：%s", fileBlobs)) 103 | return 104 | } 105 | metaGetPath := fmt.Sprintf("%s/api/%s/%s/%s/revision/main/meta_get.json", repoPath, repoType, org, repo) 106 | if exist := util.FileExists(metaGetPath); !exist { 107 | log.Errorf(fmt.Sprintf("该%s/%s不存在meta_get文件，无法修复.", org, repo)) 108 | return 109 | } 110 | log.Infof("start repair：%s/%s/%s", repoType, org, repo) 111 | cacheContent, err := ReadCacheRequest(metaGetPath) 112 | if err != nil { 113 | return 114 | } 115 | var sha CommitHfSha 116 | if err = sonic.Unmarshal(cacheContent.OriginContent, &sha); err != nil { 117 | zap.S().Errorf("unmarshal error.%v", err) 118 | return 119 | } 120 | remoteReqFilePathMap := make(map[string]*common.PathsInfo, 0) 121 | for _, item := range sha.Siblings { 122 | remoteReqFilePathMap[item.Rfilename] = nil 123 | } 124 | getPathInfoOid(remoteReqFilePathMap, repoType, fmt.Sprintf("%s/%s", org, repo), sha.Sha) 125 | for _, item := range sha.Siblings { 126 | fileName := item.Rfilename 127 | pathInfo, ok := remoteReqFilePathMap[fileName] 128 | if !ok { 129 | continue 130 | } 131 | if err = updatePathInfo(repoPath, repoType, org, repo, sha.Sha, fileName, pathInfo); err != nil { 132 | continue 133 | } 134 | var etag string 135 | if pathInfo.Lfs.Oid != "" { 136 | etag = pathInfo.Lfs.Oid 137 | } else { 138 | etag = pathInfo.Oid 139 | } 140 | filePath = fmt.Sprintf("%s/files/%s/%s/%s/resolve/%s/%s", repoPath, repoType, org, repo, sha.Sha, fileName) 141 | if exist := util.FileExists(filePath); !exist { 142 | log.Warnf(fmt.Sprintf("该文件%s不存在.", filePath)) 143 | continue 144 | } 145 | newBlobsFilePath := fmt.Sprintf("%s/files/%s/%s/%s/blobs/%s", repoPath, repoType, org, repo, etag) 146 | util.ReName(filePath, newBlobsFilePath) 147 | err = util.CreateSymlinkIfNotExists(newBlobsFilePath, filePath) 148 | if err != nil { 149 | log.Errorf("CreateSymlinkIfNotExists err.%v", err) 150 | continue 151 | } 152 | } 153 | // 删除其他版本 154 | resolvePath := fmt.Sprintf("%s/files/%s/%s/%s/resolve", repoPath, repoType, org, repo) 155 | entries, err := os.ReadDir(resolvePath) 156 | if err != nil { 157 | fmt.Printf("读取目录失败: %v\n", err) 158 | return 159 | } 160 | for _, entry := range entries { 161 | if entry.IsDir() { 162 | if entry.Name() != sha.Sha { 163 | err := os.RemoveAll(fmt.Sprintf("%s/%s", resolvePath, entry.Name())) 164 | if err != nil { 165 | fmt.Printf("删除目录失败: %v\n", err) 166 | continue 167 | } 168 | } 169 | } 170 | } 171 | log.Infof("end repair：%s/%s/%s", repoType, org, repo) 172 | } 173 | 174 | func updatePathInfo(repoPath, repoType, org, repo, commit, fileName string, pathInfo *common.PathsInfo) error { 175 | pathInfoPath := fmt.Sprintf("%s/api/%s/%s/%s/paths-info/%s/%s/paths-info_post.json", repoPath, repoType, org, repo, commit, fileName) 176 | if exist := util.FileExists(pathInfoPath); !exist { 177 | return myerr.New("file is not exist") 178 | } 179 | cacheContent, err := ReadCacheRequest(pathInfoPath) 180 | if err != nil { 181 | log.Errorf(fmt.Sprintf("read file:%s err", pathInfoPath)) 182 | return err 183 | } 184 | pathsInfos := make([]*common.PathsInfo, 0) 185 | pathsInfos = append(pathsInfos, pathInfo) 186 | b, err := sonic.Marshal(pathsInfos) 187 | if err != nil { 188 | zap.S().Errorf("pathsInfo Unmarshal err.%v", err) 189 | return err 190 | } 191 | if err = WriteCacheRequest(pathInfoPath, cacheContent.StatusCode, cacheContent.Headers, b); err != nil { 192 | zap.S().Errorf("WriteCacheRequest err.%s,%v", pathInfoPath, err) 193 | return err 194 | } 195 | return nil 196 | } 197 | 198 | func WriteCacheRequest(apiPath string, statusCode int, headers map[string]string, content []byte) error { 199 | cacheContent := common.CacheContent{ 200 | StatusCode: statusCode, 201 | Headers: headers, 202 | Content: hex.EncodeToString(content), 203 | } 204 | return util.WriteDataToFile(apiPath, cacheContent) 205 | } 206 | 207 | func ReadCacheRequest(apiPath string) (*common.CacheContent, error) { 208 | cacheContent := common.CacheContent{} 209 | bytes, err := util.ReadFileToBytes(apiPath) 210 | if err != nil { 211 | return nil, myerr.Wrap("ReadFileToBytes err.", err) 212 | } 213 | if err = sonic.Unmarshal(bytes, &cacheContent); err != nil { 214 | return nil, err 215 | } 216 | decodeByte, err := hex.DecodeString(cacheContent.Content) 217 | if err != nil { 218 | return nil, myerr.Wrap("DecodeString err.", err) 219 | } 220 | cacheContent.OriginContent = decodeByte 221 | return &cacheContent, nil 222 | } 223 | 224 | func getPathInfoOid(remoteReqFilePathMap map[string]*common.PathsInfo, repoType, orgRepo, commit string) { 225 | filePaths := make([]string, 0) 226 | for k := range remoteReqFilePathMap { 227 | filePaths = append(filePaths, k) 228 | } 229 | pathsInfoUrl := fmt.Sprintf("%s/api/%s/%s/paths-info/%s", "https://hf-mirror.com", repoType, orgRepo, commit) 230 | response, err := pathsInfoProxy(pathsInfoUrl, "", filePaths) 231 | if err != nil { 232 | zap.S().Errorf("req %s err.%v", pathsInfoUrl, err) 233 | return 234 | } 235 | if response.StatusCode != http.StatusOK { 236 | zap.S().Errorf("response.StatusCode err:%d", response.StatusCode) 237 | return 238 | } 239 | remoteRespPathsInfos := make([]common.PathsInfo, 0) 240 | err = sonic.Unmarshal(response.Body, &remoteRespPathsInfos) 241 | if err != nil { 242 | zap.S().Errorf("req %s remoteRespPathsInfos Unmarshal err.%v", pathsInfoUrl, err) 243 | return 244 | } 245 | for _, item := range remoteRespPathsInfos { 246 | // 对单个文件pathsInfo做存储 247 | if _, ok := remoteReqFilePathMap[item.Path]; ok { 248 | remoteReqFilePathMap[item.Path] = &item 249 | } 250 | } 251 | } 252 | 253 | func pathsInfoProxy(targetUrl, authorization string, filePaths []string) (*common.Response, error) { 254 | data := map[string]interface{}{ 255 | "paths": filePaths, 256 | } 257 | jsonData, err := sonic.Marshal(data) 258 | if err != nil { 259 | return nil, err 260 | } 261 | headers := map[string]string{} 262 | if authorization != "" { 263 | headers["authorization"] = authorization 264 | } 265 | return util.Post(targetUrl, "application/json", jsonData, headers) 266 | } 267 | --------------------------------------------------------------------------------