├── .tool-versions ├── tdx ├── embed │ └── datatool.ini ├── datatool.go ├── gbbq.go ├── kline.go └── gbbq_var.go ├── .gitignore ├── Containerfile ├── utils ├── cache.go ├── unzip.go ├── iocheck.go ├── csv_write.go └── download.go ├── database ├── factory.go ├── interface.go ├── duckdb │ ├── driver.go │ ├── dml.go │ └── ddl.go └── clickhouse │ ├── driver.go │ ├── dml.go │ └── ddl.go ├── cmd ├── common.go ├── init.go ├── convert.go └── cron.go ├── model ├── views.go └── tables.go ├── LICENSE ├── .github └── workflows │ └── release.yaml ├── .goreleaser.yaml ├── makefile ├── go.mod ├── main.go ├── README.md ├── calc └── fq.go └── go.sum /.tool-versions: -------------------------------------------------------------------------------- 1 | golang 1.25.5 2 | -------------------------------------------------------------------------------- /tdx/embed/datatool.ini: -------------------------------------------------------------------------------- 1 | [PATH] 2 | 3 | VIPDOC=./vipdoc 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tdx2db 2 | datatool 3 | datatool.exe 4 | .tmp 5 | -------------------------------------------------------------------------------- /Containerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 docker.io/library/debian:sid-slim 2 | RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates && apt-get clean 3 | COPY linux/amd64/tdx2db / 4 | RUN chmod +x /tdx2db 5 | ENTRYPOINT ["/tdx2db"] 6 | -------------------------------------------------------------------------------- /utils/cache.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func GetCacheDir() (string, error) { 8 | appDir, err := os.MkdirTemp("", "tdx2db-temp-") 9 | if err != nil { 10 | return "", err 11 | } 12 | 13 | return appDir, nil 14 | } 15 | -------------------------------------------------------------------------------- /database/factory.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | 7 | "github.com/jing2uo/tdx2db/database/clickhouse" 8 | "github.com/jing2uo/tdx2db/database/duckdb" 9 | ) 10 | 11 | func NewDB(dbURI string) (DataRepository, error) { 12 | if dbURI == "" { 13 | return nil, fmt.Errorf("db uri cannot be empty") 14 | } 15 | u, err := url.Parse(dbURI) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | switch u.Scheme { 21 | case "clickhouse": 22 | return clickhouse.NewClickHouseDriver(u) 23 | case "duckdb": 24 | return duckdb.NewDuckDBDriver(u) 25 | default: 26 | return nil, fmt.Errorf("unsupported scheme: %s", u.Scheme) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /database/interface.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/jing2uo/tdx2db/model" 7 | ) 8 | 9 | type DataRepository interface { 10 | Connect() error 11 | Close() error 12 | 13 | InitSchema() error 14 | 15 | ImportDailyStocks(csvPath string) error 16 | Import1MinStocks(csvPath string) error 17 | Import5MinStocks(csvPath string) error 18 | ImportAdjustFactors(csvPath string) error 19 | ImportGBBQ(csvPath string) error 20 | ImportXDXR(csvPath string) error 21 | 22 | Query(table string, conditions map[string]interface{}, dest interface{}) error 23 | QueryStockData(symbol string, startDate, endDate *time.Time) ([]model.StockData, error) 24 | GetLatestDate(tableName string, dateCol string) (time.Time, error) 25 | GetAllSymbols() ([]string, error) 26 | } 27 | -------------------------------------------------------------------------------- /cmd/common.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "path/filepath" 5 | "time" 6 | 7 | "github.com/jing2uo/tdx2db/utils" 8 | ) 9 | 10 | var Today = time.Now().Truncate(24 * time.Hour) 11 | 12 | var TempDir, _ = utils.GetCacheDir() 13 | var VipdocDir = filepath.Join(TempDir, "vipdoc") 14 | var StockDailyCSV = filepath.Join(TempDir, "stock.csv") 15 | var Stock1MinCSV = filepath.Join(TempDir, "1min.csv") 16 | var Stock5MinCSV = filepath.Join(TempDir, "5min.csv") 17 | 18 | var ValidPrefixes = []string{ 19 | "sz30", // 创业板 20 | "sz00", // 深证主板 21 | "sh60", // 上证主板 22 | "sh68", // 科创板 23 | "bj920", // 北证 24 | "sh000300", // 沪深300 25 | "sh000905", // 中证500 26 | "sh000852", // 中证1000 27 | "sh000001", // 上证指数 28 | "sz399001", // 深证指数 29 | "sz399006", // 创业板指 30 | "sh000680", // 科创综指 31 | "bj899050", // 北证50 32 | "sh880", // 通达信概念、风格板块 33 | "sh881", // 通达信行业 34 | } 35 | -------------------------------------------------------------------------------- /model/views.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "sync" 4 | 5 | type ViewID string 6 | 7 | var ( 8 | viewRegistry []ViewID 9 | viewRegistryMu sync.Mutex 10 | ) 11 | 12 | func DefineView(name string) ViewID { 13 | viewRegistryMu.Lock() 14 | defer viewRegistryMu.Unlock() 15 | 16 | id := ViewID(name) 17 | viewRegistry = append(viewRegistry, id) 18 | return id 19 | } 20 | 21 | func AllViews() []ViewID { 22 | viewRegistryMu.Lock() 23 | defer viewRegistryMu.Unlock() 24 | 25 | result := make([]ViewID, len(viewRegistry)) 26 | copy(result, viewRegistry) 27 | return result 28 | } 29 | 30 | // --- 定义视图 --- 31 | 32 | var ( 33 | ViewTurnover = DefineView("v_turnover") 34 | ViewDailyQFQ = DefineView("v_qfq_daily") 35 | ViewDailyHFQ = DefineView("v_hfq_daily") 36 | View1MinQFQ = DefineView("v_qfq_1min") 37 | View1MinHFQ = DefineView("v_hfq_1min") 38 | View5MinQFQ = DefineView("v_qfq_5min") 39 | View5MinHFQ = DefineView("v_hfq_5min") 40 | ) 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Komh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /utils/unzip.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "archive/zip" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | func UnzipFile(zipPath, targetPath string) error { 12 | r, err := zip.OpenReader(zipPath) 13 | if err != nil { 14 | return err 15 | } 16 | defer r.Close() 17 | 18 | for _, f := range r.File { 19 | // 跳过 .zip 文件 20 | if strings.HasSuffix(strings.ToLower(f.Name), ".zip") { 21 | continue 22 | } 23 | 24 | rc, err := f.Open() 25 | if err != nil { 26 | return err 27 | } 28 | defer rc.Close() 29 | 30 | path := filepath.Join(targetPath, f.Name) 31 | 32 | if f.FileInfo().IsDir() { 33 | // 创建目录 34 | err = os.MkdirAll(path, f.Mode()) 35 | if err != nil { 36 | return err 37 | } 38 | } else { 39 | // 确保文件所在目录存在 40 | if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { 41 | return err 42 | } 43 | 44 | // 创建目标文件 45 | outFile, err := os.Create(path) 46 | if err != nil { 47 | return err 48 | } 49 | defer outFile.Close() 50 | 51 | // 复制文件内容 52 | _, err = io.Copy(outFile, rc) 53 | if err != nil { 54 | return err 55 | } 56 | } 57 | } 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /cmd/init.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | _ "github.com/duckdb/duckdb-go/v2" 7 | "github.com/jing2uo/tdx2db/database" 8 | "github.com/jing2uo/tdx2db/tdx" 9 | "github.com/jing2uo/tdx2db/utils" 10 | ) 11 | 12 | func Init(dbURI, dayFileDir string) error { 13 | db, err := database.NewDB(dbURI) 14 | if err != nil { 15 | return fmt.Errorf("failed to create database driver: %w", err) 16 | } 17 | 18 | if err := db.Connect(); err != nil { 19 | return fmt.Errorf("failed to connect to database: %w", err) 20 | } 21 | defer db.Close() 22 | 23 | if err := db.InitSchema(); err != nil { 24 | return fmt.Errorf("failed to initialize schema: %w", err) 25 | } 26 | 27 | fmt.Printf("📦 开始处理日线目录: %s\n", dayFileDir) 28 | err = utils.CheckDirectory(dayFileDir) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | fmt.Println("🐢 开始转换日线数据") 34 | _, err = tdx.ConvertFilesToCSV(dayFileDir, ValidPrefixes, StockDailyCSV, ".day") 35 | if err != nil { 36 | return fmt.Errorf("failed to convert day files to csv: %w", err) 37 | } 38 | fmt.Println("🔥 转换完成") 39 | 40 | if err := db.ImportDailyStocks(StockDailyCSV); err != nil { 41 | return fmt.Errorf("failed to import stock csv: %w", err) 42 | } 43 | 44 | fmt.Println("🚀 股票数据导入成功") 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release with GoReleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" # 触发条件：推送到以 v 开头的标签（如 v1.0.0） 7 | 8 | permissions: 9 | contents: write 10 | packages: write # 必须：发布到 ghcr.io 11 | 12 | jobs: 13 | release: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 # 获取完整历史以生成 changelog 20 | 21 | - name: Install unrar 22 | run: sudo apt-get update && sudo apt-get install -y unrar 23 | 24 | - name: Set up Go 25 | uses: actions/setup-go@v5 26 | with: 27 | go-version: "1.25" # 替换为你使用的 Go 版本 28 | 29 | - name: Log in to GitHub Container Registry 30 | uses: docker/login-action@v3 31 | with: 32 | registry: ghcr.io 33 | username: ${{ github.actor }} 34 | password: ${{ secrets.GITHUB_TOKEN }} 35 | 36 | - name: Set up Docker Buildx 37 | uses: docker/setup-buildx-action@v2 38 | 39 | - name: Run GoReleaser 40 | uses: goreleaser/goreleaser-action@v6 41 | with: 42 | version: latest 43 | args: release --clean 44 | env: 45 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 46 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # .goreleaser.yaml 2 | version: 2 3 | 4 | before: 5 | hooks: 6 | - make prepare 7 | 8 | builds: 9 | - main: ./main.go 10 | binary: tdx2db 11 | goos: 12 | - linux 13 | goarch: 14 | - amd64 15 | ldflags: 16 | - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} 17 | 18 | archives: 19 | - formats: [tar.gz] 20 | # this name template makes the OS and Arch compatible with the results of `uname`. 21 | name_template: >- 22 | {{ .ProjectName }}_ 23 | {{- title .Os }}_ 24 | {{- if eq .Arch "amd64" }}x86_64 25 | {{- else if eq .Arch "386" }}i386 26 | {{- else }}{{ .Arch }}{{ end }} 27 | {{- if .Arm }}v{{ .Arm }}{{ end }} 28 | 29 | release: 30 | github: 31 | owner: jing2uo 32 | name: tdx2db 33 | 34 | changelog: 35 | sort: asc 36 | filters: 37 | exclude: 38 | - "^docs:" 39 | - "^test:" 40 | - "^chore:" 41 | 42 | dockers_v2: 43 | - id: ghcr 44 | dockerfile: ./Containerfile 45 | platforms: 46 | - linux/amd64 47 | images: 48 | - "ghcr.io/jing2uo/tdx2db" 49 | tags: 50 | - "{{ .Version }}" 51 | - "latest" 52 | labels: 53 | "org.opencontainers.image.title": "tdx2db" 54 | "org.opencontainers.image.description": "Convert and import TDX data into DuckDB" 55 | "org.opencontainers.image.created": "{{ .Date }}" 56 | "org.opencontainers.image.revision": "{{ .FullCommit }}" 57 | "org.opencontainers.image.version": "{{ .Version }}" 58 | "org.opencontainers.image.source": "{{ .GitURL }}" 59 | -------------------------------------------------------------------------------- /utils/iocheck.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func CheckDirectory(path string) error { 9 | fileInfo, err := os.Stat(path) 10 | if os.IsNotExist(err) { 11 | return fmt.Errorf("the directory does not exist: %s", path) 12 | } 13 | if err != nil { 14 | return fmt.Errorf("error checking %s: %w", path, err) 15 | } 16 | if !fileInfo.IsDir() { 17 | return fmt.Errorf("the specified path %s is not a directory", path) 18 | } 19 | return nil 20 | } 21 | 22 | func CheckFile(path string) error { 23 | fileInfo, err := os.Stat(path) 24 | if err != nil { 25 | if os.IsNotExist(err) { 26 | return fmt.Errorf("the file does not exist: %s", path) 27 | } 28 | return err 29 | } 30 | if !fileInfo.Mode().IsRegular() { 31 | return fmt.Errorf("the specified path %s is not a file", path) 32 | } 33 | file, err := os.Open(path) 34 | if err != nil { 35 | if os.IsPermission(err) { 36 | return fmt.Errorf("could not read the file: %s", err) 37 | } 38 | return err 39 | } 40 | file.Close() 41 | return nil 42 | } 43 | 44 | func CheckOutputDir(path string) error { 45 | fileInfo, err := os.Stat(path) 46 | if os.IsNotExist(err) { 47 | if err := os.MkdirAll(path, 0755); err != nil { 48 | return fmt.Errorf("could not create output directory %s: %w", path, err) 49 | } 50 | return nil 51 | } 52 | if err != nil { 53 | return fmt.Errorf("could not access output directory %s: %w", path, err) 54 | } 55 | if !fileInfo.IsDir() { 56 | return fmt.Errorf("the specified output path is not a directory: %s", path) 57 | } 58 | 59 | tmpFile, err := os.CreateTemp(path, "test-") 60 | if err != nil { 61 | return fmt.Errorf("output directory %s is not writable: %w", path, err) 62 | } 63 | tmpFile.Close() 64 | os.Remove(tmpFile.Name()) 65 | 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | # Configuration 2 | TDX_URL := https://www.tdx.com.cn/products/autoup/cyb/datatool.rar 3 | TMP_DIR := .tmp 4 | RAR_FILE := $(TMP_DIR)/datatool.rar 5 | EXTRACT_DIR := $(TMP_DIR)/extracted 6 | TDX_EMBED_DIR := tdx/embed 7 | BIN_NAME := tdx2db 8 | INSTALL_DIR := /usr/local/bin 9 | LOCAL_BIN := $(HOME)/.local/bin 10 | 11 | .PHONY: all build check-unrar download extract move_datatool clean sudo-install user-install 12 | 13 | all: build 14 | 15 | build: check-unrar download extract move_datatool clean-tmp 16 | @echo "Building Go binary..." 17 | go build -o $(BIN_NAME) 18 | 19 | prepare: check-unrar download extract move_datatool 20 | @echo "Prepare datatool..." 21 | 22 | sudo-install: build 23 | @echo "Installing system-wide (requires sudo)" 24 | sudo mkdir -p $(INSTALL_DIR) 25 | sudo cp $(BIN_NAME) $(INSTALL_DIR)/ 26 | @echo "Installed to $(INSTALL_DIR)/$(BIN_NAME)" 27 | 28 | user-install: build 29 | @echo "Installing to user directory" 30 | mkdir -p $(LOCAL_BIN) 31 | cp $(BIN_NAME) $(LOCAL_BIN)/ 32 | @echo "Installed to $(LOCAL_BIN)/$(BIN_NAME)" 33 | @echo "NOTE: Ensure $(LOCAL_BIN) is in your PATH" 34 | 35 | check-unrar: 36 | @command -v unrar >/dev/null 2>&1 || { echo >&2 "Error: unrar required..."; exit 1; } 37 | 38 | download: 39 | @echo "Downloading TDX data tool..." 40 | mkdir -p $(TMP_DIR) 41 | curl -s -L -o $(RAR_FILE) $(TDX_URL) || (echo "Download failed"; exit 1) 42 | 43 | extract: 44 | @echo "Extracting RAR archive..." 45 | mkdir -p $(EXTRACT_DIR) 46 | unrar x -o+ $(RAR_FILE) $(EXTRACT_DIR)/ > /dev/null 47 | 48 | move_datatool: 49 | @echo "Moving data tool to embed directory..." 50 | mkdir -p $(TDX_EMBED_DIR) 51 | cp $(EXTRACT_DIR)/v4/datatool $(TDX_EMBED_DIR)/ 52 | 53 | clean-tmp: 54 | @echo "Cleaning temporary files..." 55 | rm -rf $(TMP_DIR) 56 | 57 | clean: 58 | @echo "Full cleanup..." 59 | rm -rf $(TMP_DIR) 60 | rm -rf $(TDX_EMBED_DIR)/datatool 61 | rm -f $(BIN_NAME) 62 | -------------------------------------------------------------------------------- /tdx/datatool.go: -------------------------------------------------------------------------------- 1 | package tdx 2 | 3 | import ( 4 | "embed" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "runtime" 11 | "time" 12 | ) 13 | 14 | //go:embed embed/* 15 | var embedFS embed.FS 16 | var startDateStr = "19901201" 17 | 18 | //datatool [day,tick,min] create 19901201 20250610 19 | 20 | func DatatoolCreate(cacheDir, subCommand string, endDate time.Time) error { 21 | switch subCommand { 22 | case "day", "min", "tick": 23 | // 24 | default: 25 | return errors.New("unsupported datatool subcommand: " + subCommand) 26 | } 27 | 28 | toolPath, err := extractDatatool(cacheDir) 29 | if err != nil { 30 | return fmt.Errorf("failed to extract datatool: %w", err) 31 | } 32 | 33 | endDateStr := endDate.Format("20060102") 34 | 35 | cmd := exec.Command(toolPath, subCommand, "create", startDateStr, endDateStr) 36 | cmd.Dir = cacheDir 37 | if err := cmd.Run(); err != nil { 38 | return fmt.Errorf("failed to execute datatool command: %w", err) 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func extractDatatool(cacheDir string) (string, error) { 45 | toolPath, err := extractFileFromEmbed(cacheDir, "embed/datatool") 46 | if err != nil { 47 | return "", fmt.Errorf("failed to extract binary: %w", err) 48 | } 49 | 50 | if _, err := extractFileFromEmbed(cacheDir, "embed/datatool.ini"); err != nil { 51 | return "", fmt.Errorf("failed to extract config: %w", err) 52 | } 53 | 54 | cmd := exec.Command(toolPath, "-h") 55 | cmd.Dir = cacheDir 56 | if err := cmd.Run(); err != nil { 57 | return "", fmt.Errorf("failed to execute datatool: %w", err) 58 | } 59 | 60 | return toolPath, nil 61 | } 62 | 63 | func extractFileFromEmbed(cacheDir string, srcPath string) (string, error) { 64 | destFileName := filepath.Base(srcPath) 65 | destPath := filepath.Join(cacheDir, destFileName) 66 | 67 | data, err := embedFS.ReadFile(srcPath) 68 | if err != nil { 69 | return "", fmt.Errorf("failed to read embedded file %s: %w", srcPath, err) 70 | } 71 | 72 | if err := os.WriteFile(destPath, data, 0755); err != nil { 73 | return "", fmt.Errorf("failed to write file %s: %w", destPath, err) 74 | } 75 | 76 | if runtime.GOOS != "windows" { 77 | if err := os.Chmod(destPath, 0755); err != nil { 78 | return "", fmt.Errorf("failed to set file permissions for %s: %w", destPath, err) 79 | } 80 | } 81 | 82 | return destPath, nil 83 | } 84 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jing2uo/tdx2db 2 | 3 | go 1.25.3 4 | 5 | require ( 6 | github.com/ClickHouse/clickhouse-go/v2 v2.41.0 7 | github.com/duckdb/duckdb-go/v2 v2.5.0 8 | github.com/jmoiron/sqlx v1.4.0 9 | github.com/spf13/cobra v1.10.1 10 | ) 11 | 12 | require ( 13 | github.com/ClickHouse/ch-go v0.69.0 // indirect 14 | github.com/andybalholm/brotli v1.2.0 // indirect 15 | github.com/apache/arrow-go/v18 v18.4.1 // indirect 16 | github.com/duckdb/duckdb-go-bindings v0.1.21 // indirect 17 | github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21 // indirect 18 | github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21 // indirect 19 | github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21 // indirect 20 | github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21 // indirect 21 | github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21 // indirect 22 | github.com/duckdb/duckdb-go/arrowmapping v0.0.22 // indirect 23 | github.com/duckdb/duckdb-go/mapping v0.0.22 // indirect 24 | github.com/go-faster/city v1.0.1 // indirect 25 | github.com/go-faster/errors v0.7.1 // indirect 26 | github.com/go-viper/mapstructure/v2 v2.4.0 // indirect 27 | github.com/goccy/go-json v0.10.5 // indirect 28 | github.com/google/flatbuffers v25.9.23+incompatible // indirect 29 | github.com/google/uuid v1.6.0 // indirect 30 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 31 | github.com/klauspost/compress v1.18.1 // indirect 32 | github.com/klauspost/cpuid/v2 v2.3.0 // indirect 33 | github.com/paulmach/orb v0.12.0 // indirect 34 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 35 | github.com/rogpeppe/go-internal v1.14.1 // indirect 36 | github.com/segmentio/asm v1.2.1 // indirect 37 | github.com/shopspring/decimal v1.4.0 // indirect 38 | github.com/spf13/pflag v1.0.10 // indirect 39 | github.com/zeebo/xxh3 v1.0.2 // indirect 40 | go.opentelemetry.io/otel v1.38.0 // indirect 41 | go.opentelemetry.io/otel/trace v1.38.0 // indirect 42 | go.yaml.in/yaml/v3 v3.0.4 // indirect 43 | golang.org/x/exp v0.0.0-20251017212417-90e834f514db // indirect 44 | golang.org/x/mod v0.29.0 // indirect 45 | golang.org/x/sync v0.17.0 // indirect 46 | golang.org/x/sys v0.38.0 // indirect 47 | golang.org/x/telemetry v0.0.0-20251022145735-5be28d707443 // indirect 48 | golang.org/x/tools v0.38.0 // indirect 49 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /database/duckdb/driver.go: -------------------------------------------------------------------------------- 1 | package duckdb 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | _ "github.com/duckdb/duckdb-go/v2" 11 | "github.com/jing2uo/tdx2db/model" 12 | "github.com/jmoiron/sqlx" 13 | ) 14 | 15 | type DuckDBDriver struct { 16 | dsn string 17 | db *sqlx.DB 18 | viewImpls map[model.ViewID]func() error 19 | } 20 | 21 | func NewDuckDBDriver(u *url.URL) (*DuckDBDriver, error) { 22 | 23 | var dbPath string 24 | 25 | if u.Host == "." || u.Host == ".." { 26 | // 处理 ./ 或 ../ 27 | dbPath = u.Host + u.Path 28 | } else if u.Host != "" { 29 | // 处理 duckdb://filename.db 30 | dbPath = u.Host + u.Path 31 | } else { 32 | // 处理 duckdb:///absolute/path.db 33 | dbPath = u.Path 34 | } 35 | 36 | dbPath = strings.TrimSpace(dbPath) 37 | 38 | // 禁止内存模式 39 | if dbPath == "" || dbPath == ":memory:" { 40 | return nil, fmt.Errorf("duckdb driver: memory mode is not allowed, please provide a file path (e.g. duckdb://data.db)") 41 | } 42 | // home 目录展开 43 | if strings.HasPrefix(dbPath, "~/") || strings.HasPrefix(dbPath, "~\\") || dbPath == "~" { 44 | homeDir, err := os.UserHomeDir() 45 | if err != nil { 46 | return nil, fmt.Errorf("failed to get user home directory: %w", err) 47 | } 48 | 49 | if dbPath == "~" { 50 | dbPath = homeDir 51 | } else { 52 | dbPath = filepath.Join(homeDir, dbPath[1:]) 53 | } 54 | } 55 | 56 | // 3. 准备目录 57 | dir := filepath.Dir(dbPath) 58 | if dir != "." && dir != "/" { 59 | if _, err := os.Stat(dir); os.IsNotExist(err) { 60 | if err := os.MkdirAll(dir, 0755); err != nil { 61 | return nil, fmt.Errorf("failed to create directory for duckdb file: %w", err) 62 | } 63 | } 64 | } 65 | 66 | if u.RawQuery != "" { 67 | dbPath = fmt.Sprintf("%s?%s", dbPath, u.RawQuery) 68 | } 69 | 70 | return &DuckDBDriver{ 71 | dsn: dbPath, 72 | viewImpls: make(map[model.ViewID]func() error), 73 | }, nil 74 | } 75 | 76 | func (d *DuckDBDriver) Connect() error { 77 | db, err := sqlx.Open("duckdb", d.dsn) 78 | if err != nil { 79 | return err 80 | } 81 | db.SetMaxOpenConns(1) 82 | db.SetMaxIdleConns(1) 83 | db.SetConnMaxLifetime(0) 84 | 85 | if err := db.Ping(); err != nil { 86 | _ = db.Close() 87 | return fmt.Errorf("duckdb ping failed (check file permissions): %w", err) 88 | } 89 | 90 | d.db = db 91 | return nil 92 | } 93 | 94 | func (d *DuckDBDriver) Close() error { 95 | return d.db.Close() 96 | } 97 | 98 | func (d *DuckDBDriver) InitSchema() error { 99 | tables := model.AllTables() 100 | 101 | for _, t := range tables { 102 | if err := d.createTableInternal(t); err != nil { 103 | return fmt.Errorf("failed to create table %s: %w", t.TableName, err) 104 | } 105 | } 106 | d.registerViews() 107 | for _, viewID := range model.AllViews() { 108 | 109 | implFunc, exists := d.viewImpls[viewID] 110 | if !exists { 111 | return fmt.Errorf("[DuckDB] Missing implementation for required view: %s", viewID) 112 | } 113 | 114 | if err := implFunc(); err != nil { 115 | return fmt.Errorf("failed to create view %s: %w", viewID, err) 116 | } 117 | } 118 | 119 | return nil 120 | } 121 | -------------------------------------------------------------------------------- /database/duckdb/dml.go: -------------------------------------------------------------------------------- 1 | package duckdb 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/jing2uo/tdx2db/model" 10 | ) 11 | 12 | func (d *DuckDBDriver) importCSV(meta *model.TableMeta, csvPath string) error { 13 | var colMaps []string 14 | for _, col := range meta.Columns { 15 | duckType := d.mapType(col.Type) 16 | colMaps = append(colMaps, fmt.Sprintf("'%s': '%s'", col.Name, duckType)) 17 | } 18 | 19 | columnsStr := strings.Join(colMaps, ", ") 20 | 21 | query := fmt.Sprintf(` 22 | INSERT INTO %s 23 | SELECT * FROM read_csv('%s', 24 | header=true, 25 | columns={%s}, 26 | dateformat='%%Y-%%m-%%d', 27 | timestampformat='%%Y-%%m-%%d %%H:%%M' 28 | ) 29 | `, meta.TableName, csvPath, columnsStr) 30 | 31 | _, err := d.db.Exec(query) 32 | return err 33 | } 34 | 35 | func (d *DuckDBDriver) truncateTable(meta *model.TableMeta) error { 36 | 37 | query := fmt.Sprintf("TRUNCATE TABLE IF EXISTS %s", meta.TableName) 38 | 39 | _, err := d.db.Exec(query) 40 | if err != nil { 41 | return fmt.Errorf("duckdb truncate failed: %w", err) 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func (d *DuckDBDriver) ImportDailyStocks(path string) error { 48 | return d.importCSV(model.TableStocksDaily, path) 49 | } 50 | 51 | func (d *DuckDBDriver) Import1MinStocks(path string) error { 52 | return d.importCSV(model.TableStocks1Min, path) 53 | } 54 | 55 | func (d *DuckDBDriver) Import5MinStocks(path string) error { 56 | return d.importCSV(model.TableStocks5Min, path) 57 | } 58 | 59 | func (d *DuckDBDriver) ImportGBBQ(path string) error { 60 | d.truncateTable(model.TableGbbq) 61 | return d.importCSV(model.TableGbbq, path) 62 | } 63 | 64 | func (d *DuckDBDriver) ImportXDXR(path string) error { 65 | d.truncateTable(model.TableXdxr) 66 | return d.importCSV(model.TableXdxr, path) 67 | } 68 | 69 | func (d *DuckDBDriver) ImportAdjustFactors(path string) error { 70 | d.truncateTable(model.TableAdjustFactor) 71 | return d.importCSV(model.TableAdjustFactor, path) 72 | } 73 | 74 | func (d *DuckDBDriver) Query(table string, conditions map[string]interface{}, dest interface{}) error { 75 | query := fmt.Sprintf("SELECT * FROM %s", table) 76 | args := []interface{}{} 77 | if len(conditions) > 0 { 78 | whereParts := []string{} 79 | i := 1 80 | for k, v := range conditions { 81 | whereParts = append(whereParts, fmt.Sprintf("%s = $%d", k, i)) 82 | args = append(args, v) 83 | i++ 84 | } 85 | query += " WHERE " + strings.Join(whereParts, " AND ") 86 | } 87 | 88 | return d.db.Select(dest, query, args...) 89 | } 90 | 91 | func (d *DuckDBDriver) GetLatestDate(tableName string, dateCol string) (time.Time, error) { 92 | query := fmt.Sprintf("SELECT max(%s) AS latest FROM %s", dateCol, tableName) 93 | 94 | var latest sql.NullTime 95 | err := d.db.Get(&latest, query) 96 | if err != nil { 97 | return time.Time{}, err 98 | } 99 | 100 | if !latest.Valid { 101 | return time.Time{}, nil 102 | } 103 | 104 | return latest.Time, nil 105 | } 106 | 107 | func (d *DuckDBDriver) GetAllSymbols() ([]string, error) { 108 | query := fmt.Sprintf("SELECT DISTINCT symbol FROM %s", model.TableStocksDaily.TableName) 109 | 110 | var symbols []string 111 | err := d.db.Select(&symbols, query) 112 | if err != nil { 113 | return nil, fmt.Errorf("failed to query symbols: %w", err) 114 | } 115 | 116 | return symbols, nil 117 | } 118 | 119 | func (d *DuckDBDriver) QueryStockData(symbol string, startDate, endDate *time.Time) ([]model.StockData, error) { 120 | query := fmt.Sprintf( 121 | "SELECT symbol, open, high, low, close, date FROM %s WHERE symbol = ? ORDER BY date ASC", 122 | model.TableStocksDaily.TableName, 123 | ) 124 | 125 | args := []interface{}{symbol} 126 | 127 | if startDate != nil { 128 | query += " AND date >= ?" 129 | args = append(args, *startDate) 130 | } 131 | if endDate != nil { 132 | query += " AND date <= ?" 133 | args = append(args, *endDate) 134 | } 135 | 136 | var results []model.StockData 137 | err := d.db.Select(&results, query, args...) 138 | if err != nil { 139 | return nil, fmt.Errorf("failed to query stocks: %w", err) 140 | } 141 | 142 | return results, nil 143 | } 144 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/jing2uo/tdx2db/cmd" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | const dbURIInfo = "数据库连接信息" 13 | const dbURIHelp = ` 14 | 15 | Database URI: 16 | ClickHouse: clickhouse//:[user[:password]@][host][:port][/database][?http_port=p&] 17 | DuckDB: duckdb://[path]` 18 | 19 | const dayFileInfo = "通达信日线文件目录" 20 | const minLineInfo = `导入分时数据（可选） 21 | 1 导入1分钟分时数据 22 | 5 导入5分钟分时数据 23 | 1,5 导入两种` 24 | 25 | const convertHelp = ` 26 | 27 | Type & Input: 28 | -t day 转换日线文件 -i 包含 .day 的目录 29 | -t 1min 转换 1 分钟分时 -i 包含 .1 的目录 30 | -t 5min 转换 5 分钟分时 -i 包含 .05 的目录 31 | -t tic4 转换四代分笔 -i 四代 TIC 压缩文件 32 | -t day4 转换四代日线 -i 四代行情压缩文件` 33 | 34 | func main() { 35 | var rootCmd = &cobra.Command{ 36 | Use: "tdx2db", 37 | Short: "Load TDX Data to DuckDB", 38 | SilenceErrors: true, 39 | } 40 | 41 | var ( 42 | dbURI string 43 | dayFileDir string 44 | minline string 45 | 46 | // Convert 47 | inputType string 48 | inputPath string 49 | outputPath string 50 | ) 51 | 52 | var initCmd = &cobra.Command{ 53 | Use: "init", 54 | Short: "Fully import stocks data from TDX", 55 | Example: ` tdx2db init --dburi 'clickhouse://localhost' --dayfiledir /path/to/vipdoc/ 56 | tdx2db init --dburi 'duckdb://./tdx.db' --dayfiledir /path/to/vipdoc/` + dbURIHelp, 57 | RunE: func(c *cobra.Command, args []string) error { 58 | return cmd.Init(dbURI, dayFileDir) 59 | }, 60 | } 61 | 62 | var cronCmd = &cobra.Command{ 63 | Use: "cron", 64 | Short: "Cron for update data and calc factor", 65 | Example: ` tdx2db cron --dburi 'clickhouse://localhost' --minline 1,5 66 | tdx2db cron --dburi 'duckdb://./tdx.db'` + dbURIHelp, 67 | RunE: func(c *cobra.Command, args []string) error { 68 | if c.Flags().Changed("minline") { 69 | valid := map[string]bool{"1": true, "5": true, "1,5": true, "5,1": true} 70 | if !valid[minline] { 71 | return fmt.Errorf("--minline 仅支持 '1'、'5'、'1,5', 传入: %s", minline) 72 | } 73 | } 74 | return cmd.Cron(dbURI, minline) 75 | }, 76 | } 77 | 78 | var convertCmd = &cobra.Command{ 79 | Use: "convert", 80 | Short: "Convert TDX data to CSV", 81 | Example: ` tdx2db convert -t day -i /path/to/vipdoc/ -o ./ 82 | tdx2db convert -t day4 -i /path/to/20251212.zip -o ./` + convertHelp, 83 | RunE: func(c *cobra.Command, args []string) error { 84 | opts := cmd.ConvertOptions{ 85 | InputPath: inputPath, 86 | OutputPath: outputPath, 87 | } 88 | 89 | switch strings.ToLower(inputType) { 90 | case "day": 91 | opts.InputType = cmd.DayFileDir 92 | case "1min": 93 | opts.InputType = cmd.Min1FileDir 94 | case "5min": 95 | opts.InputType = cmd.Min5FileDir 96 | case "tic4": 97 | opts.InputType = cmd.TicZip 98 | case "day4": 99 | opts.InputType = cmd.DayZip 100 | default: 101 | return fmt.Errorf("未知的类型: %s%s", inputType, convertHelp) 102 | } 103 | 104 | return cmd.Convert(opts) 105 | }, 106 | } 107 | 108 | // Init Flags 109 | initCmd.Flags().StringVar(&dbURI, "dburi", "", dbURIInfo) 110 | initCmd.Flags().StringVar(&dayFileDir, "dayfiledir", "", dayFileInfo) 111 | initCmd.MarkFlagRequired("dburi") 112 | initCmd.MarkFlagRequired("dayfiledir") 113 | 114 | // Cron Flags 115 | cronCmd.Flags().StringVar(&dbURI, "dburi", "", dbURIInfo) 116 | cronCmd.MarkFlagRequired("dburi") 117 | cronCmd.Flags().StringVar(&minline, "minline", "", minLineInfo) 118 | 119 | // Convert Flags 120 | convertCmd.Flags().StringVarP(&inputType, "type", "t", "", "转换类型") 121 | convertCmd.Flags().StringVarP(&inputPath, "input", "i", "", "输入文件或目录路径") 122 | convertCmd.Flags().StringVarP(&outputPath, "output", "o", "", "CSV 文件输出目录") 123 | convertCmd.MarkFlagRequired("type") 124 | convertCmd.MarkFlagRequired("input") 125 | convertCmd.MarkFlagRequired("output") 126 | 127 | rootCmd.AddCommand(initCmd) 128 | rootCmd.AddCommand(cronCmd) 129 | rootCmd.AddCommand(convertCmd) 130 | 131 | cobra.OnFinalize(func() { 132 | os.RemoveAll(cmd.TempDir) 133 | }) 134 | 135 | if err := rootCmd.Execute(); err != nil { 136 | fmt.Fprintf(os.Stderr, "🛑 错误: %v\n", err) 137 | os.Exit(1) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /utils/csv_write.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "os" 7 | "reflect" 8 | "time" 9 | ) 10 | 11 | // CSVWriter 通用 CSV 写入器 12 | type CSVWriter[T any] struct { 13 | file *os.File 14 | writer *csv.Writer 15 | headerWritten bool 16 | columns []columnInfo 17 | } 18 | 19 | type columnInfo struct { 20 | Index int // 字段索引 21 | HeaderName string // CSV 表头 (来自 col 标签) 22 | IsTime bool // 字段本身是否是 time.Time 23 | IsPtrTime bool // 字段本身是否是 *time.Time 24 | IsDateType bool // 是否标记了 type:"date" 25 | } 26 | 27 | // NewCSVWriter 初始化 28 | func NewCSVWriter[T any](filename string) (*CSVWriter[T], error) { 29 | // 1. 创建文件 30 | f, err := os.Create(filename) 31 | if err != nil { 32 | return nil, fmt.Errorf("failed to create file: %w", err) 33 | } 34 | 35 | // 2. 初始化 CSV Writer 36 | w := csv.NewWriter(f) 37 | 38 | // 3. 解析结构体 Tag (只需做一次) 39 | cols, err := analyzeStructTags[T]() 40 | if err != nil { 41 | f.Close() 42 | return nil, err 43 | } 44 | 45 | return &CSVWriter[T]{ 46 | file: f, 47 | writer: w, 48 | columns: cols, 49 | }, nil 50 | } 51 | 52 | // analyzeStructTags 解析 col 和 type 标签 53 | func analyzeStructTags[T any]() ([]columnInfo, error) { 54 | var t T 55 | typ := reflect.TypeOf(t) 56 | if typ.Kind() == reflect.Ptr { 57 | typ = typ.Elem() 58 | } 59 | if typ.Kind() != reflect.Struct { 60 | return nil, fmt.Errorf("generic type T must be a struct") 61 | } 62 | 63 | var cols []columnInfo 64 | for i := 0; i < typ.NumField(); i++ { 65 | field := typ.Field(i) 66 | 67 | // 1. 获取 col 标签作为表头 68 | colTag := field.Tag.Get("col") 69 | if colTag == "" { 70 | // 如果没有 col 标签，可以选择跳过，或者用字段名，这里假设用字段名兜底 71 | colTag = field.Name 72 | } 73 | 74 | // 2. 获取 type 标签 75 | typeTag := field.Tag.Get("type") 76 | isDateType := (typeTag == "date") // 标记是否需要转 yyyy-mm-dd 77 | 78 | // 3. 判断是否为 Time 类型 79 | isTime := field.Type == reflect.TypeOf(time.Time{}) 80 | isPtrTime := field.Type == reflect.TypeOf((*time.Time)(nil)) 81 | 82 | cols = append(cols, columnInfo{ 83 | Index: i, 84 | HeaderName: colTag, 85 | IsTime: isTime, 86 | IsPtrTime: isPtrTime, 87 | IsDateType: isDateType, 88 | }) 89 | } 90 | return cols, nil 91 | } 92 | 93 | // Write 写入数据 94 | func (cw *CSVWriter[T]) Write(data []T) error { 95 | if len(data) == 0 { 96 | return nil 97 | } 98 | 99 | // 1. 写入表头 100 | if !cw.headerWritten { 101 | headers := make([]string, len(cw.columns)) 102 | for i, col := range cw.columns { 103 | headers[i] = col.HeaderName 104 | } 105 | if err := cw.writer.Write(headers); err != nil { 106 | return fmt.Errorf("failed to write header: %w", err) 107 | } 108 | cw.headerWritten = true 109 | } 110 | 111 | // 2. 写入数据行 112 | record := make([]string, len(cw.columns)) 113 | for _, item := range data { 114 | val := reflect.ValueOf(item) 115 | if val.Kind() == reflect.Ptr { 116 | val = val.Elem() 117 | } 118 | 119 | for i, col := range cw.columns { 120 | fieldVal := val.Field(col.Index) 121 | 122 | // --- 日期处理逻辑 --- 123 | if col.IsTime || col.IsPtrTime { 124 | var t time.Time 125 | isValid := false 126 | 127 | // 获取时间对象 128 | if col.IsTime { 129 | t = fieldVal.Interface().(time.Time) 130 | isValid = !t.IsZero() 131 | } else if !fieldVal.IsNil() { 132 | t = *fieldVal.Interface().(*time.Time) 133 | isValid = !t.IsZero() 134 | } 135 | 136 | if !isValid { 137 | record[i] = "" // 空时间或 nil 指针留空 138 | } else { 139 | // 核心判断：如果 type:"date"，用短格式；否则用默认长格式 140 | if col.IsDateType { 141 | record[i] = t.Format("2006-01-02") 142 | } else { 143 | record[i] = t.Format(time.RFC3339) // 或 "2006-01-02 15:04:05" 144 | } 145 | } 146 | continue 147 | } 148 | // ------------------ 149 | 150 | // 其他类型通用处理 151 | record[i] = fmt.Sprint(fieldVal.Interface()) 152 | } 153 | 154 | if err := cw.writer.Write(record); err != nil { 155 | return fmt.Errorf("failed to write record: %w", err) 156 | } 157 | } 158 | 159 | return nil 160 | } 161 | 162 | func (cw *CSVWriter[T]) Close() error { 163 | cw.writer.Flush() 164 | if err := cw.writer.Error(); err != nil { 165 | cw.file.Close() 166 | return fmt.Errorf("failed to flush: %w", err) 167 | } 168 | return cw.file.Close() 169 | } 170 | -------------------------------------------------------------------------------- /database/duckdb/ddl.go: -------------------------------------------------------------------------------- 1 | package duckdb 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/jing2uo/tdx2db/model" 8 | ) 9 | 10 | // mapType 将通用 DataType 转换为 DuckDB 的 SQL 类型 11 | func (d *DuckDBDriver) mapType(dt model.DataType) string { 12 | switch dt { 13 | case model.TypeString: 14 | return "VARCHAR" 15 | case model.TypeFloat64: 16 | return "DOUBLE" 17 | case model.TypeInt64: 18 | return "BIGINT" 19 | case model.TypeDate: 20 | return "DATE" 21 | case model.TypeDateTime: 22 | return "TIMESTAMP WITH TIME ZONE" 23 | default: 24 | return "VARCHAR" 25 | } 26 | } 27 | 28 | func (d *DuckDBDriver) createTableInternal(meta *model.TableMeta) error { 29 | var colDefs []string 30 | for _, col := range meta.Columns { 31 | // 1. 获取 DuckDB 具体的类型 32 | sqlType := d.mapType(col.Type) 33 | // 2. 拼接字段定义 34 | colDefs = append(colDefs, fmt.Sprintf("%s %s", col.Name, sqlType)) 35 | } 36 | 37 | query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", 38 | meta.TableName, strings.Join(colDefs, ", ")) 39 | 40 | _, err := d.db.Exec(query) 41 | if err != nil { 42 | return fmt.Errorf("failed to create table %s: %w", meta.TableName, err) 43 | } 44 | return nil 45 | } 46 | 47 | func (d *DuckDBDriver) registerViews() { 48 | // 换手率与市值视图 (ViewTurnover) 49 | d.viewImpls[model.ViewTurnover] = func() error { 50 | query := fmt.Sprintf(` 51 | CREATE OR REPLACE VIEW %s AS 52 | WITH gbbq_sorted AS ( 53 | SELECT 54 | date, 55 | symbol, 56 | post_float, 57 | post_total 58 | FROM %s 59 | WHERE category IN (2, 3, 5, 7, 8, 9, 10) 60 | ORDER BY symbol, date -- ASOF JOIN 要求右表必须排序 61 | ) 62 | SELECT 63 | r.date, 64 | r.symbol, 65 | CASE WHEN g.post_float > 0 THEN 66 | ROUND(r.volume / (g.post_float * 10000), 6) 67 | ELSE 0 END AS turnover, 68 | ROUND(g.post_float * 10000 * r.close, 2) AS float_mv, 69 | ROUND(g.post_total * 10000 * r.close, 2) AS total_mv 70 | FROM %s r 71 | ASOF JOIN gbbq_sorted g 72 | ON r.symbol = g.symbol 73 | AND r.date >= g.date -- 时间对齐 (找最近的股本) 74 | `, 75 | model.ViewTurnover, 76 | model.TableGbbq.TableName, 77 | model.TableStocksDaily.TableName) 78 | _, err := d.db.Exec(query) 79 | return err 80 | } 81 | 82 | // 通用复权视图构建函数 83 | createAdjustView := func(viewName model.ViewID, sourceTable string, factorCol string, isMin bool) error { 84 | var joinClause string 85 | timeCol := "date" 86 | 87 | if isMin { 88 | timeCol = "datetime" 89 | joinClause = fmt.Sprintf(` 90 | ASOF JOIN (SELECT * FROM %s ORDER BY symbol, date) f 91 | ON s.symbol = f.symbol AND s.datetime >= CAST(f.date AS TIMESTAMP) 92 | `, model.TableAdjustFactor.TableName) 93 | 94 | } else { 95 | // --- 日线逻辑 --- 96 | joinClause = fmt.Sprintf("LEFT JOIN %s f ON s.symbol = f.symbol AND s.date = f.date", model.TableAdjustFactor.TableName) 97 | } 98 | 99 | query := fmt.Sprintf(` 100 | CREATE OR REPLACE VIEW %s AS 101 | SELECT 102 | s.symbol, 103 | s.%s, -- date or datetime 104 | s.volume, 105 | s.amount, 106 | ROUND(s.open * f.%s, 2) AS open, 107 | ROUND(s.high * f.%s, 2) AS high, 108 | ROUND(s.low * f.%s, 2) AS low, 109 | ROUND(s.close * f.%s, 2) AS close 110 | FROM %s s 111 | %s -- Factor Join (Standard or ASOF) 112 | `, 113 | viewName, 114 | timeCol, 115 | factorCol, factorCol, factorCol, factorCol, 116 | sourceTable, 117 | joinClause, 118 | ) 119 | 120 | _, err := d.db.Exec(query) 121 | return err 122 | } 123 | 124 | // 3. 注册日线复权 125 | d.viewImpls[model.ViewDailyQFQ] = func() error { 126 | return createAdjustView(model.ViewDailyQFQ, model.TableStocksDaily.TableName, "qfq_factor", false) 127 | } 128 | d.viewImpls[model.ViewDailyHFQ] = func() error { 129 | return createAdjustView(model.ViewDailyHFQ, model.TableStocksDaily.TableName, "hfq_factor", false) 130 | } 131 | 132 | // 4. 注册分钟线复权 133 | d.viewImpls[model.View1MinQFQ] = func() error { 134 | return createAdjustView(model.View1MinQFQ, model.TableStocks1Min.TableName, "qfq_factor", true) 135 | } 136 | d.viewImpls[model.View1MinHFQ] = func() error { 137 | return createAdjustView(model.View1MinHFQ, model.TableStocks1Min.TableName, "hfq_factor", true) 138 | } 139 | d.viewImpls[model.View5MinQFQ] = func() error { 140 | return createAdjustView(model.View5MinQFQ, model.TableStocks5Min.TableName, "qfq_factor", true) 141 | } 142 | d.viewImpls[model.View5MinHFQ] = func() error { 143 | return createAdjustView(model.View5MinHFQ, model.TableStocks5Min.TableName, "hfq_factor", true) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /database/clickhouse/driver.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "strings" 10 | "time" 11 | 12 | _ "github.com/ClickHouse/clickhouse-go/v2" 13 | "github.com/jing2uo/tdx2db/model" 14 | "github.com/jmoiron/sqlx" 15 | ) 16 | 17 | type ClickHouseDriver struct { 18 | dsn string 19 | db *sqlx.DB 20 | database string 21 | 22 | // HTTP 23 | httpImportUrl string 24 | authUser string 25 | authPass string 26 | 27 | viewImpls map[model.ViewID]func() error 28 | } 29 | 30 | func NewClickHouseDriver(u *url.URL) (*ClickHouseDriver, error) { 31 | q := u.Query() 32 | 33 | // HTTP 端口 (默认 8123) 34 | httpPort := q.Get("http_port") 35 | if httpPort == "" { 36 | httpPort = "8123" 37 | } 38 | q.Del("http_port") 39 | 40 | // 2. Host 必填 41 | host := u.Hostname() 42 | if host == "" { 43 | return nil, fmt.Errorf("clickhouse host is required") 44 | } 45 | 46 | // 3. TCP 端口 (默认 9000) 47 | tcpPort := u.Port() 48 | if tcpPort == "" { 49 | tcpPort = "9000" 50 | } 51 | 52 | // 4. 处理 Database (默认 "default") 53 | database := strings.TrimPrefix(u.Path, "/") 54 | if database == "" { 55 | database = "default" 56 | } 57 | u.Path = "/" + database 58 | 59 | // 5. 处理 User (默认 "default") 60 | user := u.User.Username() 61 | if user == "" { 62 | user = "default" 63 | } 64 | 65 | // 6. 处理 Password 66 | pass, passSet := u.User.Password() 67 | 68 | // 生成 HTTP 导入用的 Base URL 69 | httpImportUrl := fmt.Sprintf("http://%s:%s", host, httpPort) 70 | 71 | // 更新 URL 对象以生成最终 DSN 72 | u.Host = fmt.Sprintf("%s:%s", host, tcpPort) 73 | 74 | // 根据是否显式设置了密码（包括空密码）来重建 UserInfo 75 | if passSet { 76 | u.User = url.UserPassword(user, pass) 77 | } else { 78 | u.User = url.User(user) 79 | } 80 | 81 | u.RawQuery = q.Encode() 82 | 83 | finalDSN := u.String() 84 | 85 | return &ClickHouseDriver{ 86 | dsn: finalDSN, 87 | httpImportUrl: httpImportUrl, 88 | authUser: user, 89 | authPass: pass, 90 | database: database, 91 | viewImpls: make(map[model.ViewID]func() error), 92 | }, nil 93 | } 94 | 95 | func (d *ClickHouseDriver) pingHTTP() error { 96 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 97 | defer cancel() 98 | 99 | req, err := http.NewRequestWithContext(ctx, "GET", d.httpImportUrl, nil) 100 | if err != nil { 101 | return fmt.Errorf("failed to create http request: %w", err) 102 | } 103 | 104 | q := req.URL.Query() 105 | q.Add("query", "SELECT 1") 106 | req.URL.RawQuery = q.Encode() 107 | 108 | if d.authUser != "" { 109 | req.SetBasicAuth(d.authUser, d.authPass) 110 | } 111 | 112 | resp, err := http.DefaultClient.Do(req) 113 | if err != nil { 114 | return fmt.Errorf("http unreachable (%s): %w", d.httpImportUrl, err) 115 | } 116 | defer resp.Body.Close() 117 | 118 | if resp.StatusCode != http.StatusOK { 119 | body, _ := io.ReadAll(resp.Body) 120 | return fmt.Errorf("http check failed (status %d): %s", resp.StatusCode, string(body)) 121 | } 122 | 123 | return nil 124 | } 125 | 126 | func (d *ClickHouseDriver) extractPort(u string) string { 127 | parsed, _ := url.Parse(u) 128 | return parsed.Port() 129 | } 130 | 131 | func (d *ClickHouseDriver) Connect() error { 132 | db, err := sqlx.Open("clickhouse", d.dsn) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | db.SetMaxOpenConns(20) 138 | db.SetMaxIdleConns(5) 139 | db.SetConnMaxLifetime(0) 140 | 141 | if err := db.Ping(); err != nil { 142 | return fmt.Errorf("clickhouse ping failed: %w", err) 143 | } 144 | 145 | if err := d.pingHTTP(); err != nil { 146 | _ = db.Close() 147 | return fmt.Errorf("tcp connected but http check (port %s) failed: %w", 148 | d.extractPort(d.httpImportUrl), err) 149 | } 150 | 151 | d.db = db 152 | return nil 153 | } 154 | 155 | func (d *ClickHouseDriver) Close() error { 156 | if d.db != nil { 157 | return d.db.Close() 158 | } 159 | return nil 160 | } 161 | 162 | func (d *ClickHouseDriver) InitSchema() error { 163 | tables := model.AllTables() 164 | 165 | for _, t := range tables { 166 | if err := d.createTableInternal(t); err != nil { 167 | return fmt.Errorf("failed to create table %s: %w", t.TableName, err) 168 | } 169 | } 170 | 171 | d.registerViews() 172 | for _, viewID := range model.AllViews() { 173 | implFunc, exists := d.viewImpls[viewID] 174 | if !exists { 175 | return fmt.Errorf("[ClickHouse] Missing implementation for view: %s", viewID) 176 | } 177 | if err := implFunc(); err != nil { 178 | return fmt.Errorf("failed to create view %s: %w", viewID, err) 179 | } 180 | } 181 | return nil 182 | } 183 | -------------------------------------------------------------------------------- /database/clickhouse/dml.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "time" 12 | 13 | "github.com/jing2uo/tdx2db/model" 14 | ) 15 | 16 | func (d *ClickHouseDriver) importCSV(meta *model.TableMeta, filePath string) error { 17 | file, err := os.Open(filePath) 18 | if err != nil { 19 | return err 20 | } 21 | defer file.Close() 22 | 23 | req, err := http.NewRequest("POST", d.httpImportUrl, file) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | req.Header.Set("Content-Type", "text/csv") 29 | 30 | // 设置参数 31 | q := req.URL.Query() 32 | 33 | if d.database != "" { 34 | q.Set("database", d.database) 35 | } 36 | 37 | q.Add("query", fmt.Sprintf("INSERT INTO %s FORMAT CSVWithNames", meta.TableName)) 38 | q.Add("date_time_input_format", "best_effort") 39 | 40 | req.URL.RawQuery = q.Encode() 41 | 42 | if d.authUser != "" { 43 | req.SetBasicAuth(d.authUser, d.authPass) 44 | } 45 | 46 | client := &http.Client{} 47 | resp, err := client.Do(req) 48 | if err != nil { 49 | return err 50 | } 51 | defer resp.Body.Close() 52 | 53 | if resp.StatusCode != http.StatusOK { 54 | bodyBytes, _ := io.ReadAll(resp.Body) 55 | errMsg := strings.TrimSpace(string(bodyBytes)) 56 | return fmt.Errorf("clickhouse insert failed (db: %s, status %d): %s", d.database, resp.StatusCode, errMsg) 57 | } 58 | 59 | return nil 60 | } 61 | 62 | func (d *ClickHouseDriver) truncateTable(meta *model.TableMeta) error { 63 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 64 | defer cancel() 65 | 66 | query := fmt.Sprintf("TRUNCATE TABLE IF EXISTS %s", meta.TableName) 67 | 68 | _, err := d.db.ExecContext(ctx, query) 69 | if err != nil { 70 | return fmt.Errorf("clickhouse truncate via tcp failed: %w", err) 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func (d *ClickHouseDriver) ImportDailyStocks(path string) error { 77 | return d.importCSV(model.TableStocksDaily, path) 78 | } 79 | 80 | func (d *ClickHouseDriver) Import1MinStocks(path string) error { 81 | return d.importCSV(model.TableStocks1Min, path) 82 | } 83 | 84 | func (d *ClickHouseDriver) Import5MinStocks(path string) error { 85 | return d.importCSV(model.TableStocks5Min, path) 86 | } 87 | 88 | func (d *ClickHouseDriver) ImportGBBQ(path string) error { 89 | d.truncateTable(model.TableGbbq) 90 | return d.importCSV(model.TableGbbq, path) 91 | } 92 | 93 | func (d *ClickHouseDriver) ImportXDXR(path string) error { 94 | d.truncateTable(model.TableXdxr) 95 | return d.importCSV(model.TableXdxr, path) 96 | } 97 | 98 | func (d *ClickHouseDriver) ImportAdjustFactors(path string) error { 99 | d.truncateTable(model.TableAdjustFactor) 100 | return d.importCSV(model.TableAdjustFactor, path) 101 | } 102 | 103 | func (d *ClickHouseDriver) Query(table string, conditions map[string]interface{}, dest interface{}) error { 104 | query := fmt.Sprintf("SELECT * FROM %s", table) 105 | args := []interface{}{} 106 | if len(conditions) > 0 { 107 | whereParts := []string{} 108 | for k, v := range conditions { 109 | whereParts = append(whereParts, fmt.Sprintf("%s = ?", k)) 110 | args = append(args, v) 111 | } 112 | query += " WHERE " + strings.Join(whereParts, " AND ") 113 | } 114 | 115 | return d.db.Select(dest, query, args...) 116 | } 117 | 118 | func (d *ClickHouseDriver) GetLatestDate(tableName string, dateCol string) (time.Time, error) { 119 | // 性能优化：利用索引快速获取最大时间 120 | query := fmt.Sprintf("SELECT max(%s) AS latest FROM %s", dateCol, tableName) 121 | 122 | var latest sql.NullTime 123 | err := d.db.Get(&latest, query) 124 | if err != nil { 125 | return time.Time{}, err 126 | } 127 | 128 | if !latest.Valid { 129 | return time.Time{}, nil 130 | } 131 | return latest.Time, nil 132 | } 133 | 134 | func (d *ClickHouseDriver) GetAllSymbols() ([]string, error) { 135 | query := fmt.Sprintf("SELECT DISTINCT symbol FROM %s", model.TableStocksDaily.TableName) 136 | 137 | var symbols []string 138 | err := d.db.Select(&symbols, query) 139 | if err != nil { 140 | return nil, fmt.Errorf("failed to query symbols: %w", err) 141 | } 142 | return symbols, nil 143 | } 144 | 145 | func (d *ClickHouseDriver) QueryStockData(symbol string, startDate, endDate *time.Time) ([]model.StockData, error) { 146 | query := fmt.Sprintf( 147 | "SELECT symbol, open, high, low, close, date FROM %s WHERE symbol = ? ORDER BY date ASC", 148 | model.TableStocksDaily.TableName, 149 | ) 150 | 151 | args := []interface{}{symbol} 152 | 153 | if startDate != nil { 154 | query += " AND date >= ?" 155 | args = append(args, *startDate) 156 | } 157 | if endDate != nil { 158 | query += " AND date <= ?" 159 | args = append(args, *endDate) 160 | } 161 | 162 | var results []model.StockData 163 | err := d.db.Select(&results, query, args...) 164 | if err != nil { 165 | return nil, fmt.Errorf("failed to query stocks: %w", err) 166 | } 167 | 168 | return results, nil 169 | } 170 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tdx2db - 获得专属的 A 股行情数据库 2 | 3 | [![GitHub release](https://img.shields.io/github/v/release/jing2uo/tdx2db?style=flat-square)](https://github.com/jing2uo/tdx2db/releases) 4 | [![Docker Image](https://img.shields.io/badge/docker-pull-blue?style=flat-square&logo=docker)](https://github.com/jing2uo/tdx2db/pkgs/container/tdx2db) 5 | [![License](https://img.shields.io/github/license/jing2uo/tdx2db?style=flat-square)](LICENSE) 6 | 7 | ## 概述 8 | 9 | `tdx2db` 是一个高效的工具，用于将通达信数据导入本地数据库，支持 DuckDB 和 ClickHouse。 10 | 11 | ## 亮点 12 | 13 | - **增量更新**: 支持间隔数天后数据补全，维护简单 14 | - **分时数据**: 支持导入 1min 和 5min 分时数据 15 | - **复权计算**: 自动计算前后复权因子，且因子支持分时使用 16 | - **衍生指标**: 自动计算换手率和市值信息 17 | - **稳定可靠**: 基于通达信数据，不依赖收费或限流接口 18 | 19 | ## 安装说明 20 | 21 | ### 使用二进制 22 | 23 | 从 [releases](https://github.com/jing2uo/tdx2db/releases) 下载，解压后移至 `$PATH`，二进制**仅支持在 x86 Linux 中**直接使用： 24 | 25 | ```bash 26 | sudo mv tdx2db /usr/local/bin/ && tdx2db -h 27 | ``` 28 | 29 | ### 使用 docker 30 | 31 | 项目会利用 github action 构建容器镜像，windows 和 mac 可以通过 docker 使用: 32 | 33 | ```bash 34 | docker run --rm --platform=linux/amd64 ghcr.io/jing2uo/tdx2db:latest -h 35 | ``` 36 | 37 | ## 导入到数据库 38 | 39 | ### 初始化 40 | 41 | 首次使用需要全量导入历史数据，可以从 [通达信券商数据](https://www.tdx.com.cn/article/vipdata.html) 下载**沪深京日线数据完整包**。 42 | 43 | 下载文件: 44 | 45 | ```shell 46 | # linux mac 47 | mkdir -p vipdoc 48 | wget https://data.tdx.com.cn/vipdoc/hsjday.zip && unzip -q hsjday.zip -d vipdoc 49 | 50 | # 若 unzip 解压后文件名如 sh\lday\sh000001.day，可以批量重命名 51 | # cd vipdoc 52 | # for f in *.day; do mv "$f" "${f##*\\}"; done 53 | 54 | # windows powershell 55 | Invoke-WebRequest -Uri "https://data.tdx.com.cn/vipdoc/hsjday.zip" -OutFile "hsjday.zip" 56 | Expand-Archive -Path "hsjday.zip" -DestinationPath "vipdoc" -Force 57 | ``` 58 | 59 | 二进制: 60 | 61 | ```shell 62 | # 导入 DuckDB, dburi 格式： duckdb://[path]，path 支持相对路径 63 | tdx2db init --dburi 'duckdb://./tdx.db' --dayfiledir ./vipdoc 64 | 65 | # 导入 ClickHouse, dburi 格式： clickhouse:[user[:password]@][host][:port][/database][?http_port=value1¶m2=value2&...] 66 | tdx2db init --dburi 'clickhouse://default:123456@localhost:9000/mydb?http_port=8123' --dayfiledir ./vipdoc 67 | 68 | # ClickHouse 有以下默认值: user=default, password="", port=9000, http_port=8123, database=default，可以根据情况简写 69 | tdx2db init --dburi 'clickhouse://localhost' --dayfiledir ./vipdoc 70 | ``` 71 | 72 | docker: 73 | 74 | ```shell 75 | # linux、mac docker 76 | docker run --rm --platform=linux/amd64 -v "$(pwd)":/data \ 77 | ghcr.io/jing2uo/tdx2db:latest \ 78 | init --dayfiledir /data/vipdoc --dburi 'duckdb:///data/tdx.db' 79 | 80 | # windows docker 81 | docker run --rm --platform=linux/amd64 -v "${PWD}:/data" \ 82 | ghcr.io/jing2uo/tdx2db:latest \ 83 | init --dayfiledir /data/vipdoc --dburi 'duckdb:///data/tdx.db' 84 | 85 | # 后续不再提示 docker 用法, 根据二进制示例修改第三行命令即可 86 | ``` 87 | 88 | **必填参数**: 89 | 90 | - `--dayfiledir`: 通达信 .day 文件所在目录 91 | - `--dburi`: 数据库连接信息 92 | 93 | ### 增量更新 94 | 95 | cron 命令会更新股票数据、股本变迁数据到最新日期，并计算前收盘价和复权因子。 96 | 97 | 初次使用时，请在 init 后立刻执行一次 cron，以获得复权相关数据。 98 | 99 | ```bash 100 | tdx2db cron --dburi 'duckdb://tdx.db' # ClickHouse schema 参考 init 部分 101 | ``` 102 | 103 | **必填参数**: 104 | 105 | - `--dburi`: 数据库连接信息 106 | 107 | ### 分时数据 108 | 109 | cron 命令支持 1min 和 5min 分时数据导入 110 | 111 | ```bash 112 | # --minline 可选 1、5、1,5 ，分别表示只处理1分钟、只处理5分钟、两种都处理 113 | tdx2db cron --dburi 'duckdb://tdx.db' --minline 1,5 114 | ``` 115 | 116 | **注意** 117 | 118 | 1. 分时数据下载和导入耗时，表数据量大 119 | 2. 通达信没提供历史分时数据，请自行检索后使用 duckdb 导入 120 | 3. 更新间隔超过 30 天以上，需手动补齐数据后才能继续处理 121 | 4. 股票代码变更不会处理历史记录 122 | 123 | ### 表查询 124 | 125 | raw\_ 前缀的表名用于存储基础数据，v\_ 前缀的表名是视图 126 | 127 | | 表/视图名 | 说明 | 128 | | :------------------ | :----------- | 129 | | `raw_stocks_daily` | 股票日线数据 | 130 | | `raw_stocks_1min` | 1 分钟 K 线 | 131 | | `raw_stocks_5min` | 5 分钟 K 线 | 132 | | `raw_adjust_factor` | 复权因子表 | 133 | | `v_xdxr` | 除权除息记录 | 134 | | `v_turnover` | 换手率与市值 | 135 | | `v_qfq_*` | 前复权数据 | 136 | | `v_hfq_*` | 后复权数据 | 137 | 138 | 复权数据： 139 | 140 | ```sql 141 | # 前复权 142 | select * from v_qfq_daily where symbol='sz000001' order by date; 143 | select * from v_qfq_5min where symbol='sz000001' order by date; 144 | 145 | # 后复权 146 | select * from v_hfq_daily where symbol='sz000001' order by date; 147 | select * from v_hfq_5min where symbol='sz000001' order by date; 148 | ``` 149 | 150 | 前收盘价和复权因子，可以根据前收盘价拓展其他复权算法： 151 | 152 | ```sql 153 | select * from raw_adjust_factor where symbol='sz000001'; 154 | ``` 155 | 156 | 算法来自 QUANTAXIS，原理参考：[点击查看](https://www.yuque.com/zhoujiping/programming/eb17548458c94bc7c14310f5b38cf25c#djL6L) 157 | 158 | 复权结果和 QUANTAXIS、通达信等比复权一致；其中前复权结果和雪球、新浪也一致。 159 | 160 | ## 通达信数据转 CSV 161 | 162 | convert 命令支持转换通达信日线、分时文件和四代行情、TIC 压缩包到 CSV，四代数据可以在 [每日数据](https://www.tdx.com.cn/article/daydata.html) 下载。 163 | 164 | ```shell 165 | tdx2db convert -t day -i ./vipdoc/ -o ./ # 转换 .day 日线文件 166 | tdx2db convert -h # 其他类型查看 help 167 | ``` 168 | 169 | 转换会查找目录中所有文件，包含指数、概念等很多非股票的记录，空文件会跳过处理。 170 | 171 | ## 欢迎 issue 和 pr 172 | 173 | 有任何使用问题都可以开 issue 讨论，也期待 pr~ 174 | -------------------------------------------------------------------------------- /model/tables.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | type DataType int 11 | 12 | const ( 13 | TypeString DataType = iota 14 | TypeFloat64 15 | TypeInt64 16 | TypeDate // YYYY-MM-DD 17 | TypeDateTime // YYYY-MM-DD HH:MM:SS 18 | ) 19 | 20 | type Column struct { 21 | Name string 22 | Type DataType 23 | } 24 | 25 | type TableMeta struct { 26 | TableName string 27 | Columns []Column 28 | OrderByKey []string 29 | } 30 | 31 | var ( 32 | tableRegistry []*TableMeta 33 | tableRegistryMu sync.Mutex 34 | ) 35 | 36 | func registerTable(t *TableMeta) { 37 | tableRegistryMu.Lock() 38 | defer tableRegistryMu.Unlock() 39 | tableRegistry = append(tableRegistry, t) 40 | } 41 | 42 | // AllTables 返回当前所有已注册的表结构 43 | func AllTables() []*TableMeta { 44 | tableRegistryMu.Lock() 45 | defer tableRegistryMu.Unlock() 46 | 47 | result := make([]*TableMeta, len(tableRegistry)) 48 | copy(result, tableRegistry) 49 | return result 50 | } 51 | 52 | // SchemaFromStruct 通过反射生成 TableMeta 并自动注册 53 | // 返回值为指针类型 *TableMeta 54 | func SchemaFromStruct(tableName string, model interface{}, orderByKey []string) *TableMeta { 55 | t := reflect.TypeOf(model) 56 | if t.Kind() == reflect.Ptr { 57 | t = t.Elem() 58 | } 59 | 60 | var cols []Column 61 | 62 | for i := 0; i < t.NumField(); i++ { 63 | field := t.Field(i) 64 | 65 | // 1. 获取列名 66 | colName := field.Tag.Get("col") 67 | if colName == "" { 68 | colName = strings.ToLower(field.Name) 69 | } 70 | 71 | // 2. 推断类型 (保持原有逻辑) 72 | var dType DataType 73 | customType := field.Tag.Get("type") 74 | switch { 75 | case customType == "date": 76 | dType = TypeDate 77 | case customType == "datetime": 78 | dType = TypeDateTime 79 | default: 80 | switch field.Type.Kind() { 81 | case reflect.String: 82 | dType = TypeString 83 | case reflect.Float64, reflect.Float32: 84 | dType = TypeFloat64 85 | case reflect.Int, reflect.Int64, reflect.Int32, reflect.Uint32: 86 | dType = TypeInt64 87 | case reflect.Struct: 88 | if field.Type == reflect.TypeOf(time.Time{}) { 89 | dType = TypeDateTime 90 | } 91 | default: 92 | dType = TypeString 93 | } 94 | } 95 | 96 | cols = append(cols, Column{Name: colName, Type: dType}) 97 | } 98 | 99 | meta := &TableMeta{ 100 | TableName: tableName, 101 | Columns: cols, 102 | OrderByKey: orderByKey, 103 | } 104 | 105 | // === 核心改动：自动注册 === 106 | registerTable(meta) 107 | 108 | return meta 109 | } 110 | 111 | // --- 结构体定义 (Schema) --- 112 | type StockData struct { 113 | Symbol string `col:"symbol"` 114 | Open float64 `col:"open"` 115 | High float64 `col:"high"` 116 | Low float64 `col:"low"` 117 | Close float64 `col:"close"` 118 | Amount float64 `col:"amount"` 119 | Volume int64 `col:"volume"` 120 | Date time.Time `col:"date" type:"date"` 121 | } 122 | 123 | type StockMinData struct { 124 | Symbol string `col:"symbol"` 125 | Open float64 `col:"open"` 126 | High float64 `col:"high"` 127 | Low float64 `col:"low"` 128 | Close float64 `col:"close"` 129 | Amount float64 `col:"amount"` 130 | Volume int64 `col:"volume"` 131 | Datetime time.Time `col:"datetime" type:"datetime" ` 132 | } 133 | 134 | type Factor struct { 135 | Symbol string `col:"symbol"` 136 | Date time.Time `col:"date" type:"date"` 137 | Close float64 `col:"close"` 138 | PreClose float64 `col:"pre_close"` 139 | QfqFactor float64 `col:"qfq_factor"` 140 | HfqFactor float64 `col:"hfq_factor"` 141 | } 142 | 143 | type GbbqData struct { 144 | Category int `col:"category"` 145 | Symbol string `col:"symbol"` 146 | Date time.Time `col:"date" type:"date"` 147 | PreFloat float64 `col:"pre_float"` 148 | PreTotal float64 `col:"pre_total"` 149 | PostFloat float64 `col:"post_float"` 150 | PostTotal float64 `col:"post_total"` 151 | } 152 | 153 | type XdxrData struct { 154 | Symbol string `col:"symbol"` 155 | Date time.Time `col:"date" type:"date"` 156 | Fenhong float64 `col:"fenhong"` 157 | Peigujia float64 `col:"peigujia"` 158 | Songzhuangu float64 `col:"songzhuangu"` 159 | Peigu float64 `col:"peigu"` 160 | } 161 | 162 | // --- 表结构元数据 (TableMeta) --- 163 | 164 | var TableStocksDaily = SchemaFromStruct( 165 | "raw_stocks_daily", 166 | StockData{}, 167 | []string{"symbol", "date"}, 168 | ) 169 | 170 | var TableStocks1Min = SchemaFromStruct( 171 | "raw_stocks_1min", 172 | StockMinData{}, 173 | []string{"symbol", "datetime"}, 174 | ) 175 | 176 | var TableStocks5Min = SchemaFromStruct( 177 | "raw_stocks_5min", 178 | StockMinData{}, 179 | []string{"symbol", "datetime"}, 180 | ) 181 | 182 | var TableAdjustFactor = SchemaFromStruct( 183 | "raw_adjust_factor", 184 | Factor{}, 185 | []string{"symbol", "date"}, 186 | ) 187 | 188 | var TableGbbq = SchemaFromStruct( 189 | "raw_gbbq", 190 | GbbqData{}, 191 | []string{"symbol", "date"}, 192 | ) 193 | 194 | var TableXdxr = SchemaFromStruct( 195 | "raw_xdxr", 196 | XdxrData{}, 197 | []string{"symbol", "date"}, 198 | ) 199 | -------------------------------------------------------------------------------- /database/clickhouse/ddl.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/jing2uo/tdx2db/model" 8 | ) 9 | 10 | // mapType 针对 ClickHouse 进行类型优化 11 | func (d *ClickHouseDriver) mapType(colName string, dt model.DataType) string { 12 | isKey := strings.Contains(strings.ToLower(colName), "symbol") //|| 13 | //strings.Contains(strings.ToLower(colName), "category") 14 | 15 | switch dt { 16 | case model.TypeString: 17 | if isKey { 18 | return "LowCardinality(String)" 19 | } 20 | return "String" 21 | case model.TypeFloat64: 22 | return "Float64" 23 | case model.TypeInt64: 24 | return "Int64" 25 | case model.TypeDate: 26 | return "Date32" // Date32 范围比 Date 更大 (1900-2299) 27 | case model.TypeDateTime: 28 | return "DateTime64(0, 'Asia/Shanghai')" 29 | default: 30 | return "String" 31 | } 32 | } 33 | 34 | func (d *ClickHouseDriver) createTableInternal(meta *model.TableMeta) error { 35 | var colDefs []string 36 | var dateCol, keyCol string 37 | 38 | // 1. 构建列定义 39 | for _, col := range meta.Columns { 40 | sqlType := d.mapType(col.Name, col.Type) 41 | colDefs = append(colDefs, fmt.Sprintf("%s %s", col.Name, sqlType)) 42 | 43 | // 自动探测用于排序键的列 44 | lowerName := strings.ToLower(col.Name) 45 | if lowerName == "date" || lowerName == "datetime" { 46 | dateCol = col.Name 47 | } 48 | if lowerName == "symbol" { 49 | keyCol = col.Name 50 | } 51 | } 52 | 53 | // 2. 确定排序键 (MergeTree 必须) 54 | orderBy := "tuple()" 55 | if keyCol != "" && dateCol != "" { 56 | orderBy = fmt.Sprintf("(%s, %s)", keyCol, dateCol) 57 | } else if dateCol != "" { 58 | orderBy = dateCol 59 | } else if keyCol != "" { 60 | orderBy = keyCol 61 | } 62 | 63 | // 3. 建表语句 64 | query := fmt.Sprintf(` 65 | CREATE TABLE IF NOT EXISTS %s ( 66 | %s 67 | ) ENGINE = MergeTree() 68 | ORDER BY %s 69 | `, meta.TableName, strings.Join(colDefs, ", "), orderBy) 70 | 71 | _, err := d.db.Exec(query) 72 | return err 73 | } 74 | 75 | func (d *ClickHouseDriver) registerViews() { 76 | // 换手率与市值视图 77 | d.viewImpls[model.ViewTurnover] = func() error { 78 | query := fmt.Sprintf(` 79 | CREATE OR REPLACE VIEW %s AS 80 | WITH gbbq_sorted AS ( 81 | SELECT 82 | t.symbol, 83 | t.date, 84 | t.post_float, 85 | t.post_total 86 | FROM %s AS t 87 | WHERE t.category IN (2, 3, 5, 7, 8, 9, 10) 88 | ORDER BY t.symbol, t.date 89 | ) 90 | SELECT 91 | r.date, 92 | r.symbol, 93 | CASE WHEN g.post_float > 0 THEN 94 | ROUND(r.volume / (g.post_float * 10000), 6) 95 | ELSE 0 END AS turnover, 96 | ROUND(g.post_float * 10000 * r.close, 2) AS float_mv, 97 | ROUND(g.post_total * 10000 * r.close, 2) AS total_mv 98 | FROM %s r 99 | ASOF LEFT JOIN gbbq_sorted g 100 | ON r.symbol = g.symbol AND r.date >= g.date 101 | `, 102 | model.ViewTurnover, 103 | model.TableGbbq.TableName, 104 | model.TableStocksDaily.TableName) 105 | _, err := d.db.Exec(query) 106 | return err 107 | } 108 | 109 | // 通用创建复权视图函数 110 | createAdjustView := func(viewName model.ViewID, sourceTable, factorCol string, isMin bool) error { 111 | // 默认日线逻辑 112 | joinClause := fmt.Sprintf("LEFT JOIN %s f ON s.symbol = f.symbol AND s.date = f.date", model.TableAdjustFactor.TableName) 113 | timeCol := "date" 114 | 115 | // 分钟线逻辑 116 | if isMin { 117 | timeCol = "datetime" 118 | joinClause = fmt.Sprintf(` 119 | ASOF LEFT JOIN ( 120 | SELECT symbol, toDateTime(date) as dt_start, %s 121 | FROM %s 122 | ORDER BY symbol, dt_start 123 | ) f ON s.symbol = f.symbol AND s.datetime >= f.dt_start 124 | `, factorCol, model.TableAdjustFactor.TableName) 125 | } 126 | 127 | // 组装 SQL 128 | query := fmt.Sprintf(` 129 | CREATE OR REPLACE VIEW %s AS 130 | SELECT 131 | s.symbol, 132 | %s, -- date or datetime 133 | s.volume, 134 | s.amount, 135 | ROUND(s.open * f.%s, 2) AS open, 136 | ROUND(s.high * f.%s, 2) AS high, 137 | ROUND(s.low * f.%s, 2) AS low, 138 | ROUND(s.close * f.%s, 2) AS close 139 | FROM %s s 140 | %s -- Factor Join 141 | `, 142 | viewName, 143 | timeCol, 144 | factorCol, factorCol, factorCol, factorCol, 145 | sourceTable, 146 | joinClause, 147 | ) 148 | _, err := d.db.Exec(query) 149 | return err 150 | } 151 | 152 | // 注册各个视图 153 | d.viewImpls[model.ViewDailyQFQ] = func() error { 154 | return createAdjustView(model.ViewDailyQFQ, model.TableStocksDaily.TableName, "qfq_factor", false) 155 | } 156 | d.viewImpls[model.ViewDailyHFQ] = func() error { 157 | return createAdjustView(model.ViewDailyHFQ, model.TableStocksDaily.TableName, "hfq_factor", false) 158 | } 159 | d.viewImpls[model.View1MinQFQ] = func() error { 160 | return createAdjustView(model.View1MinQFQ, model.TableStocks1Min.TableName, "qfq_factor", true) 161 | } 162 | d.viewImpls[model.View1MinHFQ] = func() error { 163 | return createAdjustView(model.View1MinHFQ, model.TableStocks1Min.TableName, "hfq_factor", true) 164 | } 165 | d.viewImpls[model.View5MinQFQ] = func() error { 166 | return createAdjustView(model.View5MinQFQ, model.TableStocks5Min.TableName, "qfq_factor", true) 167 | } 168 | d.viewImpls[model.View5MinHFQ] = func() error { 169 | return createAdjustView(model.View5MinHFQ, model.TableStocks5Min.TableName, "hfq_factor", true) 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /cmd/convert.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/jing2uo/tdx2db/tdx" 10 | "github.com/jing2uo/tdx2db/utils" 11 | ) 12 | 13 | type InputSourceType int 14 | 15 | type ConvertOptions struct { 16 | InputPath string 17 | InputType InputSourceType 18 | OutputPath string 19 | } 20 | 21 | const ( 22 | DayFileDir InputSourceType = iota 23 | TicZip 24 | DayZip 25 | Min1FileDir 26 | Min5FileDir 27 | ) 28 | 29 | func isDirType(t InputSourceType) bool { 30 | switch t { 31 | case DayFileDir, Min1FileDir, Min5FileDir: 32 | return true 33 | default: 34 | return false 35 | } 36 | } 37 | 38 | func Convert(opts ConvertOptions) error { 39 | if opts.InputPath == "" { 40 | return errors.New("input path cannot be empty") 41 | } 42 | if opts.OutputPath == "" { 43 | return errors.New("output path cannot be empty") 44 | } 45 | 46 | if err := utils.CheckOutputDir(opts.OutputPath); err != nil { 47 | return err 48 | } 49 | 50 | if isDirType(opts.InputType) { 51 | if err := utils.CheckDirectory(opts.InputPath); err != nil { 52 | return err 53 | } 54 | } else { 55 | if err := utils.CheckFile(opts.InputPath); err != nil { 56 | return err 57 | } 58 | } 59 | 60 | dataDir := TempDir 61 | 62 | var validPrefixes = []string{"sh", "sz", "bj"} 63 | 64 | switch opts.InputType { 65 | 66 | case DayFileDir: 67 | fmt.Printf("📦 开始处理日线目录: %s\n", opts.InputPath) 68 | output := filepath.Join(opts.OutputPath, "tdx2db_day.csv") 69 | 70 | fmt.Println("🐢 开始转换日线数据") 71 | _, err := tdx.ConvertFilesToCSV(opts.InputPath, validPrefixes, output, ".day") 72 | if err != nil { 73 | return fmt.Errorf("failed to convert day files: %w", err) 74 | } 75 | 76 | fmt.Printf("🔥 转换完成: %s\n", output) 77 | 78 | case Min1FileDir: 79 | fmt.Printf("📦 开始处理分时数据目录: %s\n", opts.InputPath) 80 | output := filepath.Join(opts.OutputPath, "tdx2db_1min.csv") 81 | 82 | fmt.Println("🐢 开始转换 1 分钟数据") 83 | _, err := tdx.ConvertFilesToCSV(opts.InputPath, validPrefixes, output, ".01") 84 | if err != nil { 85 | return fmt.Errorf("failed to convert 1min files: %w", err) 86 | } 87 | 88 | fmt.Printf("🔥 转换完成: %s\n", output) 89 | 90 | case Min5FileDir: 91 | fmt.Printf("📦 开始处理分时数据目录: %s\n", opts.InputPath) 92 | output := filepath.Join(opts.OutputPath, "tdx2db_5min.csv") 93 | 94 | fmt.Println("🐢 开始转换 5 分钟数据") 95 | _, err := tdx.ConvertFilesToCSV(opts.InputPath, validPrefixes, output, ".5") 96 | if err != nil { 97 | return fmt.Errorf("failed to convert 5min files: %w", err) 98 | } 99 | 100 | fmt.Printf("🔥 转换完成: %s\n", output) 101 | 102 | case TicZip: 103 | fmt.Printf("📦 开始处理四代 TIC 压缩文件: %s\n", opts.InputPath) 104 | 105 | filename := filepath.Base(opts.InputPath) 106 | baseName := filename[:len(filename)-len(filepath.Ext(filename))] 107 | 108 | targetPath := filepath.Join(VipdocDir, "newdatetick") 109 | if err := os.MkdirAll(targetPath, 0755); err != nil { 110 | return fmt.Errorf("failed to create directory: %w", err) 111 | } 112 | if err := utils.UnzipFile(opts.InputPath, targetPath); err != nil { 113 | return fmt.Errorf("failed to unzip file %s: %w", opts.InputPath, err) 114 | } 115 | 116 | fmt.Printf("🐢 开始转档分笔数据\n") 117 | if err := tdx.DatatoolCreate(dataDir, "tick", Today); err != nil { 118 | return fmt.Errorf("failed to execute DatatoolTickCreate: %w", err) 119 | } 120 | if err := tdx.DatatoolCreate(dataDir, "min", Today); err != nil { 121 | return fmt.Errorf("failed to execute DatatoolMinCreate: %w", err) 122 | } 123 | 124 | min1_output := filepath.Join(opts.OutputPath, fmt.Sprintf("%s_1min.csv", baseName)) 125 | min5_output := filepath.Join(opts.OutputPath, fmt.Sprintf("%s_5min.csv", baseName)) 126 | 127 | fmt.Printf("🐢 开始转换 1 分钟数据\n") 128 | _, err := tdx.ConvertFilesToCSV(VipdocDir, validPrefixes, min1_output, ".01") 129 | if err != nil { 130 | return fmt.Errorf("failed to convert 1-minute files: %w", err) 131 | } 132 | 133 | fmt.Printf("🐢 开始转换 5 分钟数据\n") 134 | _, err = tdx.ConvertFilesToCSV(VipdocDir, validPrefixes, min5_output, ".5") 135 | if err != nil { 136 | return fmt.Errorf("failed to convert 5-minute files: %w", err) 137 | } 138 | 139 | fmt.Printf("🔥 转换完成\n") 140 | fmt.Printf("📊 1 分钟数据: %s\n", min1_output) 141 | fmt.Printf("📊 5 分钟数据: %s\n", min5_output) 142 | 143 | case DayZip: 144 | fmt.Printf("📦 开始处理四代行情压缩文件: %s\n", opts.InputPath) 145 | 146 | filename := filepath.Base(opts.InputPath) 147 | baseName := filename[:len(filename)-len(filepath.Ext(filename))] 148 | 149 | unzipDestPath := filepath.Join(VipdocDir, "refmhq") 150 | if err := os.MkdirAll(unzipDestPath, 0755); err != nil { 151 | return fmt.Errorf("failed to create unzip destination directory: %w", err) 152 | } 153 | if err := utils.UnzipFile(opts.InputPath, unzipDestPath); err != nil { 154 | return fmt.Errorf("failed to unzip file %s: %w", opts.InputPath, err) 155 | } 156 | 157 | fmt.Printf("🐢 开始转换日线数据\n") 158 | if err := tdx.DatatoolCreate(dataDir, "day", Today); err != nil { 159 | return fmt.Errorf("failed to execute DatatoolDayCreate: %w", err) 160 | } 161 | 162 | output := filepath.Join(opts.OutputPath, fmt.Sprintf("%s_day.csv", baseName)) 163 | 164 | _, err := tdx.ConvertFilesToCSV(VipdocDir, validPrefixes, output, ".day") 165 | if err != nil { 166 | return fmt.Errorf("failed to convert day files: %w", err) 167 | } 168 | 169 | fmt.Printf("🔥 转换完成: %s\n", output) 170 | } 171 | 172 | return nil 173 | } 174 | -------------------------------------------------------------------------------- /tdx/gbbq.go: -------------------------------------------------------------------------------- 1 | package tdx 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/hex" 6 | "fmt" 7 | "math" 8 | "os" 9 | "strings" 10 | "time" 11 | 12 | "github.com/jing2uo/tdx2db/model" 13 | ) 14 | 15 | func DecodeGbbqFile(gbbqFile string) ([]model.GbbqData, []model.XdxrData, error) { 16 | hexStr := strings.ReplaceAll(HexKeys, " ", "") 17 | keys, err := hex.DecodeString(hexStr) 18 | if err != nil { 19 | return nil, nil, fmt.Errorf("failed to decode hex keys: %w", err) 20 | } 21 | 22 | content, err := os.ReadFile(gbbqFile) 23 | if err != nil { 24 | return nil, nil, fmt.Errorf("failed to read GBBQ file: %w", err) 25 | } 26 | 27 | if len(content) < 4 { 28 | return nil, nil, nil 29 | } 30 | 31 | // 读取记录数量 32 | count := int(binary.LittleEndian.Uint32(content[0:4])) 33 | 34 | // 预分配切片，虽然无法精确知道两类数据的比例，但可以先给一定的容量避免频繁扩容 35 | gbbqResult := make([]model.GbbqData, 0, count) 36 | xdxrResult := make([]model.XdxrData, 0, count) 37 | 38 | pos := 4 39 | var clearData [29]byte 40 | totalLen := len(content) 41 | 42 | for i := 0; i < count; i++ { 43 | // 检查边界 44 | if pos+29 > totalLen { 45 | break 46 | } 47 | 48 | // --- 解密阶段 --- 49 | decryptBlockToBuf(keys, content[pos:pos+8], clearData[0:8]) 50 | pos += 8 51 | decryptBlockToBuf(keys, content[pos:pos+8], clearData[8:16]) 52 | pos += 8 53 | decryptBlockToBuf(keys, content[pos:pos+8], clearData[16:24]) 54 | pos += 8 55 | copy(clearData[24:29], content[pos:pos+5]) 56 | pos += 5 57 | 58 | // --- 解析阶段 --- 59 | 60 | // 1. Code 处理 61 | codeBytes := clearData[1:8] 62 | strLen := 0 63 | for k := 0; k < len(codeBytes); k++ { 64 | if codeBytes[k] == 0 { 65 | break 66 | } 67 | strLen++ 68 | } 69 | code := string(codeBytes[:strLen]) 70 | 71 | // 生成 Symbol 并过滤 72 | symbol, ok := generateSymbol(code) 73 | if !ok { 74 | continue // 没匹配到规则，跳过 75 | } 76 | 77 | // 2. Date 78 | dateInt := binary.LittleEndian.Uint32(clearData[8:12]) 79 | dateTime, err := fastParseDate(dateInt) // 假设此函数已定义 80 | if err != nil { 81 | continue 82 | } 83 | 84 | // 3. Floats 85 | c1 := float64(math.Float32frombits(binary.LittleEndian.Uint32(clearData[13:17]))) 86 | c2 := float64(math.Float32frombits(binary.LittleEndian.Uint32(clearData[17:21]))) 87 | c3 := float64(math.Float32frombits(binary.LittleEndian.Uint32(clearData[21:25]))) 88 | c4 := float64(math.Float32frombits(binary.LittleEndian.Uint32(clearData[25:29]))) 89 | 90 | // 4. Category 分类处理 91 | category := int(clearData[12]) 92 | 93 | if category == 1 { 94 | // 除权除息数据 95 | xdxrResult = append(xdxrResult, model.XdxrData{ 96 | Symbol: symbol, 97 | Date: dateTime, 98 | Fenhong: c1, 99 | Peigujia: c2, 100 | Songzhuangu: c3, 101 | Peigu: c4, 102 | }) 103 | } else { 104 | // 股本变动数据 105 | gbbqResult = append(gbbqResult, model.GbbqData{ 106 | Category: category, 107 | Symbol: symbol, 108 | Date: dateTime, 109 | PreFloat: c1, 110 | PreTotal: c2, 111 | PostFloat: c3, 112 | PostTotal: c4, 113 | }) 114 | } 115 | } 116 | 117 | return gbbqResult, xdxrResult, nil 118 | } 119 | 120 | // fastParseDate 将 YYYYMMDD 整数转为 time.Time，比字符串解析快 100 倍 121 | func fastParseDate(date uint32) (time.Time, error) { 122 | if date == 0 { 123 | return time.Time{}, fmt.Errorf("zero date") 124 | } 125 | y := int(date / 10000) 126 | m := int((date % 10000) / 100) 127 | d := int(date % 100) 128 | if m < 1 || m > 12 || d < 1 || d > 31 { 129 | return time.Time{}, fmt.Errorf("invalid date: %d", date) 130 | } 131 | return time.Date(y, time.Month(m), d, 0, 0, 0, 0, time.Local), nil 132 | } 133 | 134 | func generateSymbol(code string) (string, bool) { 135 | switch { 136 | case strings.HasPrefix(code, "00") || strings.HasPrefix(code, "30"): 137 | return "sz" + code, true 138 | case strings.HasPrefix(code, "60") || strings.HasPrefix(code, "68"): 139 | return "sh" + code, true 140 | case strings.HasPrefix(code, "92") || strings.HasPrefix(code, "87") || 141 | strings.HasPrefix(code, "83") || strings.HasPrefix(code, "43"): 142 | return "bj" + code, true 143 | default: 144 | return "", false 145 | } 146 | } 147 | 148 | // decryptBlockToBuf 优化后的解密函数 149 | func decryptBlockToBuf(keys, encrypted, dst []byte) { 150 | // 安全检查，避免 panic 151 | if len(encrypted) < 8 || len(dst) < 8 { 152 | return 153 | } 154 | 155 | eax := binary.LittleEndian.Uint32(keys[0x44:0x48]) 156 | A := binary.LittleEndian.Uint32(encrypted[0:4]) 157 | B := binary.LittleEndian.Uint32(encrypted[4:8]) 158 | 159 | num := eax ^ A 160 | numold := B 161 | 162 | for j := 0x40; j >= 4; j -= 4 { 163 | ebx := (num & 0xFF0000) >> 16 164 | offset := int(ebx)*4 + 0x448 165 | eax = binary.LittleEndian.Uint32(keys[offset : offset+4]) 166 | 167 | ebx = num >> 24 168 | offset = int(ebx)*4 + 0x48 169 | eax_add := binary.LittleEndian.Uint32(keys[offset : offset+4]) 170 | eax += eax_add 171 | 172 | ebx = (num & 0xFF00) >> 8 173 | offset = int(ebx)*4 + 0x848 174 | eax_xor := binary.LittleEndian.Uint32(keys[offset : offset+4]) 175 | eax ^= eax_xor 176 | 177 | ebx = num & 0xFF 178 | offset = int(ebx)*4 + 0xC48 179 | eax_add = binary.LittleEndian.Uint32(keys[offset : offset+4]) 180 | eax += eax_add 181 | 182 | eax_xor = binary.LittleEndian.Uint32(keys[j : j+4]) 183 | eax ^= eax_xor 184 | 185 | temp := num 186 | num = numold ^ eax 187 | numold = temp 188 | } 189 | 190 | numold_op := binary.LittleEndian.Uint32(keys[0:4]) 191 | numold ^= numold_op 192 | 193 | binary.LittleEndian.PutUint32(dst[0:4], numold) 194 | binary.LittleEndian.PutUint32(dst[4:8], num) 195 | } 196 | -------------------------------------------------------------------------------- /utils/download.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strconv" 9 | "sync" 10 | ) 11 | 12 | // Download 封装下载任务 13 | type Download struct { 14 | Url string 15 | Target string 16 | TotalSections int 17 | } 18 | 19 | // DownloadFile 下载文件并返回 HTTP 状态码。 20 | // 若状态码为 404 或 200，error 为 nil。 21 | // 若服务器不支持 Range，则自动降级为单线程下载（静默处理）。 22 | func DownloadFile(url string, targetPath string) (int, error) { 23 | const totalSections = 5 24 | 25 | d := &Download{ 26 | Url: url, 27 | Target: targetPath, 28 | TotalSections: totalSections, 29 | } 30 | 31 | // Step 1: HEAD 获取文件元信息 32 | r, err := d.getNewRequest("HEAD") 33 | if err != nil { 34 | return 0, fmt.Errorf("create HEAD request: %w", err) 35 | } 36 | res, err := http.DefaultClient.Do(r) 37 | if err != nil { 38 | return 0, fmt.Errorf("execute HEAD request: %w", err) 39 | } 40 | defer res.Body.Close() 41 | 42 | statusCode := res.StatusCode 43 | 44 | if statusCode == http.StatusNotFound { 45 | return 404, nil 46 | } 47 | if statusCode != http.StatusOK { 48 | return statusCode, fmt.Errorf("unexpected status: %d", statusCode) 49 | } 50 | 51 | // Step 2: 检查是否能获取 Content-Length 52 | size, err := strconv.Atoi(res.Header.Get("Content-Length")) 53 | if err != nil || size <= 0 { 54 | return d.singleThreadDownload() 55 | } 56 | 57 | // Step 3: 检查服务器是否支持 Range 58 | testReq, _ := d.getNewRequest("GET") 59 | testReq.Header.Set("Range", "bytes=0-0") 60 | testResp, err := http.DefaultClient.Do(testReq) 61 | if err != nil { 62 | return 0, fmt.Errorf("range test request: %w", err) 63 | } 64 | defer testResp.Body.Close() 65 | 66 | if testResp.StatusCode != http.StatusPartialContent { 67 | // 不支持 Range -> 自动降级为单线程 68 | return d.singleThreadDownload() 69 | } 70 | 71 | // Step 4: 执行并发下载 72 | eachSize := size / d.TotalSections 73 | sections := make([][2]int, d.TotalSections) 74 | for i := range sections { 75 | if i == 0 { 76 | sections[i][0] = 0 77 | } else { 78 | sections[i][0] = sections[i-1][1] + 1 79 | } 80 | if i < d.TotalSections-1 { 81 | sections[i][1] = sections[i][0] + eachSize 82 | } else { 83 | sections[i][1] = size - 1 84 | } 85 | } 86 | 87 | var wg sync.WaitGroup 88 | var mu sync.Mutex 89 | var sectionErr error 90 | 91 | for i, sec := range sections { 92 | wg.Add(1) 93 | go func(i int, sec [2]int) { 94 | defer wg.Done() 95 | if err := d.downloadSection(i, sec); err != nil { 96 | mu.Lock() 97 | if sectionErr == nil { 98 | sectionErr = err 99 | } 100 | mu.Unlock() 101 | } 102 | }(i, sec) 103 | } 104 | wg.Wait() 105 | 106 | if sectionErr != nil { 107 | // 自动降级 108 | return d.singleThreadDownload() 109 | } 110 | 111 | if err := d.mergeSections(sections); err != nil { 112 | return statusCode, fmt.Errorf("merge sections: %w", err) 113 | } 114 | 115 | return statusCode, nil 116 | } 117 | 118 | func (d *Download) getNewRequest(method string) (*http.Request, error) { 119 | r, err := http.NewRequest(method, d.Url, nil) 120 | if err != nil { 121 | return nil, fmt.Errorf("create %s request: %w", method, err) 122 | } 123 | r.Header.Set("User-Agent", "GenericDownloader/1.0") 124 | return r, nil 125 | } 126 | 127 | func (d *Download) downloadSection(i int, section [2]int) error { 128 | r, err := d.getNewRequest("GET") 129 | if err != nil { 130 | return fmt.Errorf("create section %d request: %w", i, err) 131 | } 132 | r.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", section[0], section[1])) 133 | 134 | resp, err := http.DefaultClient.Do(r) 135 | if err != nil { 136 | return fmt.Errorf("execute section %d: %w", i, err) 137 | } 138 | defer resp.Body.Close() 139 | 140 | if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK { 141 | return fmt.Errorf("unexpected section %d status: %d", i, resp.StatusCode) 142 | } 143 | 144 | f, err := os.Create(fmt.Sprintf("%s.part%d", d.Target, i)) 145 | if err != nil { 146 | return fmt.Errorf("create part file %d: %w", i, err) 147 | } 148 | defer f.Close() 149 | 150 | if _, err := io.Copy(f, resp.Body); err != nil { 151 | return fmt.Errorf("write part %d: %w", i, err) 152 | } 153 | 154 | return nil 155 | } 156 | 157 | func (d *Download) mergeSections(sections [][2]int) error { 158 | f, err := os.Create(d.Target) 159 | if err != nil { 160 | return fmt.Errorf("create target: %w", err) 161 | } 162 | defer f.Close() 163 | 164 | for i := 0; i < len(sections); i++ { 165 | partFile := fmt.Sprintf("%s.part%d", d.Target, i) 166 | data, err := os.ReadFile(partFile) 167 | if err != nil { 168 | return fmt.Errorf("read part file %s: %w", partFile, err) 169 | } 170 | if _, err := f.Write(data); err != nil { 171 | return fmt.Errorf("write part %d: %w", i, err) 172 | } 173 | _ = os.Remove(partFile) 174 | } 175 | return nil 176 | } 177 | 178 | // singleThreadDownload 用于降级时的完整文件下载 179 | func (d *Download) singleThreadDownload() (int, error) { 180 | resp, err := http.Get(d.Url) 181 | if err != nil { 182 | return 0, fmt.Errorf("single-thread GET: %w", err) 183 | } 184 | defer resp.Body.Close() 185 | 186 | if resp.StatusCode == http.StatusNotFound { 187 | return 404, nil 188 | } 189 | if resp.StatusCode != http.StatusOK { 190 | return resp.StatusCode, fmt.Errorf("unexpected status: %d", resp.StatusCode) 191 | } 192 | 193 | f, err := os.Create(d.Target) 194 | if err != nil { 195 | return resp.StatusCode, fmt.Errorf("create target: %w", err) 196 | } 197 | defer f.Close() 198 | 199 | if _, err := io.Copy(f, resp.Body); err != nil { 200 | return resp.StatusCode, fmt.Errorf("write target: %w", err) 201 | } 202 | 203 | return resp.StatusCode, nil 204 | } 205 | -------------------------------------------------------------------------------- /tdx/kline.go: -------------------------------------------------------------------------------- 1 | package tdx 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "math" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/jing2uo/tdx2db/model" 15 | "github.com/jing2uo/tdx2db/utils" 16 | ) 17 | 18 | // Constants & Configuration 19 | 20 | var maxConcurrency = runtime.NumCPU() 21 | 22 | const ( 23 | recordSize = 32 // TDX 记录固定字节大小 24 | ) 25 | 26 | // batchData 泛型容器，用于在 Channel 中传递解析好的数据块 27 | type batchData[T any] struct { 28 | Rows []T 29 | Err error 30 | } 31 | 32 | // Main Entry Point 33 | 34 | func ConvertFilesToCSV(inputDir string, validPrefixes []string, outputFile string, suffix string) (string, error) { 35 | switch suffix { 36 | case ".day": 37 | // 使用泛型函数处理 StockData 38 | return runConversion[model.StockData](inputDir, validPrefixes, outputFile, suffix, processDayFile) 39 | case ".01", ".5": 40 | // 使用泛型函数处理 StockMinData 41 | return runConversion[model.StockMinData](inputDir, validPrefixes, outputFile, suffix, processMinFile) 42 | default: 43 | return "", fmt.Errorf("unsupported suffix: %s", suffix) 44 | } 45 | } 46 | 47 | // Generic Conversion Engine 48 | 49 | func runConversion[T any]( 50 | inputDir string, 51 | validPrefixes []string, 52 | outputFile string, 53 | suffix string, 54 | parser func([]byte, string) ([]T, error), 55 | ) (string, error) { 56 | 57 | // 1. 收集文件 58 | files, err := collectFiles(inputDir, validPrefixes, suffix) 59 | if err != nil { 60 | return "", err 61 | } 62 | 63 | // 创建 Writer 64 | cw, _ := utils.NewCSVWriter[T](outputFile) 65 | 66 | // 4. 并发管道设置 67 | batchChan := make(chan batchData[T], maxConcurrency*2) 68 | var producerWg sync.WaitGroup 69 | var consumerWg sync.WaitGroup 70 | sem := make(chan struct{}, maxConcurrency) // 信号量控制读取并发度 71 | 72 | // 错误收集 73 | var errors []string 74 | var errMu sync.Mutex 75 | collectError := func(e error) { 76 | errMu.Lock() 77 | errors = append(errors, e.Error()) 78 | errMu.Unlock() 79 | } 80 | 81 | // Consumer: 写入 CSV 82 | consumerWg.Go(func() { 83 | defer cw.Close() // 确保数据落盘 84 | for batch := range batchChan { 85 | if batch.Err != nil { 86 | collectError(batch.Err) 87 | continue 88 | } 89 | if len(batch.Rows) > 0 { 90 | if err := cw.Write(batch.Rows); err != nil { 91 | collectError(fmt.Errorf("csv write error: %w", err)) 92 | } 93 | } 94 | } 95 | }) 96 | 97 | // Producer: 读取并解析文件 98 | for _, file := range files { 99 | producerWg.Go(func() { 100 | sem <- struct{}{} 101 | defer func() { <-sem }() 102 | 103 | rows, err := readFileAndParse(file, suffix, parser) 104 | batchChan <- batchData[T]{Rows: rows, Err: err} 105 | }) 106 | } 107 | 108 | producerWg.Wait() 109 | close(batchChan) 110 | consumerWg.Wait() 111 | 112 | if len(errors) > 0 { 113 | return outputFile, fmt.Errorf("occurred %d errors, first: %s", len(errors), errors[0]) 114 | } 115 | return outputFile, nil 116 | } 117 | 118 | // readFileAndParse 读取文件并解析 119 | func readFileAndParse[T any](filename, suffix string, parser func([]byte, string) ([]T, error)) ([]T, error) { 120 | // TDX 单个文件通常很小，直接 ReadFile 读入内存是最快的，避免了 bufio 的开销 121 | data, err := os.ReadFile(filename) 122 | if err != nil { 123 | return nil, fmt.Errorf("read %s: %w", filename, err) 124 | } 125 | if len(data) == 0 { 126 | return nil, nil 127 | } 128 | 129 | symbol := strings.TrimSuffix(filepath.Base(filename), suffix) 130 | return parser(data, symbol) 131 | } 132 | 133 | // Parsers (Performance Critical) 134 | 135 | // processDayFile 解析日线数据 (.day) 136 | func processDayFile(data []byte, symbol string) ([]model.StockData, error) { 137 | n := len(data) 138 | if n%recordSize != 0 { 139 | return nil, fmt.Errorf("invalid file size: %d", n) 140 | } 141 | count := n / recordSize 142 | rows := make([]model.StockData, 0, count) // 预分配内存 143 | 144 | var offset int 145 | for i := 0; i < count; i++ { 146 | offset = i * recordSize 147 | 148 | // 格式布局 (32字节): 149 | // 00-03: Date (uint32 YYYYMMDD) 150 | // 04-07: Open (uint32 / 100) 151 | // 08-11: High (uint32 / 100) 152 | // 12-15: Low (uint32 / 100) 153 | // 16-19: Close (uint32 / 100) 154 | // 20-23: Amount (float32) 155 | // 24-27: Volume (uint32) 156 | // 28-31: Reserved 157 | 158 | dateRaw := binary.LittleEndian.Uint32(data[offset : offset+4]) 159 | openRaw := binary.LittleEndian.Uint32(data[offset+4 : offset+8]) 160 | highRaw := binary.LittleEndian.Uint32(data[offset+8 : offset+12]) 161 | lowRaw := binary.LittleEndian.Uint32(data[offset+12 : offset+16]) 162 | closeRaw := binary.LittleEndian.Uint32(data[offset+16 : offset+20]) 163 | 164 | // Amount 是 float32，需要 bits 转换 165 | amountBits := binary.LittleEndian.Uint32(data[offset+20 : offset+24]) 166 | amount := math.Float32frombits(amountBits) 167 | 168 | volRaw := binary.LittleEndian.Uint32(data[offset+24 : offset+28]) 169 | 170 | t, err := parseDate(dateRaw) 171 | if err != nil { 172 | continue // 忽略错误行 173 | } 174 | 175 | rows = append(rows, model.StockData{ 176 | Symbol: symbol, 177 | Open: float64(openRaw) / 100.0, 178 | High: float64(highRaw) / 100.0, 179 | Low: float64(lowRaw) / 100.0, 180 | Close: float64(closeRaw) / 100.0, 181 | Amount: float64(amount), 182 | Volume: int64(volRaw), 183 | Date: t, 184 | }) 185 | } 186 | return rows, nil 187 | } 188 | 189 | // processMinFile 解析分钟数据 (.01 / .5) 190 | func processMinFile(data []byte, symbol string) ([]model.StockMinData, error) { 191 | n := len(data) 192 | if n%recordSize != 0 { 193 | return nil, fmt.Errorf("invalid file size: %d", n) 194 | } 195 | count := n / recordSize 196 | rows := make([]model.StockMinData, 0, count) 197 | 198 | var offset int 199 | for i := 0; i < count; i++ { 200 | offset = i * recordSize 201 | 202 | // 格式布局 (32字节): 203 | // 00-01: Date (uint16 compressed) 204 | // 02-03: Time (uint16 compressed) 205 | // 04-07: Open ... 206 | 207 | dateRaw := binary.LittleEndian.Uint16(data[offset : offset+2]) 208 | timeRaw := binary.LittleEndian.Uint16(data[offset+2 : offset+4]) 209 | openRaw := binary.LittleEndian.Uint32(data[offset+4 : offset+8]) 210 | highRaw := binary.LittleEndian.Uint32(data[offset+8 : offset+12]) 211 | lowRaw := binary.LittleEndian.Uint32(data[offset+12 : offset+16]) 212 | closeRaw := binary.LittleEndian.Uint32(data[offset+16 : offset+20]) 213 | 214 | amountBits := binary.LittleEndian.Uint32(data[offset+20 : offset+24]) 215 | amount := math.Float32frombits(amountBits) 216 | 217 | volRaw := binary.LittleEndian.Uint32(data[offset+24 : offset+28]) 218 | 219 | t, err := parseDateTime(dateRaw, timeRaw) 220 | if err != nil { 221 | continue 222 | } 223 | 224 | rows = append(rows, model.StockMinData{ 225 | Symbol: symbol, 226 | Open: float64(openRaw) / 100.0, 227 | High: float64(highRaw) / 100.0, 228 | Low: float64(lowRaw) / 100.0, 229 | Close: float64(closeRaw) / 100.0, 230 | Amount: float64(amount), 231 | Volume: int64(volRaw), 232 | Datetime: t, 233 | }) 234 | } 235 | return rows, nil 236 | } 237 | 238 | // Helpers 239 | 240 | func parseDate(d uint32) (time.Time, error) { 241 | year := int(d / 10000) 242 | month := int((d % 10000) / 100) 243 | day := int(d % 100) 244 | // 基本校验 245 | if year < 1900 || month < 1 || month > 12 || day < 1 || day > 31 { 246 | return time.Time{}, fmt.Errorf("invalid date") 247 | } 248 | return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local), nil 249 | } 250 | 251 | func parseDateTime(dateRaw, timeRaw uint16) (time.Time, error) { 252 | // 通达信分钟线时间压缩算法 253 | year := int(dateRaw)/2048 + 2004 254 | month := (int(dateRaw) % 2048) / 100 255 | day := (int(dateRaw) % 2048) % 100 256 | hour := int(timeRaw) / 60 257 | minute := int(timeRaw) % 60 258 | 259 | if month < 1 || month > 12 || day < 1 || day > 31 { 260 | return time.Time{}, fmt.Errorf("invalid date") 261 | } 262 | return time.Date(year, time.Month(month), day, hour, minute, 0, 0, time.Local), nil 263 | } 264 | 265 | func collectFiles(root string, prefixes []string, suffix string) ([]string, error) { 266 | var files []string 267 | err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { 268 | if err != nil { 269 | return err 270 | } 271 | if !d.IsDir() && strings.HasSuffix(path, suffix) { 272 | fname := filepath.Base(path) 273 | symbol := strings.TrimSuffix(fname, suffix) 274 | 275 | match := false 276 | if len(prefixes) == 0 { 277 | match = true 278 | } else { 279 | for _, p := range prefixes { 280 | if strings.HasPrefix(symbol, p) { 281 | match = true 282 | break 283 | } 284 | } 285 | } 286 | 287 | if match { 288 | files = append(files, path) 289 | } 290 | } 291 | return nil 292 | }) 293 | return files, err 294 | } 295 | -------------------------------------------------------------------------------- /calc/fq.go: -------------------------------------------------------------------------------- 1 | package calc 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "sort" 7 | "sync" 8 | "time" 9 | 10 | "github.com/jing2uo/tdx2db/database" 11 | "github.com/jing2uo/tdx2db/model" 12 | "github.com/jing2uo/tdx2db/utils" 13 | ) 14 | 15 | // internalCombinedData 内部用于合并数据的结构体 16 | type internalCombinedData struct { 17 | Date time.Time 18 | Symbol string 19 | Close float64 20 | PreClose float64 21 | IsTradeDay bool 22 | Fenhong float64 23 | Peigu float64 24 | Peigujia float64 25 | Songzhuangu float64 26 | } 27 | 28 | var factorConcurrency = runtime.NumCPU() 29 | 30 | type batchData[T any] struct { 31 | Rows []T 32 | Err error 33 | } 34 | 35 | func ExportFactorsToCSV(db database.DataRepository, xdxrData []model.XdxrData, csvPath string) error { 36 | // 1. 准备数据：构建索引与获取代码列表 37 | xdxrIndex, err := buildXdxrIndex(xdxrData) 38 | if err != nil { 39 | return fmt.Errorf("failed to build XDXR index: %w", err) 40 | } 41 | 42 | symbols, err := db.GetAllSymbols() 43 | if err != nil { 44 | return fmt.Errorf("failed to query all stock symbols: %w", err) 45 | } 46 | 47 | cw, err := utils.NewCSVWriter[model.Factor](csvPath) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | // 3. 并发管道设置 53 | // Channel 用于传递处理好的一只股票的所有因子数据 54 | batchChan := make(chan batchData[model.Factor], factorConcurrency*2) 55 | sem := make(chan struct{}, factorConcurrency) 56 | 57 | var producerWg sync.WaitGroup 58 | var consumerWg sync.WaitGroup 59 | 60 | // 错误收集器 61 | var errors []string 62 | var errMu sync.Mutex 63 | collectError := func(e error) { 64 | errMu.Lock() 65 | errors = append(errors, e.Error()) 66 | errMu.Unlock() 67 | } 68 | 69 | // Consumer: 写入 CSV 70 | consumerWg.Add(1) 71 | go func() { 72 | defer consumerWg.Done() 73 | defer cw.Close() // 确保全部写完后关闭文件 74 | 75 | for batch := range batchChan { 76 | if batch.Err != nil { 77 | collectError(batch.Err) 78 | continue 79 | } 80 | if len(batch.Rows) > 0 { 81 | if err := cw.Write(batch.Rows); err != nil { 82 | collectError(fmt.Errorf("csv write error: %w", err)) 83 | } 84 | } 85 | } 86 | }() 87 | 88 | // --- Producer: 并发查询与计算 --- 89 | for _, symbol := range symbols { 90 | producerWg.Add(1) 91 | 92 | go func() { 93 | sem <- struct{}{} // 获取令牌 94 | defer func() { 95 | <-sem // 释放令牌 96 | producerWg.Done() 97 | }() 98 | 99 | // 执行业务逻辑 100 | rows, err := processStockFactor(db, xdxrIndex, symbol) 101 | 102 | // 发送结果到 Consumer 103 | batchChan <- batchData[model.Factor]{ 104 | Rows: rows, 105 | Err: err, 106 | } 107 | }() 108 | } 109 | 110 | // 等待所有生产者完成 -> 关闭通道 -> 等待消费者完成 111 | producerWg.Wait() 112 | close(batchChan) 113 | consumerWg.Wait() 114 | 115 | // 4. 返回结果 116 | if len(errors) > 0 { 117 | return fmt.Errorf("export completed with %d errors, first: %s", len(errors), errors[0]) 118 | } 119 | return nil 120 | } 121 | 122 | // processStockFactor 将具体的业务逻辑抽离，保持主流程清晰 123 | func processStockFactor(db database.DataRepository, xdxrIndex map[string][]model.XdxrData, symbol string) ([]model.Factor, error) { 124 | // 优化建议：确保 SQL 语句带上 ORDER BY date ASC， 125 | stockData, err := db.QueryStockData(symbol, nil, nil) 126 | if err != nil { 127 | return nil, fmt.Errorf("symbol %s query failed: %w", symbol, err) 128 | } 129 | 130 | if len(stockData) == 0 { 131 | return nil, nil 132 | } 133 | 134 | xdxr := getXdxrBySymbol(xdxrIndex, symbol) 135 | factors, err := CalculateFqFactor(stockData, xdxr) 136 | if err != nil { 137 | return nil, fmt.Errorf("symbol %s calc failed: %w", symbol, err) 138 | } 139 | 140 | return factors, nil 141 | } 142 | 143 | type XdxrIndex map[string][]model.XdxrData 144 | 145 | func buildXdxrIndex(xdxrData []model.XdxrData) (XdxrIndex, error) { 146 | index := make(XdxrIndex) 147 | 148 | for _, data := range xdxrData { 149 | symbol := data.Symbol 150 | index[symbol] = append(index[symbol], data) 151 | } 152 | 153 | return index, nil 154 | } 155 | 156 | func getXdxrBySymbol(index XdxrIndex, symbol string) []model.XdxrData { 157 | if data, exists := index[symbol]; exists { 158 | return data 159 | } 160 | return []model.XdxrData{} 161 | } 162 | 163 | func CalculateFqFactor(stockData []model.StockData, xdxrData []model.XdxrData) ([]model.Factor, error) { 164 | // 如果 xdxrData 为空，说明没有除权除息事件，采用快速路径处理。 165 | // 此时，前复权和后复权的因子均为 1.0。 166 | if len(xdxrData) == 0 { 167 | result := make([]model.Factor, 0, len(stockData)) 168 | if len(stockData) == 0 { 169 | return result, nil 170 | } 171 | 172 | // 直接生成结果，复权因子全部为 1.0 173 | result = append(result, model.Factor{ 174 | Symbol: stockData[0].Symbol, 175 | Date: stockData[0].Date, 176 | Close: stockData[0].Close, 177 | PreClose: stockData[0].Close, // 第一天的 PreClose 就是当天的 Close 178 | QfqFactor: 1.0, 179 | HfqFactor: 1.0, 180 | }) 181 | 182 | for i := 1; i < len(stockData); i++ { 183 | result = append(result, model.Factor{ 184 | Symbol: stockData[i].Symbol, 185 | Date: stockData[i].Date, 186 | Close: stockData[i].Close, 187 | PreClose: stockData[i-1].Close, 188 | QfqFactor: 1.0, 189 | HfqFactor: 1.0, 190 | }) 191 | } 192 | return result, nil 193 | } 194 | 195 | // 当 xdxrData 不为空时，执行完整复权计算逻辑 196 | combined, err := calculatePreClose(stockData, xdxrData) 197 | if err != nil { 198 | return nil, err 199 | } 200 | if len(combined) < 2 { 201 | return []model.Factor{}, nil 202 | } 203 | 204 | n := len(combined) 205 | 206 | // --- 1. 计算前复权因子 (QFQ) --- 207 | // 逻辑：基于 (pre_close.shift(-1) / close) 的倒序累乘 208 | qfqRatios := make([]float64, n) 209 | for i := 0; i < n-1; i++ { 210 | if combined[i].IsTradeDay && combined[i].Close != 0 { 211 | qfqRatios[i] = combined[i+1].PreClose / combined[i].Close 212 | } else { 213 | qfqRatios[i] = 1.0 214 | } 215 | } 216 | qfqRatios[n-1] = 1.0 // 最后一天的比率是1 217 | 218 | qfqFactors := make([]float64, n) 219 | accQfq := 1.0 220 | for i := n - 1; i >= 0; i-- { 221 | accQfq *= qfqRatios[i] 222 | qfqFactors[i] = accQfq 223 | } 224 | 225 | // --- 2. 计算后复权因子 (HFQ) --- 226 | // 逻辑：基于 (close / pre_close.shift(-1)) 的正序累乘，并向下平移一位 227 | hfqFactors := make([]float64, n) 228 | if n > 0 { 229 | hfqFactors[0] = 1.0 // 第一个因子总是 1 230 | accHfq := 1.0 231 | for i := 0; i < n-1; i++ { 232 | var hfqRatio float64 233 | if combined[i+1].PreClose != 0 { 234 | hfqRatio = combined[i].Close / combined[i+1].PreClose 235 | } else { 236 | hfqRatio = 1.0 237 | } 238 | accHfq *= hfqRatio 239 | hfqFactors[i+1] = accHfq 240 | } 241 | } 242 | 243 | // --- 3. 组装最终结果 --- 244 | result := make([]model.Factor, 0, len(stockData)) 245 | for i, data := range combined { 246 | // 只返回实际交易日的数据 247 | if data.IsTradeDay { 248 | result = append(result, model.Factor{ 249 | Symbol: data.Symbol, 250 | Date: data.Date, 251 | Close: data.Close, 252 | PreClose: data.PreClose, 253 | QfqFactor: qfqFactors[i], 254 | HfqFactor: hfqFactors[i], 255 | }) 256 | } 257 | } 258 | return result, nil 259 | } 260 | 261 | func calculatePreClose(stockData []model.StockData, xdxrData []model.XdxrData) ([]*internalCombinedData, error) { 262 | if len(stockData) == 0 { 263 | return []*internalCombinedData{}, nil 264 | } 265 | 266 | // 1. 数据合并与排序 267 | dataMap := make(map[string]*internalCombinedData) 268 | dateFormat := "2006-01-02" 269 | symbol := stockData[0].Symbol 270 | 271 | for _, sd := range stockData { 272 | dateStr := sd.Date.Format(dateFormat) 273 | dataMap[dateStr] = &internalCombinedData{ 274 | Date: sd.Date, Symbol: sd.Symbol, Close: sd.Close, IsTradeDay: true, 275 | } 276 | } 277 | 278 | for _, xdxr := range xdxrData { 279 | dateStr := xdxr.Date.Format(dateFormat) 280 | if data, exists := dataMap[dateStr]; exists { 281 | data.Fenhong = xdxr.Fenhong 282 | data.Peigu = xdxr.Peigu 283 | data.Peigujia = xdxr.Peigujia 284 | data.Songzhuangu = xdxr.Songzhuangu 285 | } else { 286 | dataMap[dateStr] = &internalCombinedData{ 287 | Date: xdxr.Date, Symbol: symbol, IsTradeDay: false, 288 | Fenhong: xdxr.Fenhong, Peigu: xdxr.Peigu, Peigujia: xdxr.Peigujia, Songzhuangu: xdxr.Songzhuangu, 289 | } 290 | } 291 | } 292 | 293 | combined := make([]*internalCombinedData, 0, len(dataMap)) 294 | for _, v := range dataMap { 295 | combined = append(combined, v) 296 | } 297 | sort.Slice(combined, func(i, j int) bool { return combined[i].Date.Before(combined[j].Date) }) 298 | 299 | if len(combined) == 0 { 300 | return combined, nil 301 | } 302 | 303 | // 2. 向前填充收盘价 304 | var lastClose float64 305 | // 先找到第一个有效的收盘价来初始化 lastClose，防止其为 0 306 | for _, data := range combined { 307 | if data.IsTradeDay && data.Close > 0 { 308 | lastClose = data.Close 309 | break // 找到后立即退出 310 | } 311 | } 312 | 313 | // 使用一个有效的 lastClose 来安全地向前填充 314 | for _, data := range combined { 315 | if data.IsTradeDay && data.Close > 0 { 316 | // 在每个交易日更新 lastClose 317 | lastClose = data.Close 318 | } else { 319 | // 对于非交易日或收盘价为0的异常交易日，用之前有效的收盘价填充 320 | data.Close = lastClose 321 | } 322 | } 323 | 324 | // 3. 应用 A 股复权公式计算 PreClose 325 | if len(combined) > 0 { 326 | combined[0].PreClose = combined[0].Close 327 | } 328 | 329 | for i := 1; i < len(combined); i++ { 330 | prevClose := combined[i-1].Close 331 | currData := combined[i] 332 | 333 | if prevClose == 0 { 334 | currData.PreClose = currData.Close 335 | continue 336 | } 337 | 338 | denominator := 10 + currData.Peigu + currData.Songzhuangu 339 | if denominator == 0 { 340 | // GBBQ 数据异常，但为了健壮性，我们认为价格不变，而不是返回错误中断整个流程 341 | // return nil, fmt.Errorf("division by zero on date %v for symbol %s", currData.Date, currData.Symbol) 342 | currData.PreClose = prevClose 343 | continue 344 | } 345 | 346 | numerator := (prevClose*10 - currData.Fenhong) + (currData.Peigu * currData.Peigujia) 347 | currData.PreClose = numerator / denominator 348 | } 349 | 350 | return combined, nil 351 | } 352 | -------------------------------------------------------------------------------- /cmd/cron.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/jing2uo/tdx2db/calc" 11 | "github.com/jing2uo/tdx2db/database" 12 | "github.com/jing2uo/tdx2db/model" 13 | "github.com/jing2uo/tdx2db/tdx" 14 | "github.com/jing2uo/tdx2db/utils" 15 | ) 16 | 17 | func Cron(dbURI string, minline string) error { 18 | db, err := database.NewDB(dbURI) 19 | if err != nil { 20 | return fmt.Errorf("failed to create database driver: %w", err) 21 | } 22 | 23 | if err := db.Connect(); err != nil { 24 | return fmt.Errorf("failed to connect to database: %w", err) 25 | } 26 | defer db.Close() 27 | 28 | err = UpdateStocksDaily(db) 29 | if err != nil { 30 | return fmt.Errorf("failed to update daily stock data: %w", err) 31 | } 32 | 33 | err = UpdateStocksMinLine(db, minline) 34 | if err != nil { 35 | return fmt.Errorf("failed to update minute-line stock data: %w", err) 36 | } 37 | 38 | err = UpdateGbbqAndFactors(db) 39 | if err != nil { 40 | return fmt.Errorf("failed to update GBBQ: %w", err) 41 | } 42 | fmt.Println("🚀 今日任务执行成功") 43 | return nil 44 | } 45 | 46 | func UpdateStocksDaily(db database.DataRepository) error { 47 | latestDate, err := db.GetLatestDate(model.TableStocksDaily.TableName, "date") 48 | if err != nil { 49 | return fmt.Errorf("failed to get latest date from database: %w", err) 50 | } 51 | fmt.Printf("📅 日线数据最新日期为 %s\n", latestDate.Format("2006-01-02")) 52 | 53 | validDates, err := prepareTdxData(latestDate, "day") 54 | if err != nil { 55 | return fmt.Errorf("failed to prepare tdx data: %w", err) 56 | } 57 | if len(validDates) > 0 { 58 | fmt.Printf("🐢 开始转换日线数据\n") 59 | _, err := tdx.ConvertFilesToCSV(VipdocDir, ValidPrefixes, StockDailyCSV, ".day") 60 | if err != nil { 61 | return fmt.Errorf("failed to convert day files to csv: %w", err) 62 | } 63 | if err := db.ImportDailyStocks(StockDailyCSV); err != nil { 64 | return fmt.Errorf("failed to import stock csv: %w", err) 65 | } 66 | fmt.Println("📊 日线数据导入成功") 67 | } else { 68 | fmt.Println("🌲 日线数据无需更新") 69 | 70 | } 71 | return nil 72 | } 73 | 74 | func UpdateStocksMinLine(db database.DataRepository, minline string) error { 75 | if minline == "" { 76 | return nil 77 | } 78 | 79 | parts := strings.Split(minline, ",") 80 | need1Min := false 81 | need5Min := false 82 | for _, p := range parts { 83 | if p == "1" { 84 | need1Min = true 85 | } 86 | if p == "5" { 87 | need5Min = true 88 | } 89 | } 90 | 91 | var latestDate time.Time 92 | yesterday := Today.AddDate(0, 0, -1) 93 | 94 | if need1Min && need5Min { 95 | 96 | d1, err1 := db.GetLatestDate(model.TableStocks1Min.TableName, "datetime") 97 | is1MinEmpty := (err1 != nil || d1.IsZero()) 98 | 99 | d5, err2 := db.GetLatestDate(model.TableStocks5Min.TableName, "datetime") 100 | is5MinEmpty := (err2 != nil || d5.IsZero()) 101 | 102 | if is1MinEmpty && is5MinEmpty { 103 | fmt.Println("🛑 警告：数据库中没有分时数据") 104 | fmt.Println("🚧 将处理今天的数据，历史请自行导入") 105 | latestDate = yesterday 106 | 107 | } else if !d1.Equal(d5) { 108 | return fmt.Errorf("1分钟数据最新日期[%s] 与 5分钟数据最新日期[%s] 不同。请先单独执行 '1' 或 '5' 保持一致后再使用组合命令。", 109 | d1.Format("2006-01-02"), d5.Format("2006-01-02")) 110 | 111 | } else { 112 | latestDate = d1 113 | fmt.Printf("📅 分时数据最新日期为 %s\n", latestDate.Format("2006-01-02")) 114 | } 115 | 116 | } else { 117 | var typeLabel string 118 | 119 | if need1Min { 120 | latestDate, _ = db.GetLatestDate(model.TableStocks1Min.TableName, "datetime") 121 | typeLabel = "1分钟" 122 | } else { 123 | latestDate, _ = db.GetLatestDate(model.TableStocks5Min.TableName, "datetime") 124 | typeLabel = "5分钟" 125 | } 126 | 127 | if latestDate.IsZero() { 128 | fmt.Printf("🛑 警告：数据库中没有 %s 数据\n", typeLabel) 129 | fmt.Println("🚧 将处理今天的数据，历史请自行导入") 130 | latestDate = yesterday 131 | } else { 132 | fmt.Printf("📅 %s数据最新日期为 %s\n", typeLabel, latestDate.Format("2006-01-02")) 133 | } 134 | } 135 | 136 | validDates, err := prepareTdxData(latestDate, "tic") 137 | if err != nil { 138 | return fmt.Errorf("failed to prepare tdx data: %w", err) 139 | } 140 | 141 | if len(validDates) >= 30 { 142 | return fmt.Errorf("分时数据超过30天未更新，请手动补齐后继续") 143 | 144 | } 145 | 146 | if len(validDates) > 0 { 147 | fmt.Printf("🐢 开始转换分时数据\n") 148 | for _, p := range parts { 149 | switch p { 150 | case "1": 151 | _, err := tdx.ConvertFilesToCSV(VipdocDir, ValidPrefixes, Stock1MinCSV, ".01") 152 | if err != nil { 153 | return fmt.Errorf("failed to convert .01 files to csv: %w", err) 154 | } 155 | if err := db.Import1MinStocks(Stock1MinCSV); err != nil { 156 | return fmt.Errorf("failed to import 1-minute line csv: %w", err) 157 | } 158 | fmt.Println("📊 1分钟数据导入成功") 159 | 160 | case "5": 161 | _, err := tdx.ConvertFilesToCSV(VipdocDir, ValidPrefixes, Stock5MinCSV, ".5") 162 | if err != nil { 163 | return fmt.Errorf("failed to convert .5 files to csv: %w", err) 164 | } 165 | if err := db.Import5MinStocks(Stock5MinCSV); err != nil { 166 | return fmt.Errorf("failed to import 5-minute line csv: %w", err) 167 | } 168 | fmt.Println("📊 5分钟数据导入成功") 169 | } 170 | } 171 | } else { 172 | fmt.Println("🌲 分时数据无需更新") 173 | } 174 | return nil 175 | } 176 | 177 | func UpdateGbbqAndFactors(db database.DataRepository) error { 178 | fmt.Println("🐢 开始下载股本变迁数据") 179 | 180 | gbbqFile, err := getGbbqFile(TempDir) 181 | if err != nil { 182 | return fmt.Errorf("failed to download GBBQ file: %w", err) 183 | } 184 | 185 | gbbqData, xdxrData, err := tdx.DecodeGbbqFile(gbbqFile) 186 | if err != nil { 187 | return fmt.Errorf("failed to decode GBBQ: %w", err) 188 | } 189 | 190 | gbbqCSV := filepath.Join(TempDir, "gbbq.csv") 191 | gbbqCw, _ := utils.NewCSVWriter[model.GbbqData](gbbqCSV) 192 | if err := gbbqCw.Write(gbbqData); err != nil { 193 | return err 194 | } 195 | gbbqCw.Close() 196 | if err := db.ImportGBBQ(gbbqCSV); err != nil { 197 | return fmt.Errorf("failed to import GBBQ csv into database: %w", err) 198 | } 199 | 200 | xdxrCSV := filepath.Join(TempDir, "xdxr.csv") 201 | xdxrCw, _ := utils.NewCSVWriter[model.XdxrData](xdxrCSV) 202 | if err := xdxrCw.Write(xdxrData); err != nil { 203 | return err 204 | } 205 | xdxrCw.Close() 206 | if err := db.ImportXDXR(xdxrCSV); err != nil { 207 | return fmt.Errorf("failed to import XDXR csv into database: %w", err) 208 | } 209 | 210 | fmt.Println("📈 股本变迁数据导入成功") 211 | 212 | fmt.Println("📟 计算所有股票复权因子") 213 | factorCSV := filepath.Join(TempDir, "factors.csv") 214 | 215 | if err := calc.ExportFactorsToCSV(db, xdxrData, factorCSV); err != nil { 216 | return fmt.Errorf("failed to export factor to csv: %w", err) 217 | } 218 | if err := db.ImportAdjustFactors(factorCSV); err != nil { 219 | return fmt.Errorf("failed to import factor data: %w", err) 220 | } 221 | fmt.Println("🔢 复权因子导入成功") 222 | 223 | return nil 224 | } 225 | 226 | func prepareTdxData(latestDate time.Time, dataType string) ([]time.Time, error) { 227 | var dates []time.Time 228 | 229 | for d := latestDate.Add(24 * time.Hour); !d.After(Today); d = d.Add(24 * time.Hour) { 230 | dates = append(dates, d) 231 | } 232 | 233 | if len(dates) == 0 { 234 | return nil, nil 235 | } 236 | 237 | var targetPath, urlTemplate, fileSuffix, dataTypeCN string 238 | 239 | switch dataType { 240 | case "day": 241 | targetPath = filepath.Join(VipdocDir, "refmhq") 242 | urlTemplate = "https://www.tdx.com.cn/products/data/data/g4day/%s.zip" 243 | fileSuffix = "day" 244 | dataTypeCN = "日线" 245 | case "tic": 246 | targetPath = filepath.Join(VipdocDir, "newdatetick") 247 | urlTemplate = "https://www.tdx.com.cn/products/data/data/g4tic/%s.zip" 248 | fileSuffix = "tic" 249 | dataTypeCN = "分时" 250 | default: 251 | return nil, fmt.Errorf("unknown data type: %s", dataType) 252 | } 253 | 254 | if err := os.MkdirAll(targetPath, 0755); err != nil { 255 | return nil, fmt.Errorf("failed to create target directory: %w", err) 256 | } 257 | 258 | fmt.Printf("🐢 开始下载%s数据\n", dataTypeCN) 259 | 260 | validDates := make([]time.Time, 0, len(dates)) 261 | 262 | for _, date := range dates { 263 | dateStr := date.Format("20060102") 264 | url := fmt.Sprintf(urlTemplate, dateStr) 265 | fileName := fmt.Sprintf("%s%s.zip", dateStr, fileSuffix) 266 | filePath := filepath.Join(targetPath, fileName) 267 | 268 | status, err := utils.DownloadFile(url, filePath) 269 | switch status { 270 | case 200: 271 | 272 | fmt.Printf("✅ 已下载 %s 的数据\n", dateStr) 273 | 274 | if err := utils.UnzipFile(filePath, targetPath); err != nil { 275 | fmt.Printf("⚠️ 解压文件 %s 失败: %v\n", filePath, err) 276 | continue 277 | } 278 | 279 | validDates = append(validDates, date) 280 | case 404: 281 | fmt.Printf("🟡 %s 非交易日或数据尚未更新\n", dateStr) 282 | continue 283 | default: 284 | if err != nil { 285 | return nil, nil 286 | } 287 | } 288 | 289 | } 290 | 291 | if len(validDates) > 0 { 292 | endDate := validDates[len(validDates)-1] 293 | switch dataType { 294 | case "day": 295 | if err := tdx.DatatoolCreate(TempDir, "day", endDate); err != nil { 296 | return nil, fmt.Errorf("failed to run DatatoolDayCreate: %w", err) 297 | } 298 | 299 | case "tic": 300 | endDate := validDates[len(validDates)-1] 301 | fmt.Printf("🐢 开始转档分笔数据\n") 302 | if err := tdx.DatatoolCreate(TempDir, "tick", endDate); err != nil { 303 | return nil, fmt.Errorf("failed to run DatatoolTickCreate: %w", err) 304 | } 305 | fmt.Printf("🐢 开始转换分钟数据\n") 306 | if err := tdx.DatatoolCreate(TempDir, "min", endDate); err != nil { 307 | return nil, fmt.Errorf("failed to run DatatoolMinCreate: %w", err) 308 | } 309 | } 310 | } 311 | 312 | return validDates, nil 313 | } 314 | 315 | func getGbbqFile(cacheDir string) (string, error) { 316 | zipPath := filepath.Join(cacheDir, "gbbq.zip") 317 | gbbqURL := "http://www.tdx.com.cn/products/data/data/dbf/gbbq.zip" 318 | if _, err := utils.DownloadFile(gbbqURL, zipPath); err != nil { 319 | return "", fmt.Errorf("failed to download GBBQ zip file: %w", err) 320 | } 321 | 322 | unzipPath := filepath.Join(cacheDir, "gbbq-temp") 323 | if err := utils.UnzipFile(zipPath, unzipPath); err != nil { 324 | return "", fmt.Errorf("failed to unzip GBBQ file: %w", err) 325 | } 326 | 327 | return filepath.Join(unzipPath, "gbbq"), nil 328 | } 329 | -------------------------------------------------------------------------------- /tdx/gbbq_var.go: -------------------------------------------------------------------------------- 1 | package tdx 2 | 3 | // GBBQ 详情映射（事件名称 -> 字段定义） 4 | var CategoryDetail = map[string][]string{ 5 | "除权除息": {"分红", "配股价", "送转股", "配股"}, 6 | "送配股上市": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 7 | "非流通股上市": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 8 | "未知股本变动": {"", "", "", ""}, 9 | "股本变化": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 10 | "增发新股": {"", "增发价", "增发数量", ""}, 11 | "股份回购": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 12 | "增发新股上市": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 13 | "转配股上市": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 14 | "可转债上市": {"前流通盘", "前总股本", "后流通盘", "后总股本"}, 15 | "扩缩股": {"", "", "比例", ""}, 16 | "非流通股缩股": {"", "", "比例", ""}, 17 | "送认购权证": {"行权价", "", "份数", ""}, 18 | "送认沽权证": {"行权价", "", "份数", ""}, 19 | } 20 | 21 | // GBBQ 类型编号映射（编号 -> 事件名称） 22 | var Category = map[string]string{ 23 | "1": "除权除息", 24 | "2": "送配股上市", 25 | "3": "非流通股上市", 26 | "4": "未知股本变动", 27 | "5": "股本变化", 28 | "6": "增发新股", 29 | "7": "股份回购", 30 | "8": "增发新股上市", 31 | "9": "转配股上市", 32 | "10": "可转债上市", 33 | "11": "扩缩股", 34 | "12": "非流通股缩股", 35 | "13": "送认购权证", 36 | "14": "送认沽权证", 37 | } 38 | 39 | var HexKeys = "38 A7 C2 1D E0 6A 17 E2 D1 39 A2 40 9C BA 46 AF 42 C6 FF 05 74 EA DA BB 89 B4 F8 44 AC 89 D7 F2 98 7F B6 BC E4 F7 6B 75 05 04 58 67 79 C8 6D C6 2B 06 96 8C FB 86 06 8B BF D6 E8 E1 87 49 6B 36 C7 18 02 79 53 25 72 72 13 CC 04 0B 90 24 0C DC DB 03 1A D5 2E 04 85 5C 7E 8E BD 02 26 2D BD 06 1B 50 34 99 1B A2 24 04 F2 88 35 C8 89 EA D5 FB 12 24 BB B5 3B 29 CA 14 A6 04 CE A9 A8 58 02 B9 AA E3 97 A3 A6 22 57 BB AD A0 22 5F EB 05 86 11 C3 ED B1 3F 39 C2 36 D1 4A 43 C8 64 4D B0 6E 3A 7C 51 6D F7 8E C6 DF F3 8E A4 1E 74 9D B2 22 05 4D 07 3F 96 7F 97 F9 63 B9 C4 2B 98 75 F6 D6 84 56 DC 15 D3 52 8B 60 F3 D6 0E A9 AD 07 07 E9 02 86 58 C2 32 9C 90 BC C9 19 BF B0 54 7A F8 CC A8 27 63 82 29 EE FB 98 11 BF 35 29 62 91 93 95 FC F4 F0 08 E4 B2 3A B4 5E B3 B0 2E 3E 20 C1 D7 43 59 7D C6 29 5F 69 74 7F B2 77 E1 0E FA 85 A1 C9 77 73 83 B3 CB 1C 60 DB E9 53 69 FC B3 18 59 15 0F 97 8A 7A C8 83 F5 49 DC 1B 3E 86 C1 95 45 46 E2 16 67 7F 12 35 A0 BB 27 FB CC F8 30 7E 4F C8 6D AB 18 B2 0D 01 CC 79 20 80 7B FA 37 AA 14 9E 85 E8 25 E9 D4 2D 35 4E 8F D3 DE B0 06 8D 15 15 52 65 E8 39 03 28 09 02 67 99 3D 13 BA F3 68 5C 4C 89 B0 E3 6B AE 16 5C 88 25 F8 33 03 19 02 5B 29 7B 2A 41 2D 75 49 48 9B B3 B6 B3 BF AA DF 8C 95 FE 0F 13 B8 7B 02 BB 52 E1 1C 34 C3 9B 87 59 E2 46 CC 22 77 4B D7 C4 2C 31 AA 84 7C 44 51 88 15 1A CC AE 40 9D 1F 44 97 29 98 45 60 74 47 A1 0D A5 73 F0 53 FF 01 F9 F4 9A F1 36 07 D0 2D A0 79 2D 81 23 25 AD 4B 9C C8 BC 12 55 4D D4 BB 95 B1 B9 BE 7D A6 E6 A0 53 BA 83 8C DD 7E E9 4B ED BA 28 42 D8 FF 98 69 35 CA 4E 9C 9D 57 D6 CF A0 89 5C A2 E7 54 D2 AF 4C FB 54 C4 B4 4F C3 BA F8 A2 58 69 19 79 0E A8 0E 3D C8 04 FD 26 32 C8 E1 02 8B A7 1C C3 91 25 E5 D8 49 DB DF 19 5F 16 F5 A7 8B 18 23 04 D4 BF FB 44 C4 61 7C 79 6E C8 90 15 B5 EB 50 87 CA 7A 69 47 2F AF A8 B5 A2 8A 84 C4 41 79 E8 DE 0C AC D0 D5 6F 34 C6 CB A7 76 F9 00 24 42 05 26 7E 7B 14 86 59 7B DB 1C 62 D5 B7 3E F7 17 44 27 4B D2 C6 6F FF C8 49 55 AD 65 52 2D 43 C2 33 9B 63 AB 3D 54 54 28 E2 02 65 03 9A 03 4B 8F 64 1A 92 52 DE 32 D6 2B F0 BE BE 1D 54 B1 7C 70 41 9B 90 55 DA 71 55 21 B9 B6 68 90 19 5F BC AA B4 55 0E E6 81 4C A3 BE BC 64 D7 59 00 59 BD 0F 6A 57 1A A6 A0 D5 1A 0A 80 D3 09 06 73 5A 51 E2 DD 29 66 AC A0 86 29 21 2B 7A 6D 9E 3A 68 D0 A3 DC A7 2B 85 A0 4C D4 F0 C5 C4 43 E4 CF 0C 19 81 30 B6 F6 BE 71 F5 AC 25 AA CF 42 90 06 64 1B 45 29 FD 3A A3 B6 0B 9D 29 9F FA 31 B8 6D D8 EC 43 F5 92 7E 35 22 E0 C3 D3 09 06 61 71 DA E8 36 0A 19 F6 23 81 CB 89 E0 67 6E FE B1 E6 47 72 63 5C 25 18 E0 B4 65 85 EF B5 1B 26 23 90 89 CC EE E3 01 77 95 63 DF C4 AC BF E6 37 14 99 15 49 8A 96 02 91 AA 1D 98 21 57 5E 87 96 C7 B5 87 08 3F 58 06 52 58 17 8F AB A8 4E A1 7A 60 B1 69 5E 9C BE E2 D0 C5 12 59 DF 31 EB D2 19 54 96 E2 10 11 8E 68 B4 1A 2D D3 2F AB 12 F7 FE F3 A7 F7 61 FC F7 7C CB FC 87 8C 6A 10 40 29 7B 30 D6 0D 13 4C 71 CD 5E AB 36 A2 F1 4C 05 ED 53 88 E5 FF 8E 71 79 5D B5 AF D3 67 6D C4 44 6B AB C1 A7 AA 38 D8 70 1E 08 E6 D2 36 7B 88 11 96 DB D2 68 D9 FF D8 50 2B 3A A9 CC 45 1A CA CD D2 05 C6 FC A0 35 0C EE 98 2B 5C B2 39 6A 27 12 8F 97 EC CB 7B B6 C0 27 F6 A7 48 75 09 82 98 CA 3A 5D E3 96 0C A5 D2 B3 6C A4 D1 1F AE 99 67 B0 3D D6 9A 7A 3E 00 8B FD 45 32 F7 9F 28 7C 94 03 DB 64 AA 44 80 D2 27 AF B3 73 87 57 31 EB 08 D9 BA 73 4D 2C 77 03 BF F5 0F 47 3C 22 DA 3F B9 F1 9A 1B 22 83 16 EE F4 18 FC 08 E8 3B 30 1C 04 50 AA 4C E3 28 53 AB DE F8 5F 32 D9 E1 78 7B F1 C5 A8 CA 85 B6 9F 89 1F 40 B8 2C 88 D7 C1 66 34 45 D6 46 FD 7B F3 72 A3 32 55 23 CF B5 B0 79 AB A0 F1 00 5C DB EE 3F 51 AA AE C0 89 8E 47 A5 30 4E 4B DD D6 AE D8 6D 40 1C 4E 8E FB 0C 60 8D 54 1E 2F 17 B7 3A ED DE DC 81 F5 72 85 B7 A6 39 31 6F 47 50 84 43 C5 11 F3 6A 26 8E BA 7F 81 98 31 FD 13 6B 83 C9 11 61 48 64 FA E3 F5 39 2C 12 11 C1 6D 4D 03 13 A6 C2 E0 DF F5 32 8E 5B 35 A7 7F 08 F7 85 27 0D 71 9D B8 CE 9C 1E BA 77 3A F6 A1 A7 26 94 29 C0 20 10 65 75 6E EF AA 32 0C 66 91 3A 4E 0E 74 E2 8A FE B6 F8 17 C7 A7 E4 D8 35 67 2E F0 83 A8 9F A6 28 13 40 A3 96 DC 49 83 55 E1 85 AB BD 4D ED 88 FA 36 69 A9 77 59 5A 9C D0 A0 B1 3D EB 31 16 DC 3E 29 7B 39 01 5B D4 FF 5C E5 9E DA F7 55 D5 3F E3 3B 51 76 83 8E 40 AE E1 2E E8 3E F8 08 B7 B0 24 26 91 AD 82 4C 2E 2F 37 7A 34 A1 05 BD 8C 9A 75 52 5C CD 59 80 CB 92 F8 B1 F8 A5 F2 2C 9F 4A 59 BF EF 76 A3 74 4F E1 C9 7C 7F 91 D9 0D 12 05 B2 8E D0 E0 BB 46 D4 5C 44 2F 65 6D 7A 1C 02 86 FB 7E 7D B6 2A 57 B9 DB 80 CD 02 BF E7 9E 35 21 FB BE 28 13 82 9F F0 74 F7 92 55 DE F2 7B F2 F2 7D F5 A0 14 0F 99 4D 25 F4 DC 11 17 7A 77 65 77 CC BE EF 90 88 E8 FD B2 4E 8E F5 26 FE 53 5D 65 A9 74 47 0B CB E9 E8 71 95 95 87 6C FD 86 94 A7 E5 FC 20 00 1E 0A 0A E3 85 17 24 D4 D0 73 8A 11 1E 1E EF 83 E3 D7 E1 BF CC 98 07 6D 70 37 3A 8F 31 17 55 4E 60 A8 C8 AB 4F 08 2D 37 76 E6 2B 58 DD 81 0F D1 6E 9A A6 55 3D 80 82 99 9E 2D 16 9A DF 4E CB 3B 5D DA A8 53 08 C7 FF 54 DD C6 11 31 1A B6 EB A3 03 08 4A FB B4 45 EC C0 7C 0D C6 CF CB 1B 78 46 88 8F F4 6A 15 62 2F 17 12 E6 41 64 76 58 96 78 DB 29 B5 6A AE DE 63 41 6F BE 9B 37 6C C9 D0 EC 1B F6 79 17 9E FE 79 0E B1 82 28 F2 06 15 C2 BE 96 9C E0 81 80 D7 00 DB 95 87 4B C0 0D 91 55 5B 1F 86 22 64 74 EA 1B 89 85 D2 DD F7 9F F1 D9 09 06 64 FA 6D 59 72 EF CE 66 A7 03 D1 99 E8 DF AE D7 63 5F 60 5F AB 6E C5 22 C8 3A 94 6A 3B 00 72 F8 DB 90 E7 05 DC A2 89 0F 83 AA 03 FE 42 14 1C 8A E6 1C 9E DB D8 D0 CA 97 21 6C AD ED 0A E0 A2 9E EC C1 FF D1 B4 8A 9A AD AB 34 0B 13 3F B5 18 8D 85 9E 0D F9 FB AC 21 2E DD 7A DE BF 9F 7E BD BF 84 DF F5 FD 1E BE E1 1F 0F F8 18 9D 73 09 02 29 B7 5B 26 7E 44 75 04 4D B1 AA 2F 3A DB 46 38 12 D1 41 35 91 29 06 DF C9 98 69 92 02 F2 48 12 A9 71 D2 AE 3B 23 6D 1C E2 6B 8B 75 87 4A 13 A7 1F 81 4D 29 65 53 0A 3A 34 CE 6D E6 31 8D 7E 4E DD 25 6E 76 44 82 3C 47 36 4C B9 C4 9B F4 4F 84 43 11 56 C2 94 53 7E B0 2E 36 DA EB 77 5F C1 64 E2 CA 9F BE 29 D8 06 36 53 D0 6F 82 19 DA BC 8C 5F 4D 45 E7 21 37 9E 90 A6 D4 33 A8 64 4D EC BC 90 5E FE 8E 8B CA 17 7C FF AC 96 BB 21 CF 3D 24 71 3B C2 A1 74 68 85 CF 32 8E 7F 63 39 C5 E7 8E A5 E0 CD 3A F5 9A B8 FD 43 D4 43 39 08 8E 45 76 5F DF E9 17 54 59 12 ED D0 E9 3D 6F 3F 02 14 8A 0A 47 9A D1 E7 FA 4E A1 41 00 50 EF 60 9D 4D C1 CA 87 98 40 E7 B2 0F 76 C0 9D 71 EF D7 46 93 C1 2B 9F 11 B8 F9 05 AC ED A7 72 6B F5 11 9B 3E 0A 04 21 7D 06 D7 46 76 7B AD AE 9D 95 A6 47 68 05 AD F5 38 7C C7 A5 5A CA B2 CB 48 18 C1 F2 62 55 98 36 39 08 80 C5 28 B1 06 E4 FB 46 11 3C 38 A1 4F 1C FE A1 81 B7 FC DB 94 B0 7A FE B5 74 F1 BB 92 AA FF B0 FE 1E 31 8B C6 BC F0 4F 1A FE 91 C5 7A 9C 73 09 4A 32 90 51 01 8B 12 C0 20 CA 3C CB 14 83 D3 C7 7C 5A 12 79 EE 56 1A 36 C4 09 E2 3E DC E8 CE F1 C1 A1 9E 99 DA 64 4F CF 1E D6 2B 70 27 86 3E CF BE 75 1C 39 9B F9 53 63 C1 6B 58 CC 71 D2 07 41 88 BB 14 70 96 F1 68 CE 13 75 FE F4 A0 C8 85 A2 67 18 49 56 0D 07 94 1D 74 61 89 0C 32 49 9D 0D 94 73 4A AB 1A E9 0F E0 BA B6 4A 34 F9 33 1D B3 71 C2 B8 64 D7 0B CB 19 F7 BD E0 69 3E 24 96 B1 C4 28 09 5F 58 AE 8A C0 83 99 19 64 4D 44 37 55 A6 9B A1 42 50 84 B8 18 29 B5 21 91 58 23 88 EB 8F 13 4A 24 09 EC 0F 6D 7D AF 3E FC F7 F3 9F 34 39 15 C4 84 03 BB 7E 67 39 5F 2A 2C 67 94 F4 A6 B5 02 3F 45 56 79 0C 2A 9B 25 77 67 C2 3B CC F2 71 3B 4F 83 2A 8D 8C 53 0D 18 49 54 CA 58 0E BE 8B 3A 53 74 FC 6F 47 28 07 8E C1 F5 53 D3 34 4B 08 05 FF E9 14 29 40 1B 57 AD 77 EC E8 DA DA 35 55 A7 78 03 56 4C 7C B2 ED 3B B5 61 65 91 DF 41 B4 5D C9 B7 9B 13 82 41 15 D7 B3 6E 1C C8 15 B4 F0 F3 3F 91 4B A1 C8 90 78 91 39 5A 21 55 DA 6A E1 2C BA C9 38 69 F6 AE A8 2B 8C B7 14 C1 35 82 35 A0 78 47 56 C0 9A A7 7F 74 14 64 85 F1 B7 48 BC 55 8C 6A A4 95 1C CB F3 52 F9 54 61 15 27 56 43 D0 27 95 E3 35 AA 39 DC 23 38 DA EF 1F 27 65 3A AB F7 CC BB 25 DB 00 36 34 96 D1 F7 C4 EC 44 37 42 7E 17 18 67 C8 9C 9A 5B 39 08 5C 3C F4 92 F1 16 31 88 FA 12 44 9E 79 27 1C C2 0B 46 AC CD 1F 39 B8 9F 9A 56 34 0A 85 86 C2 B1 B1 9B 31 CE 47 57 05 3E A7 AE 3F 3E 01 2D C5 B9 C1 CB BA AB 0A 2A D2 71 E4 EC F8 0A 71 85 CC A1 CA 6E EF 9D 87 22 38 5D 80 81 F7 1A 6C 31 7B 82 86 BD 7F 10 9D 89 B6 F7 AF E4 41 0D 4F 97 28 80 34 06 3E 19 3A 21 60 ED 54 18 02 0F 2F D5 D5 3B A5 87 01 21 38 1B A6 99 32 28 E9 8D 6F 02 35 60 85 BD 64 C4 B0 26 7E 68 D1 E6 97 B5 32 6E B2 4F EB 06 4C 4D C2 97 8E 6B 30 22 C0 B4 3D 47 93 78 67 AC 27 42 DD 5C 3C 27 ED 0A 6C E4 4A 0D 0F DF 52 63 A6 70 76 09 F0 2E 58 F6 05 B2 DF EE C9 1F CB 1D 11 0C A1 8B 19 26 B8 10 2C 81 48 FF 98 EF 30 36 0C 01 C5 4A D9 AC 05 72 89 C7 3F D6 4D E0 17 BA BA B3 D3 E8 1B 0C 8C C8 DF 6B FE 7E BA 91 FD F6 A0 CB 59 19 B0 01 2F D7 0B A0 62 0F 5F CE 74 B8 EB 42 89 B5 BE CA C9 EF DA 9A BB C6 66 1B E0 65 EE D4 3A CE D9 CC 0E BB 85 50 41 45 01 BA 1B 29 11 6F 34 11 55 03 DD 0C B5 99 56 3A 93 4D 4D 95 6D CE C3 51 E0 15 54 3E FF 2F A3 DA 59 EC 3D 59 2D 62 FC 64 39 D6 7B C8 80 78 1D D7 FD E8 0B 5D 8A ED 1A 9D 98 CB C2 EE 78 47 30 AD 8F 64 A5 82 12 23 DA B3 3E CA 4C 85 7A 80 D5 9F 46 20 D6 EE D1 F9 33 FA 1F C5 9C 8E F9 1E 66 51 A5 46 68 DC B7 7F A8 5A DE E6 18 D7 8C 2B 5D EA A8 EC 6B 8B 48 C1 92 5A C1 B1 6A 5E 37 82 22 4B 6A B6 F0 40 16 89 16 A5 81 F8 D4 1B 20 26 86 35 E5 AD C1 01 6E C9 B5 D0 69 C5 0B 31 08 51 5D 35 FC 74 F5 13 04 7A F4 57 10 53 5B A4 CC 8B 21 82 82 15 4B 8C 3D 6B DA 91 85 CB D6 CF 05 80 D0 F0 CF 0D DF 7A B4 99 C7 F8 D5 4C 76 56 30 E9 65 B6 58 60 C1 C0 39 8A 42 54 BC 4A 48 8B A1 D9 5C 32 05 7A 1C BB 50 51 5B 7F C7 75 2D 68 55 E6 83 7B C3 98 FD E6 D5 B8 DA A8 31 01 78 F5 60 8B 1A D2 FD 51 34 47 FA AF 23 AE E2 DE 15 A7 07 66 69 35 9A 40 61 55 25 98 23 54 2A 50 C9 7D A6 CE 74 F8 19 0C 8E 63 E5 49 2F F9 17 05 FD 39 15 55 F4 B0 91 BF 60 B7 B2 40 2E 7A D3 68 86 C0 FC 38 88 AB B9 03 8A 04 05 1A 9F 61 AE F2 D3 B8 A4 29 F8 51 43 CF 84 26 4A 90 6E 13 27 AF 7B 52 DB F9 00 E8 AE C0 B5 6F 64 03 57 20 59 7C F5 E1 65 A8 47 C3 BD EE 72 2A 85 E2 70 8D EA 9D 98 D4 2A D5 70 A2 E9 76 A2 DA E6 7C B0 F7 14 D9 23 B6 88 C0 B3 6F 42 12 F4 69 0C 15 81 D6 F7 0B B7 1B DF 15 E6 75 63 13 53 B3 20 43 79 90 34 E3 34 48 80 D6 86 BB 45 A2 85 DD F8 23 64 3B D5 68 AB 99 53 34 C6 25 0A 87 73 17 37 56 39 BA 8C 0E 39 24 4B CC AA 98 84 0C 2F 27 E6 E2 AC 86 34 5D 1E 25 AE FD 1E FF 3C 27 AD 26 18 4A 1A E5 09 61 5D 83 5F 2C DC 41 A7 C6 07 55 5B B5 0B 71 FE 86 E7 30 A1 BC 27 AF 5F 24 51 1A DD 20 F6 32 9E 3D 64 6F DC 43 65 2A 80 CB 95 C4 B6 F0 E1 F3 CF 6C F2 C2 9C EA 81 88 0C 2D D2 DA 74 82 C6 A5 1E 98 D3 BC 71 ED E2 0B 05 DA BB 0E FA 35 0A 2C D5 C8 62 E7 B1 AF 95 14 6C 83 7D F1 CE 9F 13 6B D8 68 C9 A5 F5 87 2E A5 8F D7 5C B2 C6 99 37 31 5A A4 D0 E2 43 DF C8 BE BD 10 C0 D8 22 63 95 46 1E E7 8C A8 61 E4 74 02 6C B4 30 F3 06 15 11 E6 2A 3A 0D 3B 2F B9 3B B3 83 40 18 79 FB 39 38 B7 CE 4D BA F6 9E AA E1 8F 32 1C B1 68 DD 5C 2C 37 65 61 73 3D C6 34 56 CD EA BC 77 6A A1 7D 6A F1 F9 78 AF 0F D9 C2 AA D3 D7 A8 2D A8 6E BC 19 83 96 B5 A3 3E B3 B2 5C 54 AD 77 CE 1D E5 D5 AA B3 0D 36 7A 32 7D 5C A3 60 66 8D 84 A0 BD 4F 0F A9 09 89 B8 EC 14 8A 2B 2B 74 8E 75 77 5A 8E B2 51 D0 26 D6 06 8C 9A CA 31 D6 94 17 F0 14 D7 43 1C 82 0C 00 83 E6 75 05 5C 52 AB 0C 38 8F A3 35 77 52 E8 3E 3B CB 48 81 E3 25 B1 A9 40 12 76 4F 16 F1 CE 3D D7 23 89 44 D7 3F 24 7E B7 46 66 C1 16 7A 17 B2 2A 99 F1 AC 3C C9 9D C5 FE 89 BE BF 2C 68 BC 2C A7 F1 C5 2F 26 1E CC D1 AF 7D AA 7D C5 94 4A 4D C4 87 97 2D 2B 6A 5E 5E BF 39 82 18 AB 8C B9 DC 80 83 A1 D1 80 D2 65 FE 2E CC 6A F1 02 84 B2 36 60 37 24 4E 5E 57 AD A5 C5 50 1A 5E A4 5C 31 B6 93 60 57 AC EB ED 65 3F BF EA C7 08 CA 13 00 93 E5 E6 79 F6 37 20 CA B4 6E 39 9E 83 4F 15 8B 15 CD E7 8C 90 93 B0 85 91 9B AE 21 EF 03 D0 A4 B6 2A B4 C6 D3 07 04 92 54 72 8E EC 2E B3 47 6C CE 42 06 7F E0 5B 96 F2 48 8B FA 8F 83 E2 47 10 A5 B7 30 F8 68 B0 FD 02 74 6F 48 71 D7 F1 2E DF A1 52 61 76 99 47 BE 0A 2F F8 F2 69 9D AD 03 FA E6 84 A7 CF 35 7D 8F 5F C5 A6 9B 21 66 35 BC 58 D5 89 B5 E0 9F 11 F0 A8 8A 1F C8 3C 24 B2 B7 F1 6C 8A DB 3B 39 7A CA D0 EF 15 61 22 72 FD FC 02 3D BD 76 35 9E E1 C6 D7 2C B2 59 E1 03 E0 FF 7A 87 03 79 9F 61 AB CC 49 98 C2 41 CF 6E 9B AA 52 9B D0 08 B5 9E 23 F6 C1 39 82 77 16 5D D4 E1 B3 AD A0 0C 58 F8 E2 67 00 6A 0B 4B D2 6C E1 C5 6B 9D BA 3F 40 82 C5 28 B8 C1 60 75 85 EE C4 FA 04 ED 62 64 B6 29 10 67 4B 9B D6 6C 0E 06 62 64 83 CA F0 2F 2D B8 F6 0A D7 D7 6A 1C 58 14 BE 18 60 80 29 02 CD F6 B1 95 A5 6D 2E 27 9C 08 E3 1F C5 C2 07 7F 63 7F DB 82 C6 C6 85 AC A6 D2 4C F1 7F DB 1D CF 86 20 56 60 C0 24 E0 C0 42 0B 4E 00 5F 8B 78 60 FE EA EC 6D 31 93 49 70 EB 2A 45 4F 92 9B 6C 17 28 BB 89 FC C0 07 84 CC AD 1B 85 F2 85 18 5C 3D 5A 60 54 AF 03 9D 9E E4 26 D3 86 AA 0B 7C A3 32 9C C2 0F 3A D4 3E 1F 52 43 A8 31 E9 70 FC 0C B4 7C F5 E3 C7 6F 11 ED 22 4C 0C 1B 82 CB 72 A4 95 28 1A D4 1B E5 C4 6E D7 F1 EC BF 25 2C B8 92 87 A8 D2 15 79 34 39 C0 BE 0D C8 68 2D F2 D3 8E 01 09 3C 48 94 32 69 89 D5 C0 5D E8 2C E6 A6 97 59 4B 9A C6 61 B0 9E DB 81 DC D3 F9 47 34 84 00 CA 87 BE 5D 6D 56 F3 01 02 3B FF FF FF FF 00 00 00 00" 40 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= 2 | filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= 3 | github.com/ClickHouse/ch-go v0.69.0 h1:nO0OJkpxOlN/eaXFj0KzjTz5p7vwP1/y3GN4qc5z/iM= 4 | github.com/ClickHouse/ch-go v0.69.0/go.mod h1:9XeZpSAT4S0kVjOpaJ5186b7PY/NH/hhF8R6u0WIjwg= 5 | github.com/ClickHouse/clickhouse-go/v2 v2.41.0 h1:JbLKMXLEkW0NMalMgI+GYb6FVZtpaMVEzQa/HC1ZMRE= 6 | github.com/ClickHouse/clickhouse-go/v2 v2.41.0/go.mod h1:/RoTHh4aDA4FOCIQggwsiOwO7Zq1+HxQ0inef0Au/7k= 7 | github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= 8 | github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= 9 | github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4= 10 | github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E= 11 | github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= 12 | github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= 13 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 14 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 17 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/duckdb/duckdb-go-bindings v0.1.21 h1:bOb/MXNT4PN5JBZ7wpNg6hrj9+cuDjWDa4ee9UdbVyI= 19 | github.com/duckdb/duckdb-go-bindings v0.1.21/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= 20 | github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21 h1:Sjjhf2F/zCjPF53c2VXOSKk0PzieMriSoyr5wfvr9d8= 21 | github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= 22 | github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21 h1:IUk0FFUB6dpWLhlN9hY1mmdPX7Hkn3QpyrAmn8pmS8g= 23 | github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= 24 | github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21 h1:Qpc7ZE3n6Nwz30KTvaAwI6nGkXjXmMxBTdFpC8zDEYI= 25 | github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= 26 | github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21 h1:eX2DhobAZOgjXkh8lPnKAyrxj8gXd2nm+K71f6KV/mo= 27 | github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= 28 | github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21 h1:hhziFnGV7mpA+v5J5G2JnYQ+UWCCP3NQ+OTvxFX10D8= 29 | github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= 30 | github.com/duckdb/duckdb-go/arrowmapping v0.0.22 h1:8ZVXajhU64MW3M3YAByRuGc7ceuB4UIDJtjgBx3clqU= 31 | github.com/duckdb/duckdb-go/arrowmapping v0.0.22/go.mod h1:KX7D1oNk+5yzy4Kn/ijVgTMjrXoJM6XFDUEO4uZQl5U= 32 | github.com/duckdb/duckdb-go/mapping v0.0.22 h1:t/akbsueKWl228m1PHWKWqNXZ/KeCPLgx9Mj33eMOLo= 33 | github.com/duckdb/duckdb-go/mapping v0.0.22/go.mod h1:a8NUI22rrV4dJE1VngLAmN9kTx9jzGTQwfChpFl/GQw= 34 | github.com/duckdb/duckdb-go/v2 v2.5.0 h1:s8sqyvTsQpVtrhv4tfQYNr870WHzA9BGikVuhm79UKc= 35 | github.com/duckdb/duckdb-go/v2 v2.5.0/go.mod h1:d/bhG7dzhMVSUyn0UqRRs51eGbetz49nDkxh+yHjLZQ= 36 | github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= 37 | github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= 38 | github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= 39 | github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= 40 | github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= 41 | github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= 42 | github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= 43 | github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= 44 | github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= 45 | github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 46 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 47 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 48 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 49 | github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= 50 | github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 51 | github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU= 52 | github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= 53 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 54 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 55 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 56 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 57 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 58 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 59 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 60 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 61 | github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= 62 | github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= 63 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 64 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 65 | github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= 66 | github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= 67 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 68 | github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= 69 | github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= 70 | github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= 71 | github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 72 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 73 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 74 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 75 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 78 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 79 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 80 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 81 | github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= 82 | github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 83 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= 84 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= 85 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= 86 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= 87 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 88 | github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= 89 | github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= 90 | github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= 91 | github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= 92 | github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 93 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 94 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 95 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 96 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 97 | github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= 98 | github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= 99 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 100 | github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= 101 | github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= 102 | github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= 103 | github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= 104 | github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= 105 | github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= 106 | github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 107 | github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= 108 | github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 109 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 110 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 111 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 112 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 113 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 114 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 115 | github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= 116 | github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= 117 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 118 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 119 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 120 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 121 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 122 | github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= 123 | github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= 124 | github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= 125 | github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= 126 | go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= 127 | go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= 128 | go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= 129 | go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= 130 | go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= 131 | go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= 132 | go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= 133 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 134 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 135 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 136 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 137 | golang.org/x/exp v0.0.0-20251017212417-90e834f514db h1:by6IehL4BH5k3e3SJmcoNbOobMey2SLpAF79iPOEBvw= 138 | golang.org/x/exp v0.0.0-20251017212417-90e834f514db/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= 139 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 140 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 141 | golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= 142 | golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= 143 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 144 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 145 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 146 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 147 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 148 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 149 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 150 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 151 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 152 | golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= 153 | golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= 154 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 155 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 156 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 158 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 159 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 160 | golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= 161 | golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 162 | golang.org/x/telemetry v0.0.0-20251022145735-5be28d707443 h1:eE5IhBiTMPgrcTS6Mlh7IG4MdydRrXr2y60Jn/JC6kM= 163 | golang.org/x/telemetry v0.0.0-20251022145735-5be28d707443/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= 164 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 165 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 166 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 167 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 168 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 169 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 170 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 171 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 172 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 173 | golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= 174 | golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= 175 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 176 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 177 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 178 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 179 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= 180 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= 181 | gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= 182 | gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= 183 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 184 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 185 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 186 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 187 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 188 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 189 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 190 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 191 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 192 | --------------------------------------------------------------------------------