├── .gitignore ├── README.md ├── go.mod ├── go.sum └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .idea 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Transaction Data Persistence 2 | 3 | 4 | ## 问题描述 5 | 交易所需要将所有交易数据持久化到数据库，并支持高并发写入和查询。 6 | 7 | ## 设计要点 8 | * 架构：异步写入，主从数据库分离读写。 9 | * 并发：用`Goroutine`处理批量写入，`Channel`队列化交易数据。 10 | * 一致性：用事务确保数据完整性。 11 | * 性能：批量写入减少数据库压力。 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module tx-data-persistence 2 | 3 | go 1.24.1 4 | 5 | require ( 6 | github.com/go-sql-driver/mysql v1.9.1 7 | github.com/redis/go-redis/v9 v9.7.3 8 | ) 9 | 10 | require ( 11 | filippo.io/edwards25519 v1.1.0 // indirect 12 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 13 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= 2 | filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= 3 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 4 | github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= 5 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 6 | github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= 7 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 8 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 10 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 11 | github.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI= 12 | github.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= 13 | github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= 14 | github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= 15 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | _ "github.com/go-sql-driver/mysql" 14 | "github.com/redis/go-redis/v9" 15 | ) 16 | 17 | // Trade 表示交易数据 18 | type Trade struct { 19 | ID int 20 | Price float64 21 | Quantity int 22 | Timestamp time.Time 23 | } 24 | 25 | // PersistenceManager 管理交易数据持久化 26 | type PersistenceManager struct { 27 | masterDB *sql.DB 28 | slaveDB *sql.DB 29 | redis *redis.Client 30 | tradeChan chan Trade 31 | buffer []Trade 32 | bufferSize int 33 | mu sync.Mutex 34 | ctx context.Context 35 | cancel context.CancelFunc 36 | maxRetries int // 最大重试次数 37 | retryDelay time.Duration // 重试间隔 38 | batchCount int64 // 批量插入次数 39 | batchSuccess int64 // 批量插入成功次数 40 | cacheHits int64 // 缓存命中次数 41 | cacheMisses int64 // 缓存未命中次数 42 | redisWriteChan chan Trade // 异步 Redis 写入通道 43 | } 44 | 45 | // NewPersistenceManager 初始化持久化管理器 46 | func NewPersistenceManager(bufferSize int) *PersistenceManager { 47 | masterDB, err := sql.Open("mysql", "user:password@/exchange?parseTime=true") 48 | if err != nil { 49 | log.Fatal("Master DB error:", err) 50 | } 51 | slaveDB, err := sql.Open("mysql", "user:password@/exchange?parseTime=true") 52 | if err != nil { 53 | log.Fatal("Slave DB error:", err) 54 | } 55 | rdb := redis.NewClient(&redis.Options{ 56 | Addr: "localhost:6379", 57 | }) 58 | ctx, cancel := context.WithCancel(context.Background()) 59 | pm := &PersistenceManager{ 60 | masterDB: masterDB, 61 | slaveDB: slaveDB, 62 | redis: rdb, 63 | tradeChan: make(chan Trade, 1000), 64 | buffer: make([]Trade, 0, bufferSize), 65 | bufferSize: bufferSize, 66 | ctx: ctx, 67 | cancel: cancel, 68 | maxRetries: 3, 69 | retryDelay: 1 * time.Second, 70 | redisWriteChan: make(chan Trade, 1000), 71 | } 72 | go pm.persistTrades() 73 | go pm.asyncRedisWriter() 74 | go pm.monitor() 75 | return pm 76 | } 77 | 78 | // persistTrades 处理交易持久化 79 | func (pm *PersistenceManager) persistTrades() { 80 | ticker := time.NewTicker(1 * time.Second) 81 | defer ticker.Stop() 82 | 83 | for { 84 | select { 85 | case <-pm.ctx.Done(): 86 | pm.flushBuffer() 87 | log.Println("Persistence Goroutine stopped") 88 | return 89 | case trade := <-pm.tradeChan: 90 | pm.mu.Lock() 91 | pm.buffer = append(pm.buffer, trade) 92 | if len(pm.buffer) >= pm.bufferSize { 93 | pm.flushBuffer() 94 | } 95 | pm.mu.Unlock() 96 | pm.redisWriteChan <- trade // 异步写入 Redis 97 | case <-ticker.C: 98 | pm.mu.Lock() 99 | if len(pm.buffer) > 0 { 100 | pm.flushBuffer() 101 | } 102 | pm.mu.Unlock() 103 | } 104 | } 105 | } 106 | 107 | // flushBuffer 批量插入缓冲区数据 108 | func (pm *PersistenceManager) flushBuffer() { 109 | if len(pm.buffer) == 0 { 110 | return 111 | } 112 | 113 | pm.batchCount++ 114 | var placeholders []string 115 | var args []interface{} 116 | for _, trade := range pm.buffer { 117 | placeholders = append(placeholders, "(?, ?, ?, ?)") 118 | args = append(args, trade.ID, trade.Price, trade.Quantity, trade.Timestamp) 119 | } 120 | query := fmt.Sprintf("INSERT INTO trades (id, price, quantity, timestamp) VALUES %s", 121 | strings.Join(placeholders, ",")) 122 | 123 | for i := 0; i < pm.maxRetries; i++ { 124 | _, err := pm.masterDB.ExecContext(pm.ctx, query, args...) 125 | if err == nil { 126 | pm.batchSuccess++ 127 | pm.buffer = pm.buffer[:0] 128 | return 129 | } 130 | log.Printf("Batch insert attempt %d failed: %v", i+1, err) 131 | time.Sleep(pm.retryDelay) 132 | } 133 | log.Println("Batch insert failed after retries") 134 | } 135 | 136 | // asyncRedisWriter 异步写入 Redis 137 | func (pm *PersistenceManager) asyncRedisWriter() { 138 | for { 139 | select { 140 | case <-pm.ctx.Done(): 141 | log.Println("Redis writer Goroutine stopped") 142 | return 143 | case trade := <-pm.redisWriteChan: 144 | pm.cacheTrade(trade) 145 | } 146 | } 147 | } 148 | 149 | // cacheTrade 将交易缓存到 Redis 150 | func (pm *PersistenceManager) cacheTrade(trade Trade) { 151 | data, err := json.Marshal(trade) 152 | if err != nil { 153 | log.Println("Marshal error:", err) 154 | return 155 | } 156 | key := fmt.Sprintf("trade:%d", trade.ID) 157 | for i := 0; i < pm.maxRetries; i++ { 158 | err = pm.redis.Set(pm.ctx, key, data, 1*time.Hour).Err() 159 | if err == nil { 160 | return 161 | } 162 | log.Printf("Redis cache attempt %d failed: %v", i+1, err) 163 | time.Sleep(pm.retryDelay) 164 | } 165 | log.Printf("Redis cache failed for trade %d after retries", trade.ID) 166 | } 167 | 168 | // GetTrade 从 Redis 或从库读取交易 169 | func (pm *PersistenceManager) GetTrade(id int) (*Trade, error) { 170 | key := fmt.Sprintf("trade:%d", id) 171 | 172 | // 先查 Redis 173 | if data, err := pm.redis.Get(pm.ctx, key).Bytes(); err == nil { 174 | var trade Trade 175 | if err := json.Unmarshal(data, &trade); err == nil { 176 | pm.cacheHits++ 177 | return &trade, nil 178 | } 179 | } 180 | pm.cacheMisses++ 181 | 182 | // 从从库查询 183 | var trade Trade 184 | for i := 0; i < pm.maxRetries; i++ { 185 | err := pm.slaveDB.QueryRowContext(pm.ctx, "SELECT id, price, quantity, timestamp FROM trades WHERE id = ?", id). 186 | Scan(&trade.ID, &trade.Price, &trade.Quantity, &trade.Timestamp) 187 | if err == nil { 188 | pm.cacheTrade(trade) // 异步缓存 189 | return &trade, nil 190 | } 191 | log.Printf("DB query attempt %d failed: %v", i+1, err) 192 | time.Sleep(pm.retryDelay) 193 | } 194 | return nil, fmt.Errorf("failed to get trade %d after retries", id) 195 | } 196 | 197 | // monitor 监控批量插入和缓存命中率 198 | func (pm *PersistenceManager) monitor() { 199 | ticker := time.NewTicker(5 * time.Second) 200 | defer ticker.Stop() 201 | 202 | for { 203 | select { 204 | case <-pm.ctx.Done(): 205 | log.Println("Monitor Goroutine stopped") 206 | return 207 | case <-ticker.C: 208 | pm.mu.Lock() 209 | total := pm.cacheHits + pm.cacheMisses 210 | hitRate := 0.0 211 | if total > 0 { 212 | hitRate = float64(pm.cacheHits) / float64(total) * 100 213 | } 214 | log.Printf("Batch: %d total, %d success | Cache: %.2f%% hit rate (%d hits, %d misses)", 215 | pm.batchCount, pm.batchSuccess, hitRate, pm.cacheHits, pm.cacheMisses) 216 | pm.mu.Unlock() 217 | } 218 | } 219 | } 220 | 221 | // Stop 停止持久化管理器 222 | func (pm *PersistenceManager) Stop() { 223 | pm.cancel() 224 | pm.masterDB.Close() 225 | pm.slaveDB.Close() 226 | pm.redis.Close() 227 | } 228 | 229 | func main() { 230 | pm := NewPersistenceManager(5) 231 | 232 | // 模拟交易数据 233 | go func() { 234 | for i := 1; i <= 10; i++ { 235 | pm.tradeChan <- Trade{ID: i, Price: 100.0 + float64(i), Quantity: 10, Timestamp: time.Now()} 236 | time.Sleep(200 * time.Millisecond) 237 | } 238 | }() 239 | 240 | // 模拟查询 241 | time.Sleep(2 * time.Second) 242 | trade, err := pm.GetTrade(3) 243 | if err != nil { 244 | fmt.Println("Get trade error:", err) 245 | } else { 246 | fmt.Printf("Retrieved trade: %+v\n", trade) 247 | } 248 | 249 | // 停止 250 | time.Sleep(5 * time.Second) 251 | pm.Stop() 252 | } 253 | --------------------------------------------------------------------------------