├── README.md ├── uniqueid ├── img.png ├── readme.md ├── go.mod ├── shortuuid │ ├── alphabet.go │ └── base57.go ├── snowflake │ └── snowflake.go └── benchmark │ └── uniqueid_test.go └── 2018wuhan_meetup ├── 1缓存数据优化 ├── 11NGINX缓存 │ ├── cachetest.conf │ └── main.go ├── 12NGINX去问号缓存 │ ├── main.go │ └── cachetest.conf ├── 13NGINX lock缓存 │ ├── main.go │ └── cachetest.conf └── 14GO多数据优化 │ └── slice_test.go ├── 6踩坑集锦-channel使用不当 ├── 61channel常规用法 │ └── main.go └── 62channel合理用法 │ └── main.go ├── 2实时数据优化 ├── 21IO类型做异步缓存 │ └── main.go └── 22并发获取数据 │ └── redis_test.go ├── 5踩坑集锦-多重map上锁 ├── 51错误的多重map上锁 │ └── main.go ├── 52错误的多重map上锁 │ └── main.go └── 53正确的多重map上锁 │ └── main.go └── 4踩坑集锦-指针类型串号 └── main.go /README.md: -------------------------------------------------------------------------------- 1 | # gopherlearn 2 | -------------------------------------------------------------------------------- /uniqueid/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/askuy/gopherlearn/HEAD/uniqueid/img.png -------------------------------------------------------------------------------- /uniqueid/readme.md: -------------------------------------------------------------------------------- 1 | * [benchmark](./benchmark/uniqueid_test.go) 2 | * 6种分布式id： https://blog.csdn.net/melovemingming/article/details/109108782 3 | * https://github.com/satori/go.uuid 4 | * https://www.zhihu.com/question/34876910 5 | 6 | ```bash 7 | go test -bench=. -benchmem 8 | ``` 9 | 10 | ![img.png](img.png) -------------------------------------------------------------------------------- /uniqueid/go.mod: -------------------------------------------------------------------------------- 1 | module uniqueid 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/gofrs/uuid v4.2.0+incompatible // indirect 7 | github.com/google/uuid v1.2.0 // indirect 8 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 9 | github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect 10 | github.com/speps/go-hashids v2.0.0+incompatible // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/11NGINX缓存/cachetest.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 8088; 3 | location ~ /hello { 4 | access_log /home/www/logs/hello_access.log; 5 | proxy_pass http://127.0.0.1:9090; 6 | proxy_cache vipcache; 7 | proxy_cache_valid 200 302 20s; 8 | proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504 http_403 http_404; 9 | add_header Cache-Status "$upstream_cache_status"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/11NGINX缓存/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | func main() { 10 | http.Handle("/hello", &ServeMux{}) 11 | err := http.ListenAndServe(":9090", nil) 12 | if err != nil { 13 | fmt.Println("err", err.Error()) 14 | } 15 | } 16 | 17 | type ServeMux struct { 18 | } 19 | 20 | func (p *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { 21 | fmt.Println("get one request") 22 | fmt.Println(r.RequestURI) 23 | io.WriteString(w, "hello world") 24 | } 25 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/12NGINX去问号缓存/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | func main() { 10 | http.Handle("/hello", &ServeMux{}) 11 | err := http.ListenAndServe(":9090", nil) 12 | if err != nil { 13 | fmt.Println("err", err.Error()) 14 | } 15 | } 16 | 17 | type ServeMux struct { 18 | } 19 | 20 | func (p *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { 21 | fmt.Println("get one request") 22 | fmt.Println(r.RequestURI) 23 | io.WriteString(w, "hello world") 24 | } 25 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/12NGINX去问号缓存/cachetest.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 8088; 3 | 4 | if ( $request_uri ~* "^/hello") { 5 | rewrite /hello? /hello? break; 6 | } 7 | 8 | location ~ /hello { 9 | access_log /home/www/logs/hello_access.log; 10 | proxy_pass http://127.0.0.1:9090; 11 | proxy_cache vipcache; 12 | proxy_cache_valid 200 302 20s; 13 | proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504 http_403 http_404; 14 | add_header Cache-Status "$upstream_cache_status"; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/13NGINX lock缓存/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | time "time" 8 | ) 9 | 10 | func main() { 11 | http.Handle("/hello", &ServeMux{}) 12 | err := http.ListenAndServe(":9090", nil) 13 | if err != nil { 14 | fmt.Println("err", err.Error()) 15 | } 16 | } 17 | 18 | type ServeMux struct { 19 | } 20 | 21 | func (p *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { 22 | time.Sleep(5 * time.Second) 23 | 24 | fmt.Println("get one request") 25 | fmt.Println(r.RequestURI) 26 | io.WriteString(w, "hello world") 27 | } 28 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/13NGINX lock缓存/cachetest.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 8088; 3 | 4 | if ( $request_uri ~* "^/hello") { 5 | rewrite /hello? /hello? break; 6 | } 7 | 8 | location ~ /hello { 9 | access_log /home/www/logs/hello_access.log; 10 | proxy_pass http://127.0.0.1:9090; 11 | proxy_cache vipcache; 12 | proxy_cache_valid 200 302 20s; 13 | proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504 http_403 http_404; 14 | 15 | # 当缓存没命中，相同请求只回源一个 16 | proxy_cache_lock on; 17 | 18 | # 等待锁超时时间设置 非必选配置 19 | proxy_cache_lock_timeout 1s; 20 | add_header Cache-Status "$upstream_cache_status"; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /2018wuhan_meetup/6踩坑集锦-channel使用不当/61channel常规用法/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | type monitor struct { 10 | notifyChan chan int 11 | } 12 | 13 | var monitorObj *monitor 14 | 15 | func main() { 16 | monitorObj = &monitor{ 17 | notifyChan: make(chan int, 10), 18 | } 19 | go monitorObj.syncStore() 20 | 21 | t1 := time.Now() 22 | var wg sync.WaitGroup 23 | for i := 0; i < 100; i++ { 24 | wg.Add(1) 25 | go func(i int) { 26 | monitorObj.push(i) 27 | wg.Done() 28 | }(i) 29 | } 30 | wg.Wait() 31 | fmt.Println(time.Since(t1)) 32 | } 33 | 34 | func (m *monitor) push(id int) { 35 | m.notifyChan <- id 36 | } 37 | 38 | func (m *monitor) syncStore() { 39 | for { 40 | select { 41 | case id := <-m.notifyChan: 42 | //time.Sleep(time.Nanosecond) 43 | time.Sleep(time.Microsecond) 44 | fmt.Println(id) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /2018wuhan_meetup/2实时数据优化/21IO类型做异步缓存/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/robfig/cron" 6 | "sync" 7 | ) 8 | 9 | type store struct { 10 | sync.RWMutex 11 | cron *cron.Cron 12 | data map[int]string 13 | } 14 | 15 | var storeObj *store 16 | 17 | func main() { 18 | storeObj = &store{ 19 | data: make(map[int]string, 0), 20 | cron: cron.New(), 21 | } 22 | storeObj.syncStore() 23 | storeObj.cron.AddFunc("*/10 * * * * *", storeObj.syncStore) 24 | storeObj.cron.Start() 25 | 26 | var wg sync.WaitGroup 27 | for i := 1; i < 6; i++ { 28 | wg.Add(1) 29 | go func(wg *sync.WaitGroup, i int) { 30 | fmt.Println(storeObj.Get(i)) 31 | wg.Done() 32 | }(&wg, i) 33 | } 34 | wg.Wait() 35 | } 36 | 37 | func (s *store) syncStore() { 38 | dataTmp := getRemoteData() 39 | s.Lock() 40 | s.data = dataTmp 41 | s.Unlock() 42 | } 43 | 44 | func (s *store) Get(id int) string { 45 | s.RLock() 46 | defer s.RUnlock() 47 | resp, _ := s.data[id] 48 | return resp 49 | } 50 | 51 | func getRemoteData() map[int]string { 52 | resp := make(map[int]string) 53 | resp[1] = "test1" 54 | resp[2] = "test2" 55 | resp[3] = "test3" 56 | resp[4] = "test4" 57 | resp[5] = "test5" 58 | return resp 59 | } 60 | -------------------------------------------------------------------------------- /2018wuhan_meetup/6踩坑集锦-channel使用不当/62channel合理用法/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | type info struct { 10 | id int 11 | time time.Time 12 | } 13 | 14 | type monitor struct { 15 | notifyChan chan info 16 | } 17 | 18 | var monitorObj *monitor 19 | 20 | func main() { 21 | monitorObj = &monitor{ 22 | notifyChan: make(chan info, 10), 23 | } 24 | go monitorObj.syncStore() 25 | 26 | t1 := time.Now() 27 | var wg sync.WaitGroup 28 | for i := 0; i < 100; i++ { 29 | wg.Add(1) 30 | go func(i int) { 31 | monitorObj.push(i) 32 | wg.Done() 33 | }(i) 34 | } 35 | wg.Wait() 36 | fmt.Println(time.Since(t1)) 37 | } 38 | 39 | func (m *monitor) push(id int) { 40 | select { 41 | case m.notifyChan <- info{ 42 | id: id, 43 | time: time.Now(), 44 | }: 45 | default: 46 | // todo something 47 | fmt.Println("todo something") 48 | } 49 | } 50 | 51 | func (m *monitor) syncStore() { 52 | for { 53 | select { 54 | case info := <-m.notifyChan: 55 | if time.Now().Sub(info.time) > 1*time.Second { 56 | fmt.Println("drop this message") 57 | continue 58 | } 59 | 60 | //time.Sleep(time.Microsecond) 61 | //time.Sleep(time.Second) 62 | fmt.Println(info.id) 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /uniqueid/shortuuid/alphabet.go: -------------------------------------------------------------------------------- 1 | package shortuuid 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | // DefaultAlphabet is the default alphabet used. 10 | const DefaultAlphabet = "23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" 11 | 12 | type alphabet struct { 13 | chars [57]string 14 | len int64 15 | } 16 | 17 | // Remove duplicates and sort it to ensure reproducability. 18 | func newAlphabet(s string) alphabet { 19 | abc := dedupe(strings.Split(s, "")) 20 | 21 | if len(abc) != 57 { 22 | panic("encoding alphabet is not 57-bytes long") 23 | } 24 | 25 | sort.Strings(abc) 26 | a := alphabet{ 27 | len: int64(len(abc)), 28 | } 29 | copy(a.chars[:], abc) 30 | return a 31 | } 32 | 33 | func (a *alphabet) Length() int64 { 34 | return a.len 35 | } 36 | 37 | // Index returns the index of the first instance of t in the alphabet, or an 38 | // error if t is not present. 39 | func (a *alphabet) Index(t string) (int64, error) { 40 | for i, char := range a.chars { 41 | if char == t { 42 | return int64(i), nil 43 | } 44 | } 45 | return 0, fmt.Errorf("Element '%v' is not part of the alphabet", t) 46 | } 47 | 48 | // dudupe removes duplicate characters from s. 49 | func dedupe(s []string) []string { 50 | var out []string 51 | m := make(map[string]bool) 52 | 53 | for _, char := range s { 54 | if _, ok := m[char]; !ok { 55 | m[char] = true 56 | out = append(out, char) 57 | } 58 | } 59 | 60 | return out 61 | } 62 | -------------------------------------------------------------------------------- /2018wuhan_meetup/1缓存数据优化/14GO多数据优化/slice_test.go: -------------------------------------------------------------------------------- 1 | package slice_test 2 | 3 | import ( 4 | "strconv" 5 | "sync" 6 | "testing" 7 | ) 8 | 9 | // go test -bench="." 10 | type Something struct { 11 | roomId int 12 | roomName string 13 | } 14 | 15 | func BenchmarkDefaultSlice(b *testing.B) { 16 | b.ReportAllocs() 17 | var wg sync.WaitGroup 18 | for i := 0; i < b.N; i++ { 19 | wg.Add(1) 20 | go func(wg *sync.WaitGroup) { 21 | for i := 0; i < 120; i++ { 22 | output := make([]Something, 0) 23 | output = append(output, Something{ 24 | roomId: i, 25 | roomName: strconv.Itoa(i), 26 | }) 27 | } 28 | wg.Done() 29 | }(&wg) 30 | } 31 | wg.Wait() 32 | } 33 | 34 | func BenchmarkPreAllocSlice(b *testing.B) { 35 | b.ReportAllocs() 36 | var wg sync.WaitGroup 37 | 38 | for i := 0; i < b.N; i++ { 39 | wg.Add(1) 40 | go func(wg *sync.WaitGroup) { 41 | output := make([]Something, 0, 120) 42 | for i := 0; i < 120; i++ { 43 | output = append(output, Something{ 44 | roomId: i, 45 | roomName: strconv.Itoa(i), 46 | }) 47 | } 48 | wg.Done() 49 | }(&wg) 50 | } 51 | wg.Wait() 52 | 53 | } 54 | 55 | func BenchmarkSyncPoolSlice(b *testing.B) { 56 | b.ReportAllocs() 57 | var wg sync.WaitGroup 58 | var SomethingPool = sync.Pool{ 59 | New: func() interface{} { 60 | b := make([]Something, 120) 61 | return &b 62 | }, 63 | } 64 | for i := 0; i < b.N; i++ { 65 | wg.Add(1) 66 | go func(wg *sync.WaitGroup) { 67 | obj := SomethingPool.Get().(*[]Something) 68 | for i := 0; i < 120; i++ { 69 | some := *obj 70 | some[i].roomId = i 71 | some[i].roomName = strconv.Itoa(i) 72 | } 73 | SomethingPool.Put(obj) 74 | wg.Done() 75 | }(&wg) 76 | } 77 | wg.Wait() 78 | } 79 | -------------------------------------------------------------------------------- /2018wuhan_meetup/2实时数据优化/22并发获取数据/redis_test.go: -------------------------------------------------------------------------------- 1 | package redis_test 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | "fmt" 8 | ) 9 | 10 | // go test 11 | func Test_OneRedisData(t *testing.T) { 12 | t1 := time.Now() 13 | for i := 0; i < 120; i++ { 14 | getRemoteOneRedisData(i) 15 | } 16 | fmt.Println("Test_OneRedisData cost: ",time.Since(t1)) 17 | } 18 | 19 | func Test_PipelineRedisData(t *testing.T) { 20 | t1 := time.Now() 21 | ids := make([]int,0, 120) 22 | for i := 0; i < 120; i++ { 23 | ids = append(ids, i) 24 | } 25 | getRemotePipelineRedisData(ids) 26 | fmt.Println("Test_PipelineRedisData cost: ",time.Since(t1)) 27 | 28 | } 29 | 30 | func Test_GoroutinePipelineRedisData(t *testing.T) { 31 | t1 := time.Now() 32 | ids := make([]int,0, 120) 33 | for i := 0; i < 120; i++ { 34 | ids = append(ids, i) 35 | } 36 | getGoroutinePipelineRedisData(ids) 37 | fmt.Println("Test_GoroutinePipelineRedisData cost: ",time.Since(t1)) 38 | } 39 | 40 | func getRemoteOneRedisData(i int) int { 41 | // 模拟单个redis请求，定义为600us 42 | time.Sleep(600 * time.Microsecond) 43 | return i 44 | } 45 | 46 | func getRemotePipelineRedisData(i []int) []int { 47 | length := len(i) 48 | // 使用pipeline的情况下，单个redis数据，为500us 49 | time.Sleep(time.Duration(length)*500*time.Microsecond) 50 | return i 51 | } 52 | 53 | func getGoroutinePipelineRedisData(ids []int) []int { 54 | idsNew := make(map[int][]int, 0) 55 | idsNew[0] = ids[0:30] 56 | idsNew[1] = ids[30:60] 57 | idsNew[2] = ids[60:90] 58 | idsNew[3] = ids[90:120] 59 | 60 | resp := make([]int,0,120) 61 | var wg sync.WaitGroup 62 | for j := 0; j < 4; j++ { 63 | wg.Add(1) 64 | go func(wg *sync.WaitGroup, j int) { 65 | resp = append(resp,getRemotePipelineRedisData(idsNew[j])...) 66 | wg.Done() 67 | }(&wg, j) 68 | } 69 | wg.Wait() 70 | 71 | return resp 72 | } 73 | -------------------------------------------------------------------------------- /2018wuhan_meetup/5踩坑集锦-多重map上锁/51错误的多重map上锁/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/robfig/cron" 6 | "sync" 7 | ) 8 | 9 | type store struct { 10 | cron *cron.Cron 11 | lMap map[int]sync.RWMutex 12 | data map[int]map[int]string 13 | } 14 | 15 | var storeObj *store 16 | 17 | func main() { 18 | storeObj = &store{ 19 | lMap: make(map[int]sync.RWMutex, 0), 20 | data: make(map[int]map[int]string, 0), 21 | cron: cron.New(), 22 | } 23 | storeObj.initStore() 24 | storeObj.syncStore() 25 | storeObj.cron.AddFunc("* * * * * *", storeObj.syncStore) 26 | storeObj.cron.Start() 27 | 28 | var wg sync.WaitGroup 29 | for j := 1; j < 3; j++ { 30 | for i := 1; i < 6; i++ { 31 | wg.Add(1) 32 | go func(wg *sync.WaitGroup, j, i int) { 33 | fmt.Println("j: ", j, " name: ", storeObj.Get(j, i)) 34 | wg.Done() 35 | }(&wg, j, i) 36 | } 37 | } 38 | wg.Wait() 39 | } 40 | 41 | func (s *store) initStore() { 42 | s.lMap[1] = sync.RWMutex{} 43 | s.data[1] = make(map[int]string, 0) 44 | s.lMap[2] = sync.RWMutex{} 45 | s.data[2] = make(map[int]string, 0) 46 | } 47 | 48 | func (s *store) syncStore() { 49 | tmpStore1 := getRemoteData() 50 | tmpStore2 := getRemoteData() 51 | lock1 := s.lMap[1] 52 | lock2 := s.lMap[2] 53 | lock1.Lock() 54 | s.data[1] = tmpStore1 55 | lock1.Unlock() 56 | lock2.Lock() 57 | s.data[2] = tmpStore2 58 | lock2.Unlock() 59 | } 60 | 61 | func (s *store) Get(level int, id int) string { 62 | l, ok := s.lMap[level] 63 | if !ok { 64 | return "" 65 | } 66 | l.RLock() 67 | defer l.RUnlock() 68 | 69 | resp, ok := s.data[level][id] 70 | if !ok { 71 | return "" 72 | } 73 | return resp 74 | } 75 | 76 | func getRemoteData() map[int]string { 77 | resp := make(map[int]string) 78 | resp[1] = "test1" 79 | resp[2] = "test2" 80 | resp[3] = "test3" 81 | resp[4] = "test4" 82 | resp[5] = "test5" 83 | return resp 84 | } 85 | -------------------------------------------------------------------------------- /2018wuhan_meetup/5踩坑集锦-多重map上锁/52错误的多重map上锁/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/robfig/cron" 6 | "sync" 7 | ) 8 | 9 | type store struct { 10 | cron *cron.Cron 11 | data map[int]singleData 12 | } 13 | 14 | type singleData struct { 15 | l sync.RWMutex 16 | data map[int]string 17 | } 18 | 19 | var storeObj *store 20 | 21 | func main() { 22 | storeObj = &store{ 23 | data: make(map[int]singleData, 0), 24 | cron: cron.New(), 25 | } 26 | storeObj.initStore() 27 | storeObj.syncStore() 28 | storeObj.cron.AddFunc("* * * * * *", storeObj.syncStore) 29 | storeObj.cron.Start() 30 | 31 | var wg sync.WaitGroup 32 | for j := 1; j < 3; j++ { 33 | for i := 1; i < 6; i++ { 34 | wg.Add(1) 35 | go func(wg *sync.WaitGroup, j, i int) { 36 | fmt.Println("j: ", j, " name: ", storeObj.Get(j, i)) 37 | wg.Done() 38 | }(&wg, j, i) 39 | } 40 | } 41 | wg.Wait() 42 | } 43 | 44 | func (s *store) initStore() { 45 | s.data[1] = singleData{ 46 | data: make(map[int]string, 0), 47 | } 48 | s.data[2] = singleData{ 49 | data: make(map[int]string, 0), 50 | } 51 | } 52 | 53 | func (s *store) syncStore() { 54 | tmpStore1 := getRemoteData() 55 | tmpStore2 := getRemoteData() 56 | data1 := s.data[1] 57 | data2 := s.data[2] 58 | data1.l.Lock() 59 | data1.data = tmpStore1 60 | data1.l.Unlock() 61 | data2.l.Lock() 62 | data2.data = tmpStore2 63 | data2.l.Unlock() 64 | } 65 | 66 | func (s *store) Get(level int, id int) string { 67 | dataInfo, ok := s.data[level] 68 | if !ok { 69 | return "" 70 | } 71 | dataInfo.l.RLock() 72 | defer dataInfo.l.RUnlock() 73 | 74 | resp, ok := dataInfo.data[id] 75 | if !ok { 76 | return "" 77 | } 78 | return resp 79 | } 80 | 81 | func getRemoteData() map[int]string { 82 | resp := make(map[int]string) 83 | resp[1] = "test1" 84 | resp[2] = "test2" 85 | resp[3] = "test3" 86 | resp[4] = "test4" 87 | resp[5] = "test5" 88 | return resp 89 | } 90 | -------------------------------------------------------------------------------- /2018wuhan_meetup/5踩坑集锦-多重map上锁/53正确的多重map上锁/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/robfig/cron" 6 | "sync" 7 | ) 8 | 9 | type store struct { 10 | cron *cron.Cron 11 | data map[int]*singleData 12 | } 13 | 14 | type singleData struct { 15 | l sync.RWMutex 16 | data map[int]string 17 | } 18 | 19 | var storeObj *store 20 | 21 | func main() { 22 | storeObj = &store{ 23 | data: make(map[int]*singleData, 0), 24 | cron: cron.New(), 25 | } 26 | storeObj.initStore() 27 | storeObj.syncStore() 28 | storeObj.cron.AddFunc("* * * * * *", storeObj.syncStore) 29 | storeObj.cron.Start() 30 | 31 | var wg sync.WaitGroup 32 | for j := 1; j < 3; j++ { 33 | for i := 1; i < 6; i++ { 34 | wg.Add(1) 35 | go func(wg *sync.WaitGroup, j, i int) { 36 | fmt.Println("j: ", j, " name: ", storeObj.Get(j, i)) 37 | wg.Done() 38 | }(&wg, j, i) 39 | } 40 | } 41 | wg.Wait() 42 | } 43 | 44 | func (s *store) initStore() { 45 | s.data[1] = &singleData{ 46 | data: make(map[int]string, 0), 47 | } 48 | s.data[2] = &singleData{ 49 | data: make(map[int]string, 0), 50 | } 51 | } 52 | 53 | func (s *store) syncStore() { 54 | tmpStore1 := getRemoteData() 55 | tmpStore2 := getRemoteData() 56 | data1 := s.data[1] 57 | data2 := s.data[2] 58 | data1.l.Lock() 59 | data1.data = tmpStore1 60 | data1.l.Unlock() 61 | data2.l.Lock() 62 | data2.data = tmpStore2 63 | data2.l.Unlock() 64 | } 65 | 66 | func (s *store) Get(level int, id int) string { 67 | dataInfo, ok := s.data[level] 68 | if !ok { 69 | return "" 70 | } 71 | dataInfo.l.RLock() 72 | defer dataInfo.l.RUnlock() 73 | 74 | resp, ok := dataInfo.data[id] 75 | if !ok { 76 | return "" 77 | } 78 | return resp 79 | } 80 | 81 | func getRemoteData() map[int]string { 82 | resp := make(map[int]string) 83 | resp[1] = "test1" 84 | resp[2] = "test2" 85 | resp[3] = "test3" 86 | resp[4] = "test4" 87 | resp[5] = "test5" 88 | return resp 89 | } 90 | -------------------------------------------------------------------------------- /uniqueid/snowflake/snowflake.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 1 42 52 64 3 | * +-----------------------------------------------+------------+---------------+ 4 | * | timestamp(ms) | workerID | sequence | 5 | * +-----------------------------------------------+------------+---------------+ 6 | * | 0000000000 0000000000 0000000000 0000000000 0 | 0000000000 | 0000000000 00 | 7 | * +-----------------------------------------------+------------+---------------+ 8 | * 9 | * 1. 41位时间截(毫秒级)，注意这是时间截的差值（当前时间截 - 开始时间截)。可以使用约70年: (1L << 41) / (1000L * 60 * 60 * 24 * 365) = 69 10 | * 2. 10位数据机器位，可以部署在1024个节点 11 | * 3. 12位序列，毫秒内的计数，同一机器，同一时间截并发4096个序号 12 | */ 13 | 14 | package snowflake 15 | 16 | import ( 17 | "sync" 18 | "time" 19 | ) 20 | 21 | const ( 22 | twepoch = int64(1483228800000) //开始时间截 (2017-01-01) 23 | workerIdBits = uint(10) //机器id所占的位数 24 | sequenceBits = uint(12) //序列所占的位数 25 | workerIdMax = int64(-1 ^ (-1 << workerIdBits)) //支持的最大机器id数量 26 | sequenceMask = int64(-1 ^ (-1 << sequenceBits)) // 27 | workerIdShift = sequenceBits //机器id左移位数 28 | timestampShift = sequenceBits + workerIdBits //时间戳左移位数 29 | ) 30 | 31 | // A Snowflake struct holds the basic information needed for a snowflake generator worker 32 | type Snowflake struct { 33 | sync.Mutex 34 | timestamp int64 35 | workerID int64 36 | sequence int64 37 | } 38 | 39 | // NewSnowflake returns a new snowflake worker that can be used to generate snowflake IDs 40 | func NewSnowflake(workerID int64) *Snowflake { 41 | 42 | if workerID < 0 || workerID > workerIdMax { 43 | workerID = 101 44 | } 45 | 46 | return &Snowflake{ 47 | timestamp: 0, 48 | workerID: workerID, 49 | sequence: 0, 50 | } 51 | } 52 | 53 | // Generate creates and returns a unique snowflake ID 54 | func (s *Snowflake) Generate() int64 { 55 | s.Lock() 56 | now := time.Now().UnixNano() / 1000000 57 | 58 | if s.timestamp == now { 59 | s.sequence = (s.sequence + 1) & sequenceMask 60 | 61 | if s.sequence == 0 { 62 | for now <= s.timestamp { 63 | now = time.Now().UnixNano() / 1000000 64 | } 65 | } 66 | } else { 67 | s.sequence = 0 68 | } 69 | 70 | s.timestamp = now 71 | 72 | r := (now-twepoch)< 0 { 51 | number, digit = new(big.Int).DivMod(number, big.NewInt(b.alphabet.Length()), new(big.Int)) 52 | out += b.alphabet.chars[digit.Int64()] 53 | } 54 | 55 | if padToLen > 0 { 56 | remainder := math.Max(float64(padToLen-len(out)), 0) 57 | out = out + strings.Repeat(b.alphabet.chars[0], int(remainder)) 58 | } 59 | 60 | return out 61 | } 62 | 63 | // stringToNum converts a string a number using the given alphabet. 64 | func (b *base57) stringToNum(s string) (string, error) { 65 | n := big.NewInt(0) 66 | 67 | for i := len(s) - 1; i >= 0; i-- { 68 | n.Mul(n, big.NewInt(b.alphabet.Length())) 69 | 70 | index, err := b.alphabet.Index(string(s[i])) 71 | if err != nil { 72 | return "", err 73 | } 74 | 75 | n.Add(n, big.NewInt(index)) 76 | } 77 | 78 | x := fmt.Sprintf("%x", n) 79 | 80 | if len(x) < 32 { 81 | // Pad the most significant bit (MSG) with 0 (zero) if the string is too short. 82 | x = strings.Repeat("0", 32-len(x)) + x 83 | } else if len(x) > 32 { 84 | return "", fmt.Errorf("UUID length overflow for %q", s) 85 | } 86 | 87 | return fmt.Sprintf("%s-%s-%s-%s-%s", x[0:8], x[8:12], x[12:16], x[16:20], x[20:32]), nil 88 | } 89 | -------------------------------------------------------------------------------- /2018wuhan_meetup/4踩坑集锦-指针类型串号/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/robfig/cron" 6 | "sync" 7 | ) 8 | 9 | type store struct { 10 | sync.RWMutex 11 | cron *cron.Cron 12 | data map[int]*user 13 | } 14 | 15 | type user struct { 16 | uid int 17 | name string 18 | level int 19 | } 20 | 21 | var storeObj *store 22 | 23 | func main() { 24 | storeObj = &store{ 25 | cron: cron.New(), 26 | data: make(map[int]*user, 0), 27 | } 28 | storeObj.syncStore() 29 | storeObj.cron.AddFunc("* */10 * * * *", storeObj.syncStore) 30 | storeObj.cron.Start() 31 | 32 | // 读模式下问题不大 33 | testCorrectRead() 34 | 35 | // 指针容易串号,并且这种方式是存在数据竞争a 36 | // go run --race main.go 37 | testWrongReadAndWrite() 38 | } 39 | 40 | // 模拟多个http请求获取内存中数据，并只读内存数据 41 | func testCorrectRead() { 42 | fmt.Println("==========内存正确只读模式 开始==========") 43 | var wg sync.WaitGroup 44 | for i := 1; i < 6; i++ { 45 | wg.Add(1) 46 | go func(wg *sync.WaitGroup, i int) { 47 | fmt.Println(storeObj.getUser(i)) 48 | wg.Done() 49 | }(&wg, i) 50 | } 51 | wg.Wait() 52 | fmt.Println("==========内存正确只读模式 结束==========") 53 | } 54 | 55 | // 模拟多个http请求获取内存中数据，并只读内存数据 56 | func testWrongReadAndWrite() { 57 | fmt.Println("==========内存错误的读写模式 开始==========") 58 | var wg sync.WaitGroup 59 | for i := 1; i < 6; i++ { 60 | wg.Add(1) 61 | go func(wg *sync.WaitGroup, i int) { 62 | fmt.Println(storeObj.getUser(i)) 63 | wg.Done() 64 | }(&wg, i) 65 | } 66 | 67 | wg.Add(1) 68 | go func(wg *sync.WaitGroup, i int) { 69 | // 1. 有时候会存在实时计算，调整某个用户级别 70 | // 2. 有的开发会偷懒，拿着内存数据结构进行简单修改，吐给前端 71 | // 3. 这种修改方式，会导致内存中数据串号 72 | user := storeObj.getUser(i) 73 | user.level = 8 74 | fmt.Println("modify user===>", storeObj.getUser(i)) 75 | wg.Done() 76 | }(&wg, 2) 77 | wg.Wait() 78 | fmt.Println("==========内存错误的读写模式 结束==========") 79 | } 80 | 81 | func (s *store) syncStore() { 82 | // 1模拟远端拉去大量用户数据放入到内存缓存中 83 | userList := getRemoterUserList() 84 | 85 | // 2将远端数据进行缓存 86 | tmpStore := make(map[int]*user, 0) 87 | for _, user := range userList { 88 | tmpStore[user.uid] = user 89 | } 90 | s.Lock() 91 | s.data = tmpStore 92 | s.Unlock() 93 | } 94 | 95 | func (s *store) getUser(uid int) *user { 96 | s.RLock() 97 | defer s.RUnlock() 98 | resp, ok := s.data[uid] 99 | if !ok { 100 | return nil 101 | } 102 | return resp 103 | } 104 | 105 | func getRemoterUserList() []*user { 106 | resp := make([]*user, 0) 107 | resp = append(resp, &user{ 108 | 1, "hello1", 1, 109 | }, &user{ 110 | 2, "hello2", 2, 111 | }, &user{ 112 | 3, "hello3", 3, 113 | }, &user{ 114 | 4, "hello4", 4, 115 | }, &user{ 116 | 5, "hello5", 5, 117 | }) 118 | return resp 119 | } 120 | --------------------------------------------------------------------------------