├── xmap ├── README ├── concurrent_hash_map_benchmark_test.go ├── xmap.go ├── concurrent_hash_map_test.go ├── map_test.go ├── entry │ └── rbtree.go └── concurrent_raw_hash_map.go ├── docs ├── README.md ├── img │ ├── README │ ├── xds02.png │ ├── xmap01.png │ ├── xmap02.png │ ├── xmap03.png │ ├── xmap04.png │ ├── xmap05.png │ └── xds-logo01.png └── Xmap-Implement.md ├── example ├── README.md ├── xmap_test1.go ├── xslice_test0.go └── xmap_test0.go ├── go.mod ├── .gitignore ├── xslice ├── xslice_test.go └── xslice.go ├── go.sum ├── common.go ├── README.md └── LICENSE /xmap/README: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/img/README: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/img/xds02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xds02.png -------------------------------------------------------------------------------- /docs/img/xmap01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xmap01.png -------------------------------------------------------------------------------- /docs/img/xmap02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xmap02.png -------------------------------------------------------------------------------- /docs/img/xmap03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xmap03.png -------------------------------------------------------------------------------- /docs/img/xmap04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xmap04.png -------------------------------------------------------------------------------- /docs/img/xmap05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xmap05.png -------------------------------------------------------------------------------- /docs/img/xds-logo01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heiyeluren/xds/HEAD/docs/img/xds-logo01.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/heiyeluren/xds 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/heiyeluren/xmm v0.2.7 7 | github.com/spf13/cast v1.4.1 8 | ) 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .idea -------------------------------------------------------------------------------- /example/xmap_test1.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/heiyeluren/xmm" 6 | "github.com/heiyeluren/xds" 7 | "github.com/heiyeluren/xds/xmap" 8 | ) 9 | 10 | //------------------ 11 | // xmap快速使用案例 12 | //------------------ 13 | 14 | func main() { 15 | 16 | //创建XMM内存块 17 | f := &xmm.Factory{} 18 | mm, err := f.CreateMemory(0.75) 19 | if err != nil { 20 | panic("error") 21 | } 22 | 23 | //用xmap创建一个map，结构类似于 map[string]uint，操作类似于 m: = make(map[string]uint) 24 | m, err := xmap.NewMap(mm, xds.String, xds.Int) 25 | if err != nil { 26 | panic("error") 27 | } 28 | // 调用Set()给xmap写入数据 29 | err = m.Set("k1", 1) 30 | err = m.Set("k2", 2) 31 | err = m.Set("k3", 3) 32 | err = m.Set("k4", 4) 33 | err = m.Set("k5", 5) 34 | 35 | // 调用Get()读取单个数据 36 | ret, exists, err := m.Get("k1") 37 | if exists == false { 38 | panic("key not found") 39 | } 40 | fmt.Printf("Get data key:[%s] value:[%s] \n", "k1", ret) 41 | 42 | // 使用Each()访问所有xmap元素 43 | // 在遍历操作中，让全局变量可以在匿名函数中访问（如果需要使用外部变量，可以像这样） 44 | gi := 1 45 | //调用 Each 遍历函数 46 | m.Each(func(key, val interface{}) error { 47 | //针对每个KV进行操作，比如打印出来 48 | fmt.Printf("For each XMap all key:[%s] value:[%s] \n", key, val) 49 | 50 | //外部变量使用操作 51 | gi++ 52 | return nil 53 | }) 54 | fmt.Printf("For each success, var[gi], val[%s]\n", gi); 55 | 56 | //使用Len()获取xmap元素总数量 57 | len := m.Len() 58 | fmt.Printf("Xmap item size:[%s]\n", len); 59 | 60 | } 61 | -------------------------------------------------------------------------------- /example/xslice_test0.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | // "github.com/heiyeluren/xmm" 6 | "github.com/heiyeluren/xds" 7 | "github.com/heiyeluren/xds/xslice" 8 | ) 9 | 10 | //--------------------- 11 | // xslice快速使用案例 12 | //--------------------- 13 | 14 | func main() { 15 | 16 | fmt.Println("\n--[ Xslice API example]--\n") 17 | 18 | 19 | //创建XSlice对象, 类似于 s0 []Int64 = make([]Int64, 1) 20 | s0 := xslice.NewXslice(xds.Int,1) 21 | 22 | fmt.Println("\n### Xslice - Set/Get OP ###\n") 23 | 24 | //Set压入数据, 类似于 s0 :=[] int { 11, 22, 33 } 25 | s0.Set(1, 11) 26 | s0.Set(2, 22) 27 | s0.Set(3, 33) 28 | 29 | //Get读取一个数据 30 | vi,nil := s0.Get(1) 31 | fmt.Println("XSlice get data 1: ", vi) 32 | vi,nil = s0.Get(2) 33 | fmt.Println("XSlice get data 2: ", vi) 34 | vi,nil = s0.Get(3) 35 | fmt.Println("XSlice get data 3: ", vi) 36 | 37 | //读取整个slice长度, 类似于 len() 38 | fmt.Println("XSlice data size: ", s0.Len(), "\n") 39 | 40 | //释放资源 41 | s0.Free() 42 | 43 | fmt.Println("\n### Xslice - Append/ForEach OP ###\n") 44 | 45 | //批量压入Int ( 类似于 s1 []Int64 = make([]Int64, 1) ) 46 | s1 := xslice.NewXslice(xds.Int, 1) 47 | for i:=50; i<=55; i++ { 48 | s1.Append(i) 49 | } 50 | 51 | //fmt.Println(s1.Get(2)) 52 | 53 | //使用ForEach读取所有数据 54 | s1.ForEach(func(i int, v interface{}) error { 55 | fmt.Println("XSlice foreach data i: ",i, " v: ", v) 56 | return nil 57 | }) 58 | 59 | fmt.Println("\n### Xslice - Append/ForEach Data struct ###\n") 60 | fmt.Println(s1, "\n") 61 | 62 | ////读取整个slice长度, 类似于 len() 63 | fmt.Println("XSlice data size: ", s1.Len(), "\n") 64 | } 65 | -------------------------------------------------------------------------------- /docs/Xmap-Implement.md: -------------------------------------------------------------------------------- 1 | 2 | # Xds - XMap技术设计与实现 3 | 4 | ## 一、XMap 背景目标 5 | 6 | - 名词解释： 7 | ``` 8 | XMap - eXtensible Map Struct（高性能的第三方Map数据结构） 9 | Map - Golang原生map数据结构 10 | sync.Map - Golang原生并发map结构 11 | ``` 12 | 13 | #### XMap设计背景： 14 | 现有Golang中的map数据结构无法解决并发读写问题，Sync.map 并发性能偏差，针对这个情况，为XCache服务需要一个高性能、大容量、高并发、无GC的Map，所以开发实现 XMap。 15 | 针对我们需求调研了市场上主要的 hashmap 结构，不能满足我们性能和功能要求。 16 | 17 |
18 | 19 | #### XMap设计目标： 20 | 要求设计一个可以并发读写不会出现panic，要求并发读写 200w+ OPS/s 的并发map结构。 21 | （写20%，读80%场景；说明：go自带map读写性能在80w ops/s，大量并发读写下可能panic；sync.map 写入性能在 100w OPS/s） 22 | 23 |
24 |
25 | 26 | ## 二、XMap数据结构设计 27 |
28 | 29 | 为了保证读写性能的要求，同时保证内存的友好。我们采用了开链法方式来做为存储结构，如图为基本数据结构： 30 |
31 | 32 | ```go 33 | type ConcurrentHashMap struct { 34 | size uint64 //当前key的数量 35 | threshold uint64 // 扩容阈值 36 | initCap uint64 //初始化bulks大小 37 | 38 | //off-heap 39 | bulks *[]uintptr // 保存链表头结点指针的数组 40 | mm xmm.XMemory // 非堆的对象初始化器 41 | lock sync.RWMutex 42 | 43 | treeSize uint64 // 由链表转为红黑树的阈值 44 | 45 | //resize 46 | sizeCtl int64 // 状态： -1 正在扩容 47 | 48 | nextBulks *[]uintptr //正在扩容的目标Bulks 49 | transferIndex uint64 //扩容的索引 50 | } 51 | ``` 52 | 53 |
54 | 55 | 核心数据结构示例图： 56 | 57 | 58 | 59 | - 核心说明：使用key的hash值计算出数组索引，每个数组中保存一个链表或者红黑树头结点，找到所处的索引，利用尾插发追加节点。每个节点中有key、value等数据。 60 | 61 |
62 |
63 | 64 | ## 三、XMap内部实现流程图 65 | 66 | #### 3.1、Set 存储 67 | 68 | 69 |
70 | 71 | #### 3.2、Get 查询 72 | 73 |
74 | 75 | #### 3.3、Resize 扩容 76 | 77 |
78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /xslice/xslice_test.go: -------------------------------------------------------------------------------- 1 | package xslice 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | 7 | "github.com/heiyeluren/xds" 8 | 9 | ) 10 | 11 | func TestNewXslice(t *testing.T) { 12 | xs := NewXslice(xds.String,10) 13 | log.Println(xs.s) 14 | } 15 | 16 | func TestXslice_Append(t *testing.T) { 17 | xs2 := NewXslice(xds.Int,1) 18 | log.Println(xs2.s) 19 | 20 | for i:=0;i<=30;i++ { 21 | xs2.Append(i) 22 | } 23 | 24 | log.Println(xs2.s) 25 | } 26 | 27 | func TestXslice_Set(t *testing.T) { 28 | xs3 := NewXslice(xds.Int,1) 29 | log.Println(xs3.s) 30 | 31 | err := xs3.Set(1,123) 32 | if err != nil{ 33 | panic(err) 34 | } 35 | 36 | log.Println(xs3.s) 37 | 38 | } 39 | 40 | func TestXslice_Get(t *testing.T) { 41 | xs4 := NewXslice(xds.Int,1) 42 | err := xs4.Set(1,123) 43 | if err != nil{ 44 | panic(err) 45 | } 46 | 47 | v,err := xs4.Get(1) 48 | 49 | if err !=nil{ 50 | panic(err) 51 | } 52 | 53 | log.Println(v) 54 | } 55 | 56 | func TestXslice_Free(t *testing.T) { 57 | xs5 := NewXslice(xds.Int,1) 58 | err := xs5.Set(1,123) 59 | if err != nil{ 60 | panic(err) 61 | } 62 | 63 | log.Println(xs5) 64 | 65 | xs5.Free() 66 | 67 | log.Println(xs5) 68 | } 69 | 70 | func TestXslice_ForEach(t *testing.T) { 71 | xs6 := NewXslice(xds.String,1) 72 | xs6.Append("aaaa") 73 | xs6.Append("bbb") 74 | xs6.Append("cccc") 75 | xs6.Append("cccc") 76 | xs6.Append("cccc") 77 | xs6.Append("cccc") 78 | xs6.Append("cccc") 79 | xs6.Append("cccc") 80 | xs6.Append("cccc") 81 | xs6.Append("eee") 82 | 83 | 84 | xs6.ForEach(func(i int, v []byte) error { 85 | log.Println("i: ",i, "v: ", string(v)) 86 | return nil 87 | }) 88 | } 89 | 90 | func TestXslice_Len(t *testing.T) { 91 | xs8 := NewXslice(xds.Int,1) 92 | log.Println(xs8.s) 93 | 94 | err := xs8.Set(1,123) 95 | if err != nil{ 96 | panic(err) 97 | } 98 | 99 | log.Println(xs8.s) 100 | xs8.Set(2,123) 101 | xs8.Set(3,123) 102 | 103 | log.Println("len: ",xs8.Len()) 104 | } 105 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= 5 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 6 | github.com/heiyeluren/xmm v0.2.7 h1:jC3LX4bd7VezczzCJwDLHFQTUAfW9uVNzQZL0jNQNKk= 7 | github.com/heiyeluren/xmm v0.2.7/go.mod h1:l/H95AVDlcr0eGIbyGb7T7RVUPTKrGGK5knl75nsWk8= 8 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 9 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 10 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 11 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 12 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 13 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 16 | github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= 17 | github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 18 | github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= 19 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 20 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 21 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 22 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 24 | -------------------------------------------------------------------------------- /xslice/xslice.go: -------------------------------------------------------------------------------- 1 | package xslice 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "sync" 7 | 8 | "github.com/heiyeluren/xds" 9 | ) 10 | 11 | var InvalidType = errors.New("type Error") // 类型错误 12 | 13 | type _gap []byte 14 | const _blockSize int = 8 15 | 16 | type Xslice struct{ 17 | s [][_blockSize]_gap //[]*uintptr 改为[]数组。数组内存放基本类型结构体和指针。string的话，存放指针。 18 | 19 | //s []uintptr uintptr地址指向一个连续内存(数组)的头。 20 | // s 的创建和扩容，通过xmm的alloc(元素个数*8)。迁移完后销毁。 21 | // s 中的元素创建：通过xmm的alloc(元素个数*元素单位长度)。append到s中。 22 | // 基本类型存放：int、uint、直接放入到s的元素中。 23 | // 非基本类型存放：string、[]byte,先拷贝内存到xmm中，然后，将指针append到s的元素中。 24 | 25 | lock *sync.RWMutex 26 | 27 | sizeCtl int64 // -1 正在扩容 28 | 29 | //游标位置 30 | curRPos int //读光标 31 | curWGapPos int //写gap光标 32 | curWSlotPos int //写slot光标 33 | 34 | _sizeof int //每个元素大小 35 | _stype xds.Kind //类型 36 | _len int //元素个数计数 37 | 38 | _scap int //容量 39 | } 40 | 41 | //func NewXslice(mm xmm.XMemory,_type Kind,_cap int) * Xslice { 42 | func NewXslice(_type xds.Kind,_cap int) *Xslice { 43 | 44 | //计算分配slot数量，有余数会多分配一个 45 | slotCap := float64(_cap % _blockSize) 46 | var slotNum int64 = int64(math.Ceil(slotCap)) 47 | 48 | if slotNum <= 0{ 49 | slotNum = 1 50 | } 51 | 52 | slot := make([][8]_gap,slotNum,slotNum) 53 | return &Xslice{ 54 | s: slot, 55 | lock: new(sync.RWMutex), 56 | _scap: _cap, 57 | _stype: _type, 58 | } 59 | } 60 | 61 | func(xs *Xslice) Append(v interface{}) (*Xslice,error) { 62 | xs.lock.Lock() 63 | defer xs.lock.Unlock() 64 | 65 | 66 | //判断类型：基本copy进去。 67 | // s中将要存放的uintptr拿到，uintptr为一个起始地址，加上一个index偏移量为写入地址。 68 | 69 | 70 | var gapPos int64 = 0 71 | 72 | slotCap := float64(xs.curWGapPos % _blockSize) 73 | 74 | gapPos = int64(slotCap) 75 | // 没有空位，扩容 76 | if slotCap == 0 && xs.curWGapPos >= _blockSize{ 77 | newSlot := [_blockSize]_gap{} 78 | xs.s = append(xs.s,newSlot) 79 | xs.curWSlotPos += 1 80 | } 81 | b,err := xds.Marshal(xs._stype,v) 82 | if err != nil{ 83 | return xs,err 84 | } 85 | 86 | xs.s[xs.curWSlotPos][gapPos] = b 87 | 88 | xs.curWGapPos += 1 89 | xs._len += 1 90 | 91 | return xs,nil 92 | } 93 | 94 | func (xs *Xslice) Set(n int,v interface{}) error { 95 | xs.lock.Lock() 96 | defer xs.lock.Unlock() 97 | 98 | var gapPos int64 = 0 99 | slotCap := float64(n % _blockSize) 100 | gapPos = int64(slotCap) 101 | 102 | // 没有空位，扩容 103 | if slotCap == 0 && xs.curWGapPos >= _blockSize{ 104 | newSlot := [_blockSize]_gap{} 105 | xs.s = append(xs.s,newSlot) 106 | xs.curWSlotPos += 1 107 | } 108 | 109 | b,err := xds.Marshal(xs._stype,v) 110 | if err != nil{ 111 | return err 112 | } 113 | 114 | xs.s[xs.curWSlotPos][gapPos] = b 115 | xs.curWGapPos += 1 116 | xs._len += 1 117 | 118 | return nil 119 | } 120 | 121 | func (xs *Xslice) Get(n int) (interface{}, error) { 122 | xs.lock.Lock() 123 | defer xs.lock.Unlock() 124 | 125 | var gapPos int64 = 0 126 | slotCap := float64(n % _blockSize) 127 | gapPos = int64(slotCap) 128 | b := xs.s[xs.curWSlotPos][gapPos] 129 | v,err := xds.UnMarshal(xs._stype,b) 130 | if err != nil{ 131 | return nil,err 132 | } 133 | 134 | return v,nil 135 | } 136 | 137 | func (xs *Xslice) Free () { 138 | xs.lock.Lock() 139 | xs.curWGapPos = 0 140 | xs.curRPos = 0 141 | xs._scap = 0 142 | xs._sizeof = 0 143 | xs._len = 0 144 | xs.curWSlotPos = 0 145 | xs.s = nil 146 | xs.lock.Unlock() 147 | } 148 | 149 | func(xs *Xslice) ForEach(f func(i int, v interface{}) error) error{ 150 | xs.lock.Lock() 151 | defer xs.lock.Unlock() 152 | 153 | var c int = 0 154 | for i,gaps := range xs.s { 155 | for _, b := range gaps { 156 | //判断坐标是否到头 157 | if xs.curWSlotPos == i && xs.curWGapPos == c{ 158 | return nil 159 | } 160 | c = c + 1 161 | 162 | v,err := xds.UnMarshal(xs._stype,b) 163 | if err != nil{ 164 | return nil 165 | } 166 | 167 | if err := f(c,v);err != nil{ 168 | return err 169 | } 170 | } 171 | } 172 | 173 | return nil 174 | } 175 | 176 | //返回xslice长度 177 | func (xs *Xslice) Len() int { 178 | xs.lock.Lock() 179 | defer xs.lock.Unlock() 180 | return xs._len 181 | } 182 | 183 | //@todo 把扩容逻辑封装起来 184 | func (xs *Xslice) increase() *Xslice { 185 | return nil 186 | } -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 XDS project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // XDS Project Site: https://github.com/heiyeluren 16 | // XDS URL: https://github.com/heiyeluren/xds 17 | // 18 | 19 | package xds 20 | 21 | import ( 22 | "errors" 23 | "reflect" 24 | "unsafe" 25 | // "github.com/heiyeluren/xmm" 26 | ) 27 | 28 | // 定义XDS共用的常量和一些基础函数 29 | //调用方法： 30 | /* 31 | //包含 32 | import( 33 | "github.com/heiyeluren/xds" 34 | ) 35 | 36 | //结构体中使用 37 | type ConcurrentHashMap struct { 38 | keyKind xds.Kind 39 | valKind xds.Kind 40 | data *ConcurrentRawHashMap 41 | } 42 | 43 | //一般数据结构中使用 44 | m, err := xds.NewMap(mm, xds.String, xds.Int) 45 | 46 | //序列化处理 (其他类型到 []byte，两个函数效果一样) 47 | data, err := xds.Marshal(xds.String, "heiyeluren") 48 | data, err := xds.RawToByte(xds.String, "heiyeluren") 49 | 50 | // 反序列化处理 (从 []byte到原始类型) 51 | str, err := xds.UnMarshal(xds.String, data) 52 | data, err := xds.ByteToRaw(xds.String, "heiyeluren") 53 | 54 | */ 55 | 56 | 57 | 58 | //======================================================== 59 | // 60 | // XDS 常量和数据结构定义区 61 | // XDS constant and data structure definition 62 | // 63 | //======================================================== 64 | 65 | // XDS 中数据类型的定义 66 | // map[keyKind]valKind == xds.NewMap(mm, xds.keyKind, xds.valKind) 67 | // make([]typeKind, len) == xds.NewSlice(mm, xds.dataKind) 68 | 69 | type Kind uint 70 | 71 | const ( 72 | Invalid Kind = iota 73 | Bool 74 | Int 75 | Int8 76 | Int16 77 | Int32 78 | Int64 79 | Uint 80 | Uint8 81 | Uint16 82 | Uint32 83 | Uint64 84 | Uintptr 85 | Float32 86 | Float64 87 | Complex64 88 | Complex128 89 | Array 90 | Chan 91 | Func 92 | Interface 93 | Map 94 | Ptr 95 | ByteSlice 96 | String 97 | Struct 98 | UnsafePointer 99 | ) 100 | 101 | // 类型错误 102 | // type error 103 | var InvalidType = errors.New("Kind type Error") 104 | 105 | // type ConcurrentHashMap struct { 106 | // keyKind Kind 107 | // valKind Kind 108 | // data *ConcurrentRawHashMap 109 | // } 110 | 111 | 112 | 113 | //======================================================== 114 | // 115 | // XDS 常用函数定义区 116 | // XDS common function definition area 117 | // 118 | //======================================================== 119 | 120 | // 针对一些Kind数据类型的序列化 121 | // Serialization for some kind data types 122 | func Marshal(kind Kind, content interface{}) (data []byte, err error) { 123 | switch kind { 124 | case String: 125 | data, ok := content.(string) 126 | if !ok { 127 | return nil, InvalidType 128 | } 129 | sh := (*reflect.StringHeader)(unsafe.Pointer(&data)) 130 | return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: sh.Data, Len: sh.Len, Cap: sh.Len})), nil 131 | case ByteSlice: 132 | data, ok := content.([]byte) 133 | if !ok { 134 | return nil, InvalidType 135 | } 136 | return data, nil 137 | case Int: 138 | h, ok := content.(int) 139 | if !ok { 140 | return nil, InvalidType 141 | } 142 | return (*[8]byte)(unsafe.Pointer(&h))[:], nil 143 | case Uintptr: 144 | h, ok := content.(uintptr) 145 | if !ok { 146 | return nil, InvalidType 147 | } 148 | return (*[8]byte)(unsafe.Pointer(&h))[:], nil 149 | } 150 | return 151 | } 152 | 153 | // 针对一些Kind数据类型的反序列化 154 | // Deserialization for some kind data types 155 | func UnMarshal(kind Kind, data []byte) (content interface{}, err error) { 156 | switch kind { 157 | case String: 158 | return *(*string)(unsafe.Pointer(&data)), nil 159 | case Uintptr: 160 | sh := (*reflect.SliceHeader)(unsafe.Pointer(&data)) 161 | return *(*uintptr)(unsafe.Pointer(sh.Data)), nil 162 | case Int: 163 | sh := (*reflect.SliceHeader)(unsafe.Pointer(&data)) 164 | return *(*int)(unsafe.Pointer(sh.Data)), nil 165 | case ByteSlice: 166 | return data, nil 167 | } 168 | return 169 | } 170 | 171 | // Marshal 函数的别名 172 | // Marshal function link name 173 | func RawToByte(kind Kind, content interface{}) (data []byte, err error) { 174 | return Marshal(kind, content) 175 | } 176 | 177 | // UnMarshal 函数的别名 178 | // UnMarshal function link name 179 | func ByteToRaw(kind Kind, data []byte) (content interface{}, err error) { 180 | return UnMarshal(kind, data) 181 | } 182 | 183 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

3 | GoDoc 4 | GoDoc 5 | Go Report Card 6 |

7 | 8 | 9 | ## XDS - eXtensible Data Structure Sets
(第三方可扩展的 Golang 高性能数据结构和数据类型合集) 10 | 11 | A third-party extensible collection of high-performance data structures and data types in Go 12 | 13 | - [XDS - eXtensible Data Structure
（第三方可扩展的 Go 语言中高性能数据结构和数据类型合集）](#xds---extensible-data-structure-第三方可扩展的-go-语言中高性能数据结构和数据类型合集) 14 | - [XDS 介绍：（什么是 Xds）](#xds-介绍什么是-xds) 15 | - [XDS - XMap 概要介绍](#xds---xmap-概要介绍) 16 | - [为什么要设计 XMap？](#为什么要设计-xmap) 17 | - [XMap 设计目标是什么？](#xmap-设计目标是什么) 18 | - [XMap 的技术特点](#xmap-的技术特点) 19 | - [XMap 性能数据和实现对比](#xmap-性能数据和实现对比) 20 | - [XMap 与 Go 官方数据结构特点对比：(20% 写入，80% 读场景)](#xmap-与-go-官方数据结构特点对比20-写入80-读场景) 21 | - [如何使用 XMap？](#如何使用-xmap) 22 | - [XMap 各类 API 使用案例](#xmap-各类-api-使用案例) 23 | - [- XMap 使用示例](#--xmap-使用示例) 24 | - [XMap 内部是如何实现的？](#xmap-内部是如何实现的) 25 | - [XDS 项目开发者](#xds-项目开发者) 26 | - [XDS 技术交流](#xds-技术交流) 27 | 28 |
29 | 30 | ## XDS 介绍：（什么是 Xds） 31 | 32 | XDS - eXtensible Data Structure（第三方可扩展的 Go 语言中高性能数据结构和数据类型合集） 33 | 34 | XDS 主要是为了解决现有 Go 语言官方内置的各类数据结构性能在高并发场景中不尽如人意的情况而开发，核心主要是依赖于 [XMM](https://github.com/heiyeluren/xmm) 内存管理库基础之上开发，保证了高性能和内存可控。 35 | 36 | XDS 集合目前主要包含： 37 | - XMap - 高性能的类似 map/sync.map 的 Map 型数据结构类型（已开源） 38 | - XSlice - 高性能类似 slice 的数组型数据结构类型（开发中） 39 | - XChannel - 高性能的 channel 管道类型结构（调研中） 40 | - 更多... 41 | 42 |
43 | 44 |
45 | 46 |
47 | 48 | ## XDS - XMap 概要介绍 49 | 50 | XMap 是属于高性能开源 Go 数据结构 Xds 中的 map 数据结构类型的实现，主要是基于高性能内存管理库 [XMM](https://github.com/heiyeluren/xmm) 基础之上进行的开发，主要弥补了 Go 内置 map 的无法并发读写，并且总体读写性能比较差的问题而开发。 51 | 52 |
53 | 54 | ### 为什么要设计 XMap？ 55 | 56 | 现有 Golang 中的 map 数据结构无法解决并发读写问题，Sync.map 并发性能偏差，针对这个情况，各种高性能底层服务需要一个高性能、大容量、高并发、无 GC 的 Map，所以开发实现 XMap。 57 | 针对我们需求调研了市场上主要的 hashmap 结构，不能满足我们性能和功能要求。 58 | 59 |
60 | 61 | ### XMap 设计目标是什么？ 62 | 63 | 要求设计一个可以并发读写不会出现 panic，要求并发读写 200w+ OPS/s 的并发 map 结构。 64 | （写 20%，读 80% 场景；说明：go 自带 map 读写性能在 80w ops/s，大量并发读写下可能 panic；sync.map 写入性能在 100w OPS/s） 65 | 66 |
67 | 68 | ### XMap 的技术特点 69 | 70 | - 绝对高性能的 map 数据结构（map 的 3 倍，sync.map 的 2 倍并发性能） 71 | - 内部实现机制对比 Go 原生 map/sync.map 技术设计上要更细致，更考虑性能，使用包括开地址法，红黑树等等结构提升性能； 72 | - 为了节约内存，初始的值比较低，但是依赖于 XMM 高性能内存扩容方式，能够快速进行内存扩容，保证并发写入性能 73 | - 底层采用 XMM 内存管理，不会受到 Go 系统本身 GC 机制的卡顿影响，保证高性能； 74 | - 提供 API 更具备扩展性，在一些高性能场景提供更多调用定制设置，并且能够同时支持 map 类型操作和底层 hash 表类型操作，适用场景更多； 75 | - 其他特性 76 | 77 |
78 | 79 | ### XMap 性能数据和实现对比 80 | 81 | XMap 目前并发读写场景下性能可以达到 200 万 op/s，对比原生 map 单机性能 80 万 op/s，提升了 3 倍 +，对比 Go 官方扩展库 sync.Map 性能有 2 倍的提升。 82 | 83 |
84 | 85 | #### XMap 与 Go 官方数据结构特点对比：(20% 写入，80% 读场景) 86 | 87 | | map 模块 | 性能数据
| 加锁机制 | 底层数据结构 | 内存机制 | 88 | |------|------|------|------|------| 89 | |map | 80w+ read/s
并发读写会 panic | 无 | Hashtable + Array | Go gc | 90 | |sync.Map | 100w+ op/s | RWMutex | map | Go gc | 91 | | Xds.XMap | 200w+ op/s | CAS + RWMutex | Hashtable + Array + RBTree | XMM | 92 | 93 |
94 |
95 | 96 | ## 如何使用 XMap？ 97 | 98 | 快速使用： 99 | 100 | 1. 下载对应包 101 | 102 | ```shell 103 | go get -u github.com/heiyeluren/xds 104 | go get -u github.com/heiyeluren/xmm 105 | ``` 106 | 107 | 2. 快速包含调用库： 108 | 109 | ```go 110 | //注意：本代码只是伪代码，大家最好看这个使用测试案例更充分一些 111 | //详细使用案例：https://github.com/heiyeluren/xds/blob/main/example/xmap_test0.go 112 | //快速使用入门：https://github.com/heiyeluren/xds/blob/main/example/xmap_test1.go 113 | 114 | import ( 115 | xmm "github.com/heiyeluren/xmm" 116 | xds "github.com/heiyeluren/xds" 117 | xmap "github.com/heiyeluren/xds/xmap" 118 | ) 119 | 120 | // 创建一个 XMM 内存块 121 | f := &xmm.Factory{} 122 | mm, err := f.CreateMemory(0.75) 123 | 124 | // 构建一个 map[string]string 的 xmap 125 | m, err := xmap.NewMap(mm, xds.String, xds.String) 126 | 127 | // 写入、读取、删除一个元素 128 | err = m.Set("name", "heiyeluren") 129 | ret, key_exists, err := m.Get("name") 130 | err = m.Remove("name") 131 | 132 | // 遍历整个map 133 | m.Each(func(key, val interface{}) error { 134 | fmt.Printf("For each XMap all key:[%s] value:[%s] \n", key, val) 135 | return nil 136 | }) 137 | 138 | //... 139 | 140 | ``` 141 | 142 | 3. 执行对应代码 143 | 144 | ```shell 145 | go run map-test.go 146 | ``` 147 | 148 |
149 | 150 | ### XMap 各类 API 使用案例 151 | 152 | #### - [Xmap 快速使用入门](https://github.com/heiyeluren/xds/blob/main/example/xmap_test1.go) 153 | 154 | #### - [XMap 详细使用示例](https://github.com/heiyeluren/xds/blob/main/example/xmap_test0.go) - 155 | 156 | - 更多案例（期待） 157 | 158 | 以上代码案例执行输出： 159 |
160 | 161 | 162 |
163 | 164 | ## XMap 内部是如何实现的？ 165 | 166 | #### - [《Xds-XMap技术设计与实现》](https://github.com/heiyeluren/xds/blob/main/docs/Xmap-Implement.md) - 167 | 168 |
169 | 170 | - 参考：[《Go map 内部实现机制一》](https://www.jianshu.com/p/aa0d4808cbb8) | [《Go map 内部实现二》](https://zhuanlan.zhihu.com/p/406751292) | [《Golang sync.Map 性能及原理分析》](https://blog.csdn.net/u010853261/article/details/103848666) 171 | - 其他 172 | 173 |
174 | 175 |
176 | 177 |
178 |
179 |
180 | 181 | ## XDS 项目开发者 182 | 183 | | 项目角色 | 项目成员 | 184 | | ----------- | ----------- | 185 | | 项目发起人/负责人 | 黑夜路人 ( @heiyeluren )
老张 ( @Zhang-Jun-tao ) | 186 | | 项目开发者 | 老张 ( @Zhang-Jun-tao )
黑夜路人 ( @heiyeluren )
Viktor ( @guojun1992 ) | 187 | 188 |

189 | 190 | ## XDS 技术交流 191 | 192 | XDS 还在早期，当然也少不了一些问题和 bug，欢迎大家一起共创，或者直接提交 PR 等等。 193 | 194 | 195 | 欢迎加入 XDS 技术交流微信群，要加群，可以先关注添加如下公众号：
196 | （如无法看到图片，请直接微信里搜索公众号“黑夜路人技术”，关注发送“加群”字样信息即可 ） 197 | 198 | 199 | 200 | 201 | 202 |
203 |
204 | -------------------------------------------------------------------------------- /xmap/concurrent_hash_map_benchmark_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 XDS project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // XDS Project Site: https://github.com/heiyeluren 16 | // XDS URL: https://github.com/heiyeluren/xds 17 | // 18 | 19 | package xmap 20 | 21 | import ( 22 | "fmt" 23 | "math/rand" 24 | "reflect" 25 | "strconv" 26 | "sync" 27 | "testing" 28 | "time" 29 | "unsafe" 30 | // "xds/xmap/entry" 31 | // "github.com/spf13/cast" 32 | "github.com/heiyeluren/xmm" 33 | "github.com/heiyeluren/xds/xmap/entry" 34 | ) 35 | 36 | // 1000000000 37 | func BenchmarkCHM_Put(b *testing.B) { 38 | f := &xmm.Factory{} 39 | mm, err := f.CreateMemory(0.75) 40 | if err != nil { 41 | b.Fatal(err) 42 | } 43 | chm, err := NewConcurrentRawHashMap(mm, 16, 2, 8) 44 | if err != nil { 45 | b.Fatal(err) 46 | } 47 | keys := make([]string, 10000000) 48 | for i := 0; i < 10000000; i++ { 49 | keys[i] = strconv.Itoa(rand.Int()) 50 | } 51 | length := len(keys) 52 | b.ResetTimer() 53 | for i := 0; i < b.N; i++ { 54 | key := []byte(keys[rand.Int()%length]) 55 | if err := chm.Put(key, key); err != nil { 56 | b.Error(err) 57 | } 58 | } 59 | } 60 | 61 | func BenchmarkCHM_Get(b *testing.B) { 62 | f := &xmm.Factory{} 63 | mm, err := f.CreateMemory(0.75) 64 | if err != nil { 65 | b.Fatal(err) 66 | } 67 | chm, err := NewConcurrentRawHashMap(mm, 16, 2, 8) 68 | if err != nil { 69 | b.Fatal(err) 70 | } 71 | keys := make([]string, 8000000) 72 | for i := 0; i < 8000000; i++ { 73 | keys[i] = strconv.Itoa(rand.Int()) 74 | } 75 | length := len(keys) 76 | for _, key := range keys { 77 | 78 | if err := chm.Put([]byte(key), []byte(key)); err != nil { 79 | b.Error(err) 80 | } 81 | } 82 | b.ResetTimer() 83 | for i := 0; i < b.N; i++ { 84 | key := keys[rand.Int()%length] 85 | if _, exist, err := chm.Get([]byte(key)); err != nil || !exist { 86 | b.Error(err) 87 | } 88 | } 89 | } 90 | 91 | func Test_CHM_Concurrent_Get(t *testing.T) { 92 | // Init() 93 | f := &xmm.Factory{} 94 | mm, err := f.CreateMemory(0.75) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | chm, err := NewConcurrentRawHashMap(mm, 16, 2, 8) 99 | if err != nil { 100 | t.Fatal(err) 101 | } 102 | keys := make([]string, 8000000) 103 | for i := 0; i < 8000000; i++ { 104 | keys[i] = strconv.Itoa(rand.Int() % 800000) 105 | } 106 | for _, key := range keys { 107 | if err := chm.Put([]byte(key), []byte(key)); err != nil { 108 | t.Error(err) 109 | } 110 | } 111 | var wait sync.WaitGroup 112 | wait.Add(10) 113 | fmt.Println("开始") 114 | t1 := time.Now() 115 | for j := 0; j < 10; j++ { 116 | go func(z int) { 117 | defer wait.Done() 118 | start, end := z*800000, (z+1)*800000 119 | for _, s := range keys[start:end] { 120 | if err := chm.Put([]byte(s), []byte(s)); err != nil { 121 | t.Error(err) 122 | } 123 | } 124 | }(j) 125 | } 126 | wait.Wait() 127 | fmt.Println(len(keys), time.Now().Sub(t1), len(keys)) 128 | <-time.After(time.Minute) 129 | } 130 | 131 | func TestGoCreateEntry(t *testing.T) { 132 | var wait sync.WaitGroup 133 | wait.Add(10) 134 | node := &entry.NodeEntry{} 135 | // nodes := make([]*entry.NodeEntry, 8000000) 136 | tt := time.Now() 137 | for j := 0; j < 10; j++ { 138 | go func(z int) { 139 | defer wait.Done() 140 | for i := 0; i < 800000; i++ { 141 | node.Key = []byte("keyPtr") 142 | node.Value = []byte("valPtr") 143 | node.Hash = 12121 144 | // nodes[z*800000+i] = node 145 | } 146 | }(j) 147 | } 148 | wait.Wait() 149 | fmt.Println(time.Now().Sub(tt)) 150 | } 151 | 152 | func TestCreateEntry(t *testing.T) { 153 | f := &xmm.Factory{} 154 | mm, err := f.CreateMemory(0.75) 155 | if err != nil { 156 | t.Fatal(err) 157 | } 158 | entryPtr, err := mm.Alloc(_NodeEntrySize) 159 | if err != nil { 160 | t.Fatal(err) 161 | } 162 | /* 80445440 \ 283598848 \ 228630528 \ 267530240 \97157120 \ 129908736 163 | keyPtr, valPtr, err := mm.From2("hjjhj", "jjjshjfhsdf") 164 | if err != nil { 165 | t.Fatal(err) 166 | }*/ 167 | pageNum := float64(uintptr(entryPtr)) / 4096.0 168 | fmt.Println(uintptr(entryPtr), pageNum, (pageNum+1)*4096 > float64(uintptr(entryPtr)+_NodeEntrySize)) 169 | node := (*entry.NodeEntry)(entryPtr) 170 | tt := time.Now() 171 | var wait sync.WaitGroup 172 | wait.Add(10) 173 | for j := 0; j < 10; j++ { 174 | go func(z int) { 175 | defer wait.Done() 176 | for i := 0; i < 8000000; i++ { 177 | node.Key = []byte("keyPtr") 178 | node.Value = []byte("valPtr") 179 | node.Hash = 12121 180 | } 181 | }(j) 182 | } 183 | wait.Wait() 184 | fmt.Println(time.Now().Sub(tt)) 185 | } 186 | 187 | // todo 优秀一些，使用这种方式 188 | func TestFieldCopy(t *testing.T) { 189 | f := &xmm.Factory{} 190 | mm, err := f.CreateMemory(0.75) 191 | if err != nil { 192 | t.Fatal(err) 193 | } 194 | 195 | var wait sync.WaitGroup 196 | wait.Add(10) 197 | for j := 0; j < 10; j++ { 198 | go func(z int) { 199 | defer wait.Done() 200 | for i := 0; i < 800000; i++ { 201 | entryPtr, err := mm.Alloc(_NodeEntrySize) 202 | if err != nil { 203 | t.Fatal(err) 204 | } 205 | keyPtr, valPtr, err := mm.Copy2([]byte("hjjhj"), []byte("jjjshjfhsdf")) 206 | if err != nil { 207 | t.Fatal(err) 208 | } 209 | source := entry.NodeEntry{Value: keyPtr, Key: valPtr, Hash: 12312} 210 | offset := unsafe.Offsetof(source.Next) // 40 211 | srcData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&source)), Len: int(offset), Cap: int(offset)})) 212 | dstData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(entryPtr), Len: int(offset), Cap: int(offset)})) 213 | copy(*dstData, *srcData) 214 | } 215 | }(j) 216 | } 217 | wait.Wait() 218 | } 219 | -------------------------------------------------------------------------------- /example/xmap_test0.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/spf13/cast" 6 | "github.com/heiyeluren/xmm" 7 | "github.com/heiyeluren/xds" 8 | "github.com/heiyeluren/xds/xmap" 9 | ) 10 | 11 | // TestMap testing 12 | // ----------------------------------- 13 | // 把Xmap当做普通map来使用 14 | // 说明：类型不限制，初始化必须设定好数据类型，写入数据必须与这个数据类型一致，类似于 map[KeyType]ValType 必须相互符合 15 | // ----------------------------------- 16 | /* 17 | 目前支持的类似于 map[keyType][valType] key value 类型如下： 18 | 就是调用：m, err := xds.NewMap(mm, xmap.String, xmap.Int) 后面的两个 keyType 和 valType 类型定义如下： 19 | const ( 20 | Invalid Kind = iota 21 | Bool 22 | Int 23 | Int8 24 | Int16 25 | Int32 26 | Int64 27 | Uint 28 | Uint8 29 | Uint16 30 | Uint32 31 | Uint64 32 | Uintptr 33 | Float32 34 | Float64 35 | Complex64 36 | Complex128 37 | Array 38 | Chan 39 | Func 40 | Interface 41 | Map 42 | Ptr 43 | ByteSlice 44 | String 45 | Struct 46 | UnsafePointer 47 | ) 48 | */ 49 | func TestMap(mm xmm.XMemory) { 50 | 51 | // ------------------- 52 | // 常规的map操作 53 | // ------------------- 54 | 55 | fmt.Println("\n--[ XMap NewMap() API example]--") 56 | // 初始化xmap的时候必须制定key和val的数据类型，数据类型是在xmap中定义的 57 | // 构建一个 map[string]int 的xmap 58 | m, err := xmap.NewMap(mm, xds.String, xds.Int) 59 | if err != nil { 60 | panic("call NewMap() fail") 61 | } 62 | 63 | var ( 64 | k11 string = "id" 65 | v11 int = 9527 66 | ) 67 | 68 | // set的时候需要关注类型是否跟初始化对象的时候一致 69 | err = m.Set(k11, v11) 70 | if err != nil { 71 | panic("XMap.Set fail") 72 | } 73 | fmt.Println("XMap.Set key: [", k11, "] success") 74 | 75 | // get数据不用关心类型，也不用做类型转换 76 | ret, exists, err := m.Get(k11) 77 | if err != nil { 78 | panic("XMap.Get fail") 79 | } 80 | fmt.Println("XMap.Get key: [", k11, "] , value: [", ret, "]") 81 | 82 | // Remove数据 83 | err = m.Remove(k11) 84 | if err != nil { 85 | panic("XMap.Remove fail") 86 | } 87 | fmt.Println("XMap.Remove key: [", k11, "] succes") 88 | ret, exists, err = m.Get(k11) 89 | if !exists { 90 | fmt.Println("XMap.Get key: [", k11, "] not found") 91 | } 92 | 93 | 94 | // ------------------- 95 | // 调用扩展的Map函数使用方法(可以获得更多定制性和更高性能) 96 | // ------------------- 97 | 98 | fmt.Println("\n--[ XMap NewMapEx() API example]--") 99 | 100 | 101 | // 生成KV数据 102 | var ( 103 | k22 = "name" 104 | v22 = "heiyeluren" 105 | ) 106 | // 生成一个 map[string]string 数据结构，默认大小256个元素，占用了75%后进行map扩容(这个初始化函数可以获得更好性能，看个人使用场景） 107 | m1, err := xmap.NewMapEx(mm, xds.String, xds.String, uintptr(256), 0.75) 108 | // set数据 109 | m1.Set(k22, v22) 110 | // get数据 111 | ret, exists, err = m1.Get(k22) 112 | fmt.Println("XMap.Get key: [", k22, "] , value: [", ret, "]") 113 | 114 | 115 | 116 | // ------------------- 117 | // 遍历所有map数据 118 | // ------------------- 119 | 120 | fmt.Println("\n--[ XMap ForEach all Key ]--") 121 | 122 | // 写入数据 123 | err = m.Set("k1", 1) 124 | err = m.Set("k2", 2) 125 | err = m.Set("k3", 3) 126 | err = m.Set("k4", 4) 127 | err = m.Set("k5", 5) 128 | 129 | //全局变量可以在匿名函数中访问（如果需要使用外部变量，可以像这样） 130 | gi := 1 131 | fmt.Printf("for each itam start, gi: [%s] \n", gi) 132 | 133 | // 遍历xmap中所有元素 134 | m.Each(func(key, val interface{}) error { 135 | //针对每个KV进行操作，比如打印出来 136 | fmt.Printf("for each XMap all key:[%s] value:[%s] \n", key, val) 137 | //外部变量使用操作 138 | gi++ 139 | return nil 140 | }) 141 | fmt.Printf("for each itam done, gi: [%s] \n", gi) 142 | 143 | //读取map长度 144 | len := m.Len() 145 | fmt.Printf("\nMap length(size): [%s] \n", len) 146 | 147 | } 148 | 149 | 150 | // TestHashMap testing 151 | // ----------------------------------- 152 | // 把Xmap当做普通hashmap来使用 153 | // 说明：Key/Value 都必须是 []byte 154 | // ----------------------------------- 155 | func TestHashMap(mm xmm.XMemory) { 156 | 157 | fmt.Println("\n\n===== XMap X(eXtensible) Raw Map (HashMap) example ======\n") 158 | 159 | hm, err := xmap.NewHashMap(mm) 160 | if err != nil { 161 | panic("call NewHashMap() fail") 162 | } 163 | 164 | var ( 165 | k1 string = "name" 166 | v1 string = "heiyeluren" 167 | k2 string = "id" 168 | v2 uint32 = 9527 169 | ) 170 | 171 | // 新增Key 172 | fmt.Println("\n--[ Raw Map Set Key ]--") 173 | err = hm.Set([]byte(k1), []byte(v1)) 174 | if err != nil { 175 | panic("xmap.Set fail") 176 | } 177 | fmt.Println("Xmap.Set key: [", k1, "] success") 178 | err = hm.Set([]byte(k2), []byte(cast.ToString(v2))) 179 | if err != nil { 180 | panic("xmap.Set fail") 181 | } 182 | fmt.Println("Xmap.Set key: [", k2, "] success") 183 | 184 | // 读取Key 185 | fmt.Println("\n--[ Raw Map Get Key ]--") 186 | s1, exists, err := hm.Get([]byte(k1)) 187 | if err != nil { 188 | panic("xmap.Get fail") 189 | } 190 | fmt.Println("Xmap.Get key: [", k1, "], value: [", cast.ToString(s1), "]") 191 | s2, exists, err := hm.Get([]byte(k2)) 192 | if err != nil { 193 | panic("xmap.Get fail") 194 | } 195 | fmt.Println("Xmap.Get key: [", k2, "], value: [", cast.ToString(s2), "]") 196 | 197 | // 删除Key 198 | fmt.Println("\n--[ Raw Map Remove Key ]--") 199 | err = hm.Remove([]byte(k1)) 200 | if err != nil { 201 | panic("xmap.Remove fail") 202 | } 203 | fmt.Println("Xmap.Remove key: [", k1, "]") 204 | s1, exists, err = hm.Get([]byte(k1)) 205 | // fmt.Println(s1, exists, err) 206 | if !exists { 207 | fmt.Println("Xmap.Get key: [", k1, "] Not Found") 208 | } 209 | s2, exists, err = hm.Get([]byte(k2)) 210 | if err != nil { 211 | panic("xmap.Get fail") 212 | } 213 | fmt.Println("Xmap.Get key: [", k2, "], value: [", cast.ToString(s2), "]") 214 | err = hm.Remove([]byte(k2)) 215 | if err != nil { 216 | panic("xmap.Remove fail") 217 | } 218 | fmt.Println("Xmap.Remove key: [", k2, "]") 219 | s2, exists, err = hm.Get([]byte(k2)) 220 | // fmt.Println(s1, exists, err) 221 | if !exists { 222 | fmt.Println("Xmap.Get key: [", k2, "] Not Found") 223 | } 224 | s1, exists, err = hm.Get([]byte(k1)) 225 | // fmt.Println(s1, exists, err) 226 | if !exists { 227 | fmt.Println("Xmap.Get key: [", k1, "] Not Found") 228 | } 229 | 230 | //-------------------- 231 | // 遍历RawMap 232 | //-------------------- 233 | fmt.Println("\n--[ Raw Map for each all Key ]--") 234 | hm1, err := xmap.NewHashMap(mm) 235 | 236 | // 写入数据 237 | hm1.Set([]byte("K1"), []byte("V1")) 238 | hm1.Set([]byte("K2"), []byte("V2")) 239 | hm1.Set([]byte("K3"), []byte("V3")) 240 | hm1.Set([]byte("K4"), []byte("V4")) 241 | 242 | //hm1.Each(func(key, val []byte)(error) { 243 | // fmt.Println("for each raw map key:[", key, "], val[", val, "]") 244 | //}) 245 | 246 | 247 | //全局变量可以在匿名函数中访问（如果需要使用外部变量，可以像这样） 248 | gi := 1 249 | fmt.Printf("for each itam start, gi: [%s] \n", gi) 250 | 251 | // 遍历xmap中所有元素 252 | hm1.Each(func(key, val []byte) error { 253 | //针对每个KV进行操作，比如打印出来 254 | fmt.Printf("for each XMap all key:[%s] value:[%s] \n", key, val) 255 | //外部变量使用操作 256 | gi++ 257 | return nil 258 | }) 259 | //读取map长度 260 | len := hm1.Len() 261 | fmt.Printf("\nMap length(size): [%s] \n", len) 262 | 263 | } 264 | 265 | // xmap测试代码 266 | func main() { 267 | f := &xmm.Factory{} 268 | mm, err := f.CreateMemory(0.75) 269 | if err != nil { 270 | panic("xmm.CreateConcurrentHashMapMemory fail") 271 | } 272 | fmt.Println("\n===== XMap X(eXtensible) Map example ======\n") 273 | 274 | // var NotFound = errors.New("not found") 275 | 276 | // 把Xmap当做普通map来使用 277 | TestMap(mm) 278 | 279 | // 把Xmap当做普通hashmap来使用 280 | TestHashMap(mm) 281 | 282 | fmt.Println("\nXmap test case done.\n\n") 283 | 284 | } 285 | -------------------------------------------------------------------------------- /xmap/xmap.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 XDS project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // XDS Project Site: https://github.com/heiyeluren 16 | // XDS URL: https://github.com/heiyeluren/xds 17 | // 18 | 19 | package xmap 20 | 21 | import ( 22 | "github.com/heiyeluren/xds" 23 | // "xds" 24 | "github.com/heiyeluren/xmm" 25 | ) 26 | 27 | // XMap is a map of maps. 28 | // ------------------------------------------------ 29 | // 当做map[]来使用的场景 30 | // 本套API主要是提供给把Xmap当做map来使用的场景 31 | // ------------------------------------------------ 32 | 33 | // Xmap struct 34 | type XMap struct { 35 | chm *ConcurrentHashMap 36 | } 37 | 38 | // NewMap returns a new XMap. 39 | // 初始化调用xmap生成对象 40 | // mm 是XMM的内存池对象 41 | // keyKind 是要生成 map[keyType]valType 中的 keyType 42 | // valKind 是要生成 map[keyType]valType 中的 valType 43 | // 说明：keyKind / valKind 都是直接调用xmap中对应的kind类型，必须提前初始化写入 44 | // 45 | // 本函数调用参考： 46 | // 生成一个 map[int]string 数据结构，默认大小16个元素，占用了75%后进行map扩容 47 | // m, err := xds.NewMapEx(mm, xmap.Int, xmap.String) 48 | func NewMap(mm xmm.XMemory, keyKind xds.Kind, valKind xds.Kind) (*XMap, error) { 49 | chm, err := NewDefaultConcurrentHashMap(mm, keyKind, valKind) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return &XMap{chm: chm}, nil 54 | } 55 | 56 | // NewMapEx returns a new XMap. 57 | // 初始化调用xmap生成对象 - 可扩展方法 58 | // mm 是XMM的内存池对象 59 | // keyKind 是要生成 map[keyType]valType 中的 keyType 60 | // valKind 是要生成 map[keyType]valType 中的 valType 61 | // 说明：keyKind / valKind 都是直接调用xmap中对应的kind类型，必须提前初始化写入 62 | // capSize 是默认初始化整个map的大小，本值最好是 2^N 的数字比较合适（2的N次方）； 63 | // 这个值主要提升性能，比如如果你预估到最终会有128个元素，可以初始化时候设置好，这样Xmap不会随意的动态扩容（默认值是16），提升性能 64 | // factSize 负载因子，当存放的元素超过该百分比，就会触发扩容；建议值是0.75 (75%)，这就是当存储数据容量达到75%会触发扩容机制； 65 | // 66 | // 本函数调用参考： 67 | // //生成一个 map[int]string 数据结构，默认大小256个元素，占用了75%后进行map扩容 68 | // m, err := xds.NewMapEx(mm, xmap.Int, xmap.String, (uintptr)256, 0.75) 69 | func NewMapEx(mm xmm.XMemory, keyKind xds.Kind, valKind xds.Kind, capSize uintptr, factSize float64) (*XMap, error) { 70 | chm, err := NewConcurrentHashMap(mm, capSize, factSize, 8, keyKind, valKind) 71 | if err != nil { 72 | return nil, err 73 | } 74 | return &XMap{chm: chm}, nil 75 | } 76 | 77 | // Set 写入数据 78 | func (xm *XMap) Set(key interface{}, val interface{}) (err error) { 79 | return xm.chm.Put(key, val) 80 | } 81 | 82 | // Remove 删除数据 83 | func (xm *XMap) Remove(key interface{}) (err error) { 84 | return xm.chm.Del(key) 85 | } 86 | 87 | // Get 数据 88 | func (xm *XMap) Get(key interface{}) (val interface{}, keyExists bool, err error) { 89 | return xm.chm.Get(key) 90 | } 91 | 92 | // Each 遍历所有数据 93 | func (xm *XMap) Each(f func(key, val interface{}) error) error { 94 | return xm.chm.ForEach(f) 95 | } 96 | 97 | // Len 获取整个map的元素数量 98 | func (xm *XMap) Len() uint64 { 99 | return xm.chm.Len() 100 | } 101 | 102 | 103 | 104 | // RawMap - Hashmap 105 | // ------------------------------------------------ 106 | // 当做原生Hash Map来使用场景 107 | // 本套API主要是提供给把Xmap当做Hash表来使用的场景 108 | // ------------------------------------------------ 109 | // 定义xmap的入口主结构 110 | type RawMap struct { 111 | chm *ConcurrentRawHashMap 112 | } 113 | 114 | // NewHashMap returns a new RawMap. 115 | // 初始化调用xmap - hashmap 生成对象 mm 是XMM的内存池对象 116 | func NewHashMap(mm xmm.XMemory) (*RawMap, error) { 117 | chm, err := NewDefaultConcurrentRawHashMap(mm) 118 | if err != nil { 119 | return nil, err 120 | } 121 | return &RawMap{chm: chm}, nil 122 | } 123 | 124 | // Set 写入数据 125 | func (xm *RawMap) Set(key []byte, val []byte) error { 126 | return xm.chm.Put(key, val) 127 | } 128 | 129 | // Remove 删除数据 130 | func (xm *RawMap) Remove(key []byte) error { 131 | return xm.chm.Del(key) 132 | } 133 | 134 | // Get 数据 135 | func (xm *RawMap) Get(key []byte) ([]byte, bool, error) { 136 | return xm.chm.Get(key) 137 | } 138 | 139 | // Each 遍历所有数据 140 | func (xm *RawMap) Each(f func(key, val []byte) error) error { 141 | return xm.chm.ForEach(f) 142 | } 143 | 144 | // Len 获取整个map的元素数量 145 | func (xm *RawMap) Len() uint64 { 146 | return xm.chm.size 147 | } 148 | 149 | 150 | 151 | 152 | 153 | // The data structure and interface between xMAP and the bottom layer 154 | // ------------------------------------------------ 155 | // xmap与底层交互的数据结构和接口 156 | // 157 | // 主要提供给上层xmap api调用，完成一些转换工作 158 | // ------------------------------------------------ 159 | 160 | //底层交互数据结构（带有传入数据类型保存） 161 | type ConcurrentHashMap struct { 162 | keyKind xds.Kind 163 | valKind xds.Kind 164 | data *ConcurrentRawHashMap 165 | } 166 | 167 | // NewDefaultConcurrentHashMap 类似于make(map[keyKind]valKind) 168 | // mm 内存分配模块 169 | // keyKind: map中key的类型 170 | // valKind: map中value的类型 171 | func NewDefaultConcurrentHashMap(mm xmm.XMemory, keyKind, valKind xds.Kind) (*ConcurrentHashMap, error) { 172 | return NewConcurrentHashMap(mm, 16, 0.75, 8, keyKind, valKind) 173 | } 174 | 175 | // NewConcurrentHashMap 类似于make(map[keyKind]valKind) 176 | // mm 内存分配模块 177 | // keyKind: map中key的类型 178 | // cap:初始化bucket长度 179 | // fact:负载因子，当存放的元素超过该百分比，就会触发扩容。 180 | // treeSize：bucket中的链表长度达到该值后，会转换为红黑树。 181 | // valKind: map中value的类型 182 | func NewConcurrentHashMap(mm xmm.XMemory, cap uintptr, fact float64, treeSize uint64, keyKind, valKind xds.Kind) (*ConcurrentHashMap, error) { 183 | chm, err := NewConcurrentRawHashMap(mm, cap, fact, treeSize) 184 | if err != nil { 185 | return nil, err 186 | } 187 | return &ConcurrentHashMap{keyKind: keyKind, valKind: valKind, data: chm}, nil 188 | } 189 | 190 | //Get 191 | func (chm *ConcurrentHashMap) Get(key interface{}) (val interface{}, keyExists bool, err error) { 192 | // k, err := chm.Marshal(chm.keyKind, key) 193 | k, err := xds.RawToByte(chm.keyKind, key) 194 | if err != nil { 195 | return nil, false, err 196 | } 197 | valBytes, exists, err := chm.data.Get(k) 198 | if err != nil { 199 | return nil, exists, err 200 | } 201 | // ret, err := chm.UnMarshal(chm.valKind, valBytes) 202 | ret, err := xds.ByteToRaw(chm.valKind, valBytes) 203 | if err != nil { 204 | return nil, false, err 205 | } 206 | return ret, true, nil 207 | } 208 | 209 | //Put 210 | func (chm *ConcurrentHashMap) Put(key interface{}, val interface{}) (err error) { 211 | // k, err := chm.Marshal(chm.keyKind, key) 212 | k, err := xds.RawToByte(chm.keyKind, key) 213 | if err != nil { 214 | return err 215 | } 216 | // v, err := chm.Marshal(chm.valKind, val) 217 | v, err := xds.RawToByte(chm.valKind, val) 218 | return chm.data.Put(k, v) 219 | } 220 | 221 | //Del 222 | func (chm *ConcurrentHashMap) Del(key interface{}) (err error) { 223 | // k, err := chm.Marshal(chm.keyKind, key) 224 | k, err := xds.RawToByte(chm.keyKind, key) 225 | if err != nil { 226 | return err 227 | } 228 | return chm.data.Del(k) 229 | } 230 | 231 | // ForEach 遍历所有key-vel对 232 | func (chm *ConcurrentHashMap) ForEach(fun func(key, val interface{}) error) error { 233 | return chm.data.ForEach(func(keyRaw, valRaw []byte) error { 234 | key, err := xds.ByteToRaw(chm.keyKind, keyRaw) 235 | if err != nil { 236 | return err 237 | } 238 | val, err := xds.ByteToRaw(chm.valKind, valRaw) 239 | if err != nil { 240 | return err 241 | } 242 | return fun(key, val) 243 | }) 244 | } 245 | 246 | // Len 存储元素个数 247 | func (chm *ConcurrentHashMap) Len() uint64 { 248 | if chm.data == nil { 249 | return 0 250 | } 251 | return chm.data.size 252 | } 253 | -------------------------------------------------------------------------------- /xmap/concurrent_hash_map_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 XDS project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // XDS Project Site: https://github.com/heiyeluren 16 | // XDS URL: https://github.com/heiyeluren/xds 17 | // 18 | 19 | package xmap 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "io/ioutil" 25 | "math/rand" 26 | "os" 27 | "path/filepath" 28 | "strings" 29 | "sync" 30 | "testing" 31 | "time" 32 | "unsafe" 33 | // "xds/xmap/entry" 34 | "github.com/heiyeluren/xds/xmap/entry" 35 | "github.com/heiyeluren/xmm" 36 | "github.com/spf13/cast" 37 | ) 38 | 39 | func TestMap(t *testing.T) { 40 | t1 := time.Now() 41 | var data sync.Map 42 | var wait sync.WaitGroup 43 | wait.Add(10) 44 | for h := 0; h < 10; h++ { 45 | go func() { 46 | defer wait.Done() 47 | for i := 0; i < 1000000; i++ { 48 | key := cast.ToString(i) 49 | data.Store(key, key) 50 | } 51 | }() 52 | } 53 | wait.Wait() 54 | fmt.Println(time.Now().Sub(t1)) 55 | } 56 | 57 | func TestConcurrentRawHashMap_Performance(t *testing.T) { 58 | Init() 59 | f := &xmm.Factory{} 60 | mm, err := f.CreateMemory(0.6) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | chm, err := NewConcurrentRawHashMap(mm, 16, 0.75, 8) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | fmt.Println("http://localhost:6060/debug/pprof/profile") 69 | // http://localhost:6060/debug/pprof/profile 70 | for i := 0; i < 1000000000; i++ { 71 | key := cast.ToString(i) 72 | if err := chm.Put([]byte(key), []byte(key)); err != nil { 73 | t.Error(err) 74 | } 75 | } 76 | } 77 | 78 | func TestConcurrentRawHashMap_Function_Second(t *testing.T) { 79 | f := &xmm.Factory{} 80 | mm, err := f.CreateMemory(0.75) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | chm, err := NewConcurrentRawHashMap(mm, 16, 4, 8) 85 | if err != nil { 86 | t.Fatal(err) 87 | } 88 | var wait sync.WaitGroup 89 | wait.Add(10) 90 | keys := make(chan string, 1000000) 91 | quit := make(chan bool, 1) 92 | go func() { 93 | <-time.After(time.Second * 22) 94 | t2 := time.Now() 95 | for { 96 | key, ok := <-keys 97 | if !ok { 98 | fmt.Println("read time:", time.Now().Sub(t2)) 99 | quit <- true 100 | return 101 | } 102 | val, exist, err := chm.Get([]byte(cast.ToString(key))) 103 | if bytes.Compare(val, []byte(cast.ToString(key))) != 0 || !exist { 104 | t.Error(err, key) 105 | } 106 | } 107 | }() 108 | t1 := time.Now() 109 | for j := 0; j < 10; j++ { 110 | go func(z int) { 111 | defer wait.Done() 112 | for i := 0; i < 100000; i++ { 113 | key := cast.ToString(i + (z * 10000)) 114 | if err := chm.Put([]byte(key), []byte(key)); err != nil { 115 | t.Error(err) 116 | } else { 117 | keys <- key 118 | } 119 | } 120 | }(j) 121 | } 122 | wait.Wait() 123 | fmt.Println(len(keys), time.Now().Sub(t1), len(keys)) 124 | close(keys) 125 | f.PrintStatus() 126 | 127 | <-quit 128 | <-time.After(time.Second * 10) 129 | } 130 | 131 | func TestConcurrentRawHashMap_Function1(t *testing.T) { 132 | Init() 133 | // runtime.GOMAXPROCS(16) 134 | f := &xmm.Factory{} 135 | mm, err := f.CreateMemory(0.75) 136 | if err != nil { 137 | t.Fatal(err) 138 | } 139 | chm, err := NewConcurrentRawHashMap(mm, 16, 4, 8) 140 | if err != nil { 141 | t.Fatal(err) 142 | } 143 | var wait sync.WaitGroup 144 | wait.Add(10) 145 | keys := make(chan string, 8000000) 146 | quit := make(chan bool, 1) 147 | go func() { 148 | <-time.After(time.Second * 10) 149 | t2 := time.Now() 150 | for { 151 | key, ok := <-keys 152 | if !ok { 153 | fmt.Println("read time:", time.Now().Sub(t2)) 154 | quit <- true 155 | return 156 | } 157 | val, exist, err := chm.Get([]byte(key)) 158 | if bytes.Compare(val, []byte(key)) != 0 || !exist { 159 | t.Fatal(err, key) 160 | } 161 | } 162 | }() 163 | t1 := time.Now() 164 | for j := 0; j < 10; j++ { 165 | go func(z int) { 166 | defer wait.Done() 167 | for i := 0; i < 800000; i++ { 168 | key := cast.ToString(i + (z * 10000)) 169 | if err := chm.Put([]byte(key), []byte(key)); err != nil { 170 | t.Error(err) 171 | } else { 172 | keys <- key 173 | } 174 | } 175 | }(j) 176 | } 177 | wait.Wait() 178 | fmt.Println(len(keys), time.Now().Sub(t1), len(keys)) 179 | close(keys) 180 | <-quit 181 | } 182 | 183 | func TestMMM(t *testing.T) { 184 | // /usr/local/go/src/runtime 185 | filepath.Walk("/usr/local/go/src/runtime", func(path string, info os.FileInfo, err error) error { 186 | if info.IsDir() { 187 | return nil 188 | } 189 | if path[len(path)-3:] != ".go" { 190 | return nil 191 | } 192 | bytes, err := ioutil.ReadFile(path) 193 | if err != nil { 194 | return err 195 | } 196 | for _, s1 := range strings.Split(string(bytes), "\n") { 197 | s1 = strings.TrimSpace(s1) 198 | if len(s1) < 3 || s1[:len("//")] == "//" || strings.Contains(s1, " = ") || strings.Contains(info.Name(), "_test.go") { 199 | continue 200 | } 201 | for _, s2 := range strings.Split(s1, " ") { 202 | s := strings.TrimSpace(s2) 203 | if len(s) < 1 { 204 | continue 205 | } 206 | if int(s[0]) <= 64 || int(s[0]) >= 91 { 207 | continue 208 | } 209 | fmt.Println( /*path, info.Name(),*/ s, s1) 210 | } 211 | } 212 | return nil 213 | }) 214 | } 215 | 216 | type A struct { 217 | lock sync.RWMutex 218 | age int 219 | } 220 | 221 | func TestSizeOf(t *testing.T) { 222 | fmt.Println(unsafe.Sizeof(sync.RWMutex{}), unsafe.Sizeof(A{})) 223 | fmt.Println(unsafe.Sizeof(Bucket2{})) 224 | } 225 | 226 | type Bucket2 struct { 227 | forwarding bool // 已经迁移完成 228 | rwlock sync.RWMutex 229 | index uint64 230 | newBuckets *[]uintptr 231 | Head *entry.NodeEntry 232 | /* 233 | Tree *entry.Tree 234 | isTree bool 235 | size uint64 236 | */ 237 | } 238 | 239 | func TestMMCopyString(t *testing.T) { 240 | f := &xmm.Factory{} 241 | mm, err := f.CreateMemory(0.75) 242 | if err != nil { 243 | t.Fatal(err) 244 | } 245 | var item1, item2 string 246 | for i := 0; i < 10000000; i++ { 247 | item1, item2, err = mm.From2("sdsddsds", "sdsddsdssdsds") 248 | if err != nil { 249 | t.Error(err) 250 | } 251 | } 252 | fmt.Println(item1, item2) 253 | } 254 | 255 | func TestGoCopyString(t *testing.T) { 256 | f := &xmm.Factory{} 257 | mm, err := f.CreateMemory(0.75) 258 | if err != nil { 259 | t.Fatal(err) 260 | } 261 | var item1, item2 string 262 | for i := 0; i < 10000000; i++ { 263 | item1, item2, err = mm.From2("sdsddsds", "sdsddsdssdsds") 264 | if err != nil { 265 | t.Error(err) 266 | } 267 | } 268 | fmt.Println(item1, item2) 269 | } 270 | 271 | type KV struct { 272 | K string 273 | V string 274 | } 275 | 276 | func TestDataSet(t *testing.T) { 277 | fmt.Println(string(make([]byte, 100))) 278 | num, maxLen := 10000000, 1000 279 | kvs := make([]*KV, num) 280 | for i := 0; i < num; i++ { 281 | r := rand.New(rand.NewSource(time.Now().UnixNano())).Int() 282 | k := RandString(r % maxLen) 283 | kvs[i] = &KV{ 284 | K: cast.ToString(r % num), 285 | V: k, 286 | } 287 | } 288 | fmt.Println(RandString(1000)) 289 | } 290 | 291 | func RandString(len int) string { 292 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 293 | bytes := make([]byte, len) 294 | for i := 0; i < len; i++ { 295 | b := r.Intn(26) + 65 296 | bytes[i] = byte(b) 297 | } 298 | return string(bytes) 299 | } 300 | 301 | /*func TestDecode(t *testing.T) { 302 | node := entry.NodeEntry{} 303 | sss := node.Int64Encode(212121) 304 | fmt.Println(node.Int64Decode(sss)) 305 | 306 | }*/ 307 | -------------------------------------------------------------------------------- /xmap/map_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 XDS project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // XDS Project Site: https://github.com/heiyeluren 16 | // XDS URL: https://github.com/heiyeluren/xds 17 | // 18 | 19 | package xmap 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "github.com/heiyeluren/xds" 25 | "github.com/heiyeluren/xds/xmap/entry" 26 | "github.com/heiyeluren/xmm" 27 | "log" 28 | "math/rand" 29 | "net/http" 30 | _ "net/http/pprof" 31 | "os" 32 | "reflect" 33 | "runtime" 34 | "strconv" 35 | "strings" 36 | "sync" 37 | "sync/atomic" 38 | "testing" 39 | "time" 40 | "unsafe" 41 | // "xds/xmap/entry" 42 | "github.com/spf13/cast" 43 | ) 44 | 45 | type User struct { 46 | Name int 47 | Age int 48 | Addr string 49 | } 50 | 51 | // mudi : 1001 -> 10000 52 | // 1001 << 1 -> 10010 53 | // 10010 & (1001 & 0) 54 | func round(n uintptr) uintptr { 55 | return (n << 1) & (0 & (n)) 56 | } 57 | 58 | type mmanClass uint8 59 | 60 | func makemmanClass(sizeclass uint8, noscan bool) mmanClass { 61 | return mmanClass(sizeclass<<1) | mmanClass(bool2int(noscan)) 62 | } 63 | 64 | func (sc mmanClass) sizeclass() int8 { 65 | return int8(sc >> 1) 66 | } 67 | 68 | func (sc mmanClass) noscan() bool { 69 | return sc&1 != 0 70 | } 71 | 72 | func bool2int(x bool) int { 73 | // Avoid branches. In the SSA compiler, this compiles to 74 | // exactly what you would want it to. 75 | return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) 76 | } 77 | 78 | func Init() { 79 | // 略 80 | runtime.GOMAXPROCS(6) // 限制 CPU 使用数，避免过载 81 | runtime.SetMutexProfileFraction(1) // 开启对锁调用的跟踪 82 | runtime.SetBlockProfileRate(1) // 开启对阻塞操作的跟踪 83 | 84 | go func() { 85 | // 启动一个 http server，注意 pprof 相关的 handler 已经自动注册过了 86 | if err := http.ListenAndServe(":6060", nil); err != nil { 87 | log.Fatal(err) 88 | } 89 | os.Exit(0) 90 | }() 91 | <-time.After(time.Second * 10) 92 | } 93 | 94 | func Test_xmmanPool222(t *testing.T) { 95 | // 同步扩容 680 异步扩容 551 96 | fmt.Println(100000 / (4096 / 48)) 97 | f := &xmm.Factory{} 98 | 99 | mm, err := f.CreateMemory(0.6) 100 | if err != nil { 101 | t.Fatal(err) 102 | } 103 | var s unsafe.Pointer 104 | sl, err := mm.AllocSlice(unsafe.Sizeof(s), 100000+1, 0) 105 | if err != nil { 106 | t.Fatal(err) 107 | } 108 | ssss := *(*[]unsafe.Pointer)(sl) 109 | var start unsafe.Pointer 110 | data := (*reflect.SliceHeader)(unsafe.Pointer(&ssss)).Data 111 | var us []unsafe.Pointer 112 | for i := 0; i < 100000; i++ { 113 | p, err := mm.Alloc(unsafe.Sizeof(User{})) 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | user := (*User)(p) 118 | user.Age = i 119 | user.Name = rand.Int() 120 | us = append(us, p) 121 | ssss = append(ssss, p) 122 | if start == nil { 123 | start = ssss[0] 124 | } 125 | } 126 | if (*reflect.SliceHeader)(unsafe.Pointer(&ssss)).Data != data { 127 | t.Fatal("扩容了") 128 | } 129 | fmt.Printf("sssssss %d %d\n ", start, ssss[0]) 130 | if ssss[0] != start { 131 | t.Fatal("-") 132 | } 133 | for i, pointer := range us { 134 | tep := (*User)(ssss[i]) 135 | if sss := (*User)(pointer); sss.Age != i || tep.Age != sss.Age { 136 | t.Fatalf("%+v\n", pointer) 137 | } 138 | } 139 | } 140 | 141 | func TestPointer2(t *testing.T) { 142 | tmp := make([]*User, 10000000) 143 | us := &tmp 144 | var wait sync.WaitGroup 145 | wait.Add(80) 146 | var sm sync.Map 147 | for j := 0; j < 80; j++ { 148 | go func(z int) { 149 | defer wait.Done() 150 | for i := 0; i < 10000000; i++ { 151 | key := cast.ToString(i + (z * 1000)) 152 | addr := (*unsafe.Pointer)(unsafe.Pointer(&(*us)[i])) 153 | user := atomic.LoadPointer(addr) 154 | if user == nil { 155 | ut := &User{ 156 | Name: i, 157 | Age: i, 158 | Addr: key, 159 | } 160 | ptr := unsafe.Pointer(ut) 161 | if atomic.CompareAndSwapPointer(addr, nil, ptr) { 162 | sm.Store(i, uintptr(ptr)) 163 | } 164 | } 165 | } 166 | }(j) 167 | } 168 | wait.Wait() 169 | 170 | for i := 0; i < 10000000; i++ { 171 | addr := (*unsafe.Pointer)(unsafe.Pointer(&(*us)[i])) 172 | user := atomic.LoadPointer(addr) 173 | u := (*User)(user) 174 | if val, ok := sm.Load(i); !ok || val != uintptr(unsafe.Pointer(u)) { 175 | t.Fatal(val, user, i, u) 176 | } 177 | } 178 | } 179 | 180 | func TestPointer(t *testing.T) { 181 | tmp := make([]uintptr, 1000000) 182 | var users []*User 183 | us := &tmp 184 | var wait sync.WaitGroup 185 | wait.Add(80) 186 | var sm sync.Map 187 | for j := 0; j < 80; j++ { 188 | go func(z int) { 189 | defer wait.Done() 190 | for i := 0; i < 100000; i++ { 191 | key := cast.ToString(i + (z * 1000)) 192 | addr := &((*us)[i]) 193 | user := atomic.LoadUintptr(addr) 194 | if user == 0 { 195 | ut := &User{ 196 | Name: i, 197 | Age: i, 198 | Addr: key, 199 | } 200 | ptr := uintptr(unsafe.Pointer(ut)) 201 | if atomic.CompareAndSwapUintptr(addr, 0, ptr) { 202 | users = append(users, ut) 203 | sm.Store(i, ptr) 204 | // fmt.Printf("i:%d ptr:%d\n", i, ptr) 205 | } 206 | } 207 | } 208 | }(j) 209 | } 210 | wait.Wait() 211 | 212 | for i := 0; i < 100000; i++ { 213 | addr := &((*us)[i]) 214 | user := atomic.LoadUintptr(addr) 215 | u := (*User)(unsafe.Pointer(user)) 216 | if val, ok := sm.Load(i); !ok || val != user { 217 | t.Fatal(val, user, i, u) 218 | } 219 | } 220 | } 221 | 222 | // todo CompareAndSwapPointer xuexi 223 | 224 | func TestRBTree(t *testing.T) { 225 | rbt := new(entry.Tree) 226 | rbt.SetComparator(func(o1, o2 interface{}) int { 227 | key1, key2 := o1.(string), o2.(string) 228 | return strings.Compare(key1, key2) 229 | }) 230 | num := 10000000 231 | st := time.Now() 232 | for i := 0; i < num/10; i++ { 233 | key := cast.ToString(i) 234 | ce := &entry.NodeEntry{ 235 | Value: []byte(key), 236 | Key: []byte(key), 237 | Hash: uint64(rand.Int()), 238 | } 239 | rbt.Put(ce) 240 | } 241 | for i := 0; i < num/10; i++ { 242 | exist, node := rbt.Get([]byte(strconv.Itoa(i))) 243 | if !exist || bytes.Compare(node, []byte(strconv.Itoa(i))) != 0 { 244 | panic(i) 245 | } 246 | } 247 | fmt.Println(rbt.Get([]byte(strconv.Itoa(5)))) 248 | fmt.Println(time.Now().Sub(st).Seconds()) 249 | rbt.Walk(&entry.HookVisitor{Hook: func(node *entry.NodeEntry) { 250 | fmt.Println(node) 251 | }}) 252 | } 253 | 254 | func Test_NewDefaultConcurrentHashMap(t *testing.T) { 255 | f := &xmm.Factory{} 256 | mm, err := f.CreateMemory(0.6) 257 | if err != nil { 258 | t.Fatal(err) 259 | } 260 | chmp, err := NewDefaultConcurrentHashMap(mm, xds.Uintptr, xds.Uintptr) 261 | if err != nil { 262 | t.Fatal(err) 263 | } 264 | for i := 0; i < 10000000; i++ { 265 | s := uintptr(i) 266 | if err := chmp.Put(s, s); err != nil { 267 | t.Error(err) 268 | } 269 | } 270 | for i := 0; i < 10000000; i++ { 271 | s := uintptr(i) 272 | if val, exist, err := chmp.Get(s); err != nil || val != s || !exist { 273 | t.Error("sss", err) 274 | } 275 | } 276 | } 277 | 278 | func TestXmap_ForEach(t *testing.T) { 279 | f := &xmm.Factory{} 280 | mm, err := f.CreateMemory(0.6) 281 | if err != nil { 282 | t.Fatal(err) 283 | } 284 | chmp, err := NewDefaultConcurrentHashMap(mm, xds.Uintptr, xds.Uintptr) 285 | if err != nil { 286 | t.Fatal(err) 287 | } 288 | for i := 0; i < 10000; i++ { 289 | s := uintptr(i) 290 | if err := chmp.Put(s, s); err != nil { 291 | t.Error(err) 292 | } 293 | } 294 | for i := 0; i < 10000; i++ { 295 | s := uintptr(i) 296 | if val, exist, err := chmp.Get(s); err != nil || val != s || !exist { 297 | t.Error("sss", err) 298 | } 299 | } 300 | i := 0 301 | err = chmp.ForEach(func(key, val interface{}) error { 302 | //fmt.Printf("ForEach key:%s value:%s \n", key, val) 303 | i++ 304 | return nil 305 | }) 306 | fmt.Println(i, chmp.Len()) 307 | if err != nil { 308 | t.Error("sss", err) 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /xmap/entry/rbtree.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 Gavin Bong. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, 11 | software distributed under the License is distributed on an 12 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 13 | either express or implied. See the License for the specific 14 | language governing permissions and limitations under the 15 | License. 16 | */ 17 | 18 | // Package entry Package redblacktree provides a pure Golang implementation 19 | // of a red-black tree as described by Thomas H. Cormen's et al. 20 | // in their seminal Algorithms book (3rd ed). This data structure 21 | // is not multi-goroutine safe. 22 | package entry 23 | 24 | import ( 25 | "bytes" 26 | "errors" 27 | "fmt" 28 | "io" 29 | "io/ioutil" 30 | "log" 31 | "os" 32 | "reflect" 33 | "strings" 34 | "sync" 35 | ) 36 | 37 | // NodeEntry interface defines the methods that a node must implement. 38 | type NodeEntry struct { 39 | Key []byte 40 | Value []byte 41 | Hash uint64 42 | Next *NodeEntry 43 | 44 | right *NodeEntry 45 | left *NodeEntry 46 | parent *NodeEntry 47 | color Color // 比二叉查找树要多出一个颜色属性 48 | } 49 | 50 | func round(n, a uint64) uint64 { 51 | return (n + a - 1) &^ (a - 1) 52 | } 53 | 54 | type encodedNodeEntry struct { 55 | TotalLen uint64 56 | KeyLen uint64 57 | ValueLen uint64 58 | Hash uint64 59 | Next *NodeEntry 60 | 61 | right *NodeEntry 62 | left *NodeEntry 63 | parent *NodeEntry 64 | color Color // 比二叉查找树要多出一个颜色属性 65 | 66 | // 追加string key、value 要不然就内存泄露了 67 | 68 | } 69 | 70 | /*// total'len(64) + key'len(64) + key'content + 71 | func (n *NodeEntry) encode() []byte { 72 | var keyLen, valLen = round(uint64(len(n.Key)), 8), round(uint64(len(n.Value)), 8) 73 | total := keyLen + valLen + 8*8 74 | encoded := encodedNodeEntry{ 75 | TotalLen: total, 76 | KeyLen: keyLen, 77 | ValueLen: valLen, 78 | Hash: 0, 79 | Next: nil, 80 | right: nil, 81 | left: nil, 82 | parent: nil, 83 | color: false, 84 | } 85 | 86 | }*/ 87 | 88 | // BytesAscSort is a helper function that sorts a slice of byte slices 89 | var BytesAscSort Comparator = func(o1, o2 interface{}) int { 90 | key1, key2 := o1.([]byte), o2.([]byte) 91 | return bytes.Compare(key1, key2) 92 | } 93 | 94 | // Color of a redblack tree node is either 95 | // `Black` (true) & `Red` (false) 96 | type Color bool 97 | 98 | // Direction points to either the Left or Right subtree 99 | type Direction byte 100 | 101 | func (c Color) String() string { 102 | switch c { 103 | case true: 104 | return "Black" 105 | default: 106 | return "Red" 107 | } 108 | } 109 | 110 | func (d Direction) String() string { 111 | switch d { 112 | case LEFT: 113 | return "left" 114 | case RIGHT: 115 | return "right" 116 | case NODIR: 117 | return "center" 118 | default: 119 | return "not recognized" 120 | } 121 | } 122 | 123 | const ( 124 | BLACK, RED Color = true, false 125 | ) 126 | const ( 127 | LEFT Direction = iota 128 | RIGHT 129 | NODIR 130 | ) 131 | 132 | // A node needs to be able to answer the query: 133 | // (i) Who is my parent node ? 134 | // (ii) Who is my grandparent node ? 135 | // The zero value for Node has color Red. 136 | 137 | func (n *NodeEntry) String() string { 138 | if n == nil { 139 | return "" 140 | } 141 | return fmt.Sprintf("(%#v : %s)", n.Key, n.Color()) 142 | } 143 | 144 | // Parent node will be nil if the node is the root node. 145 | func (n *NodeEntry) Parent() *NodeEntry { 146 | return n.parent 147 | } 148 | 149 | // SetColor nodes color to the given color. 150 | func (n *NodeEntry) SetColor(color Color) { 151 | n.color = color 152 | } 153 | 154 | // Color returns the color of the node. 155 | func (n *NodeEntry) Color() Color { 156 | return n.color 157 | } 158 | 159 | // Left indicates the left child of the node. 160 | func (n *NodeEntry) Left() *NodeEntry { 161 | return n.left 162 | } 163 | 164 | // Right indicates the right child of the node. 165 | func (n *NodeEntry) Right() *NodeEntry { 166 | return n.right 167 | } 168 | 169 | // Visitor is a function that is called on each node of the tree. 170 | type Visitor interface { 171 | Visit(*NodeEntry) 172 | } 173 | 174 | // Visitable A redblack tree is `Visitable` by a `Visitor`. 175 | type Visitable interface { 176 | Walk(Visitor) 177 | } 178 | 179 | // Comparator is a function that compares two objects. 180 | // Keys must be comparable. It's mandatory to provide a Comparator, 181 | // which returns zero if o1 == o2, -1 if o1 < o2, 1 if o1 > o2 182 | type Comparator func(o1, o2 interface{}) int 183 | 184 | // IntComparator is a Comparator that compares ints. 185 | // Default comparator expects keys to be of type `int`. 186 | // Warning: if either one of `o1` or `o2` cannot be asserted to `int`, it panics. 187 | func IntComparator(o1, o2 interface{}) int { 188 | i1 := o1.(int) 189 | i2 := o2.(int) 190 | switch { 191 | case i1 > i2: 192 | return 1 193 | case i1 < i2: 194 | return -1 195 | default: 196 | return 0 197 | } 198 | } 199 | 200 | // StringComparator ... 201 | // Keys of type `string`. 202 | // Warning: if either one of `o1` or `o2` cannot be asserted to `string`, it panics. 203 | func StringComparator(o1, o2 interface{}) int { 204 | s1 := o1.(string) 205 | s2 := o2.(string) 206 | return bytes.Compare([]byte(s1), []byte(s2)) 207 | } 208 | 209 | // Tree encapsulates the data structure. 210 | type Tree struct { 211 | root *NodeEntry // tip of the tree 212 | cmp Comparator // required function to order keys 213 | lock sync.RWMutex 214 | } 215 | 216 | // `lock` protects `logger` 217 | var loggerlock sync.Mutex 218 | var logger *log.Logger 219 | 220 | func init() { 221 | logger = log.New(ioutil.Discard, "", log.LstdFlags) 222 | } 223 | 224 | // TraceOn turns on logging output to Stderr 225 | func TraceOn() { 226 | SetOutput(os.Stderr) 227 | } 228 | 229 | // TraceOff turns off logging. 230 | // By default logging is turned off. 231 | func TraceOff() { 232 | SetOutput(ioutil.Discard) 233 | } 234 | 235 | // SetOutput redirects log output 236 | func SetOutput(w io.Writer) { 237 | loggerlock.Lock() 238 | defer loggerlock.Unlock() 239 | logger = log.New(w, "", log.LstdFlags) 240 | } 241 | 242 | // NewTree returns an empty Tree with default comparator `IntComparator`. 243 | // `IntComparator` expects keys to be type-assertable to `int`. 244 | func NewTree() *Tree { 245 | return &Tree{root: nil, cmp: IntComparator} 246 | } 247 | 248 | // NewTreeWith returns an empty Tree with a supplied `Comparator`. 249 | func NewTreeWith(c Comparator) *Tree { 250 | return &Tree{root: nil, cmp: c} 251 | } 252 | 253 | // SetComparator in the Tree of the supplied `Comparator`. 254 | func (t *Tree) SetComparator(c Comparator) { 255 | t.cmp = c 256 | } 257 | 258 | // Get looks for the node with supplied key and returns its mapped payload. 259 | // Return value in 1st position indicates whether any payload was found. 260 | func (t *Tree) Get(key []byte) (bool, []byte) { 261 | if err := mustBeValidKey(key); err != nil { 262 | // logger.Printf("Get was prematurely aborted: %s\n", err.Error()) 263 | return false, nil 264 | } 265 | ok, node := t.getNode(key) 266 | if ok { 267 | return true, node.Value 268 | } 269 | return false, nil 270 | } 271 | 272 | func (t *Tree) getNode(key interface{}) (bool, *NodeEntry) { 273 | found, parent, dir := t.GetParent(key) 274 | if found { 275 | if parent == nil { 276 | return true, t.root 277 | } 278 | var node *NodeEntry 279 | switch dir { 280 | case LEFT: 281 | node = parent.left 282 | case RIGHT: 283 | node = parent.right 284 | } 285 | 286 | if node != nil { 287 | return true, node 288 | } 289 | } 290 | return false, nil 291 | } 292 | 293 | // getMinimum returns the node with minimum key starting 294 | // at the subtree rooted at node x. Assume x is not nil. 295 | func (t *Tree) getMinimum(x *NodeEntry) *NodeEntry { 296 | for { 297 | if x.left != nil { 298 | x = x.left 299 | } else { 300 | return x 301 | } 302 | } 303 | } 304 | 305 | // GetParent looks for the node with supplied key and returns the parent node. 306 | func (t *Tree) GetParent(key interface{}) (found bool, parent *NodeEntry, dir Direction) { 307 | if err := mustBeValidKey(key); err != nil { 308 | // logger.Printf("GetParent was prematurely aborted: %s\n", err.Error()) 309 | return false, nil, NODIR 310 | } 311 | 312 | if t.root == nil { 313 | return false, nil, NODIR 314 | } 315 | t.lock.RLock() 316 | defer t.lock.RUnlock() 317 | return t.internalLookup(nil, t.root, key, NODIR) 318 | } 319 | 320 | func (t *Tree) internalLookup(parent *NodeEntry, this *NodeEntry, key interface{}, dir Direction) (bool, *NodeEntry, Direction) { 321 | switch { 322 | case this == nil: 323 | return false, parent, dir 324 | case t.cmp(key, this.Key) == 0: 325 | return true, parent, dir 326 | case t.cmp(key, this.Key) < 0: 327 | return t.internalLookup(this, this.left, key, LEFT) 328 | case t.cmp(key, this.Key) > 0: 329 | return t.internalLookup(this, this.right, key, RIGHT) 330 | default: 331 | return false, parent, NODIR 332 | } 333 | } 334 | 335 | // RotateRight Reverses actions of RotateLeft 336 | func (t *Tree) RotateRight(y *NodeEntry) { 337 | if y == nil { 338 | // logger.Printf("RotateRight: nil arg cannot be rotated. Noop\n") 339 | return 340 | } 341 | if y.left == nil { 342 | // logger.Printf("RotateRight: y has nil left subtree. Noop\n") 343 | return 344 | } 345 | // logger.Printf("\t\t\trotate right of %s\n", y) 346 | x := y.left 347 | y.left = x.right 348 | if x.right != nil { 349 | x.right.parent = y 350 | } 351 | x.parent = y.parent 352 | if y.parent == nil { 353 | t.root = x 354 | } else { 355 | if y == y.parent.left { 356 | y.parent.left = x 357 | } else { 358 | y.parent.right = x 359 | } 360 | } 361 | x.right = y 362 | y.parent = x 363 | } 364 | 365 | // RotateLeft Reverses actions of RotateRight 366 | // Side-effect: red-black tree properties is maintained. 367 | func (t *Tree) RotateLeft(x *NodeEntry) { 368 | if x == nil { 369 | // logger.Printf("RotateLeft: nil arg cannot be rotated. Noop\n") 370 | return 371 | } 372 | if x.right == nil { 373 | // logger.Printf("RotateLeft: x has nil right subtree. Noop\n") 374 | return 375 | } 376 | // logger.Printf("\t\t\trotate left of %s\n", x) 377 | 378 | y := x.right 379 | x.right = y.left 380 | if y.left != nil { 381 | y.left.parent = x 382 | } 383 | y.parent = x.parent 384 | if x.parent == nil { 385 | t.root = y 386 | } else { 387 | if x == x.parent.left { 388 | x.parent.left = y 389 | } else { 390 | x.parent.right = y 391 | } 392 | } 393 | y.left = x 394 | x.parent = y 395 | } 396 | 397 | // Put saves the mapping (key, data) into the tree. 398 | // If a mapping identified by `key` already exists, it is overwritten. 399 | // Constraint: Not everything can be a key. 400 | func (t *Tree) Put(node *NodeEntry) error { 401 | node.parent, node.left, node.Next, node.right = nil, nil, nil, nil 402 | key, data := node.Key, node.Value 403 | if err := mustBeValidKey(key); err != nil { 404 | // logger.Printf("Put was prematurely aborted: %s\n", err.Error()) 405 | return err 406 | } 407 | t.lock.Lock() 408 | defer t.lock.Unlock() 409 | if t.root == nil { 410 | node.color = BLACK 411 | t.root = node 412 | // logger.Printf("Added %s as root node\n", t.root.String()) 413 | return nil 414 | } 415 | 416 | found, parent, dir := t.internalLookup(nil, t.root, key, NODIR) 417 | if found { 418 | if parent == nil { 419 | // logger.Printf("Put: parent=nil & found. Overwrite ROOT node\n") 420 | t.root.Value = data 421 | } else { 422 | // logger.Printf("Put: parent!=nil & found. Overwriting\n") 423 | switch dir { 424 | case LEFT: 425 | parent.left.Value = data 426 | case RIGHT: 427 | parent.right.Value = data 428 | } 429 | } 430 | 431 | } else { 432 | if parent != nil { 433 | node.parent = parent 434 | newNode := node 435 | switch dir { 436 | case LEFT: 437 | parent.left = newNode 438 | case RIGHT: 439 | parent.right = newNode 440 | } 441 | // logger.Printf("Added %s to %s node of parent %s\n", newNode.String(), dir, parent.String()) 442 | t.fixupPut(newNode) 443 | } 444 | } 445 | return nil 446 | } 447 | 448 | func isRed(n *NodeEntry) bool { 449 | key := reflect.ValueOf(n) 450 | if key.IsNil() { 451 | return false 452 | } 453 | return n.color == RED 454 | } 455 | 456 | // fix possible violations of red-black-tree properties 457 | // with combinations of: 458 | // 1. recoloring 459 | // 2. rotations 460 | // 461 | // Preconditions: 462 | // P1) z is not nil 463 | // 464 | // @param z - the newly added Node to the tree. 465 | func (t *Tree) fixupPut(z *NodeEntry) { 466 | // logger.Printf("\tfixup new node z %s\n", z.String()) 467 | loop: 468 | for { 469 | // logger.Printf("\tcurrent z %s\n", z.String()) 470 | switch { 471 | case z.parent == nil: 472 | fallthrough 473 | case z.parent.color == BLACK: 474 | fallthrough 475 | default: 476 | // When the loop terminates, it does so because p[z] is black. 477 | // logger.Printf("\t\t=> bye\n") 478 | break loop 479 | case z.parent.color == RED: 480 | grandparent := z.parent.parent 481 | // logger.Printf("\t\tgrandparent is nil %t addr:%d\n", grandparent == nil, unsafe.Pointer(t)) 482 | if z.parent == grandparent.left { 483 | // logger.Printf("\t\t%s is the left child of %s\n", z.parent, grandparent) 484 | y := grandparent.right 485 | // //logger.Printf("\t\ty (right) %s\n", y) 486 | if isRed(y) { 487 | // case 1 - y is RED 488 | // logger.Printf("\t\t(*) case 1\n") 489 | z.parent.color = BLACK 490 | y.color = BLACK 491 | grandparent.color = RED 492 | z = grandparent 493 | 494 | } else { 495 | if z == z.parent.right { 496 | // case 2 497 | // logger.Printf("\t\t(*) case 2\n") 498 | z = z.parent 499 | t.RotateLeft(z) 500 | } 501 | 502 | // case 3 503 | // logger.Printf("\t\t(*) case 3\n") 504 | z.parent.color = BLACK 505 | grandparent.color = RED 506 | t.RotateRight(grandparent) 507 | } 508 | } else { 509 | // logger.Printf("\t\t%s is the right child of %s\n", z.parent, grandparent) 510 | y := grandparent.left 511 | // logger.Printf("\t\ty (left) %s\n", y) 512 | if isRed(y) { 513 | // case 1 - y is RED 514 | // logger.Printf("\t\t..(*) case 1\n") 515 | z.parent.color = BLACK 516 | y.color = BLACK 517 | grandparent.color = RED 518 | z = grandparent 519 | 520 | } else { 521 | // logger.Printf("\t\t## %s\n", z.parent.left) 522 | if z == z.parent.left { 523 | // case 2 524 | // logger.Printf("\t\t..(*) case 2\n") 525 | z = z.parent 526 | t.RotateRight(z) 527 | } 528 | 529 | // case 3 530 | // logger.Printf("\t\t..(*) case 3\n") 531 | z.parent.color = BLACK 532 | grandparent.color = RED 533 | t.RotateLeft(grandparent) 534 | } 535 | } 536 | } 537 | } 538 | t.root.color = BLACK 539 | } 540 | 541 | // Size returns the number of items in the tree. 542 | func (t *Tree) Size() uint64 { 543 | visitor := &countingVisitor{} 544 | t.Walk(visitor) 545 | return visitor.Count 546 | } 547 | 548 | // Has checks for existence of a item identified by supplied key. 549 | func (t *Tree) Has(key interface{}) bool { 550 | if err := mustBeValidKey(key); err != nil { 551 | // logger.Printf("Has was prematurely aborted: %s\n", err.Error()) 552 | return false 553 | } 554 | found, _, _ := t.internalLookup(nil, t.root, key, NODIR) 555 | return found 556 | } 557 | 558 | func (t *Tree) transplant(u *NodeEntry, v *NodeEntry) { 559 | if u.parent == nil { 560 | t.root = v 561 | } else if u == u.parent.left { 562 | u.parent.left = v 563 | } else { 564 | u.parent.right = v 565 | } 566 | if v != nil && u != nil { 567 | v.parent = u.parent 568 | } 569 | } 570 | 571 | // Delete removes the item identified by the supplied key. 572 | // Delete is a noop if the supplied key doesn't exist. 573 | func (t *Tree) Delete(key []byte) *NodeEntry { 574 | if !t.Has(key) { 575 | // logger.Printf("Delete: bail as no node exists for key %d\n", key) 576 | return nil 577 | } 578 | _, z := t.getNode(key) 579 | y := z 580 | yOriginalColor := y.color 581 | var x *NodeEntry 582 | 583 | if z.left == nil { 584 | // one child (RIGHT) 585 | // logger.Printf("\t\tDelete: case (a)\n") 586 | x = z.right 587 | // logger.Printf("\t\t\t--- x is right of z") 588 | t.transplant(z, z.right) 589 | 590 | } else if z.right == nil { 591 | // one child (LEFT) 592 | // logger.Printf("\t\tDelete: case (b)\n") 593 | x = z.left 594 | // logger.Printf("\t\t\t--- x is left of z") 595 | t.transplant(z, z.left) 596 | 597 | } else { 598 | // two children 599 | // logger.Printf("\t\tDelete: case (c) & (d)\n") 600 | y = t.getMinimum(z.right) 601 | // logger.Printf("\t\t\tminimum of z.right is %s (color=%s)\n", y, y.color) 602 | yOriginalColor = y.color 603 | x = y.right 604 | // logger.Printf("\t\t\t--- x is right of minimum") 605 | 606 | if y.parent == z { 607 | if x != nil { 608 | x.parent = y 609 | } 610 | } else { 611 | t.transplant(y, y.right) 612 | y.right = z.right 613 | y.right.parent = y 614 | } 615 | t.transplant(z, y) 616 | y.left = z.left 617 | y.left.parent = y 618 | y.color = z.color 619 | } 620 | if yOriginalColor == BLACK { 621 | t.fixupDelete(x) 622 | } 623 | return z 624 | } 625 | 626 | func (t *Tree) fixupDelete(x *NodeEntry) { 627 | // logger.Printf("\t\t\tfixupDelete of node %s\n", x) 628 | if x == nil { 629 | return 630 | } 631 | loop: 632 | for { 633 | switch { 634 | case x == t.root: 635 | // logger.Printf("\t\t\t=> bye .. is root\n") 636 | break loop 637 | case x.color == RED: 638 | // logger.Printf("\t\t\t=> bye .. RED\n") 639 | break loop 640 | case x == x.parent.right: 641 | // logger.Printf("\t\tBRANCH: x is right child of parent\n") 642 | w := x.parent.left // is nillable 643 | if isRed(w) { 644 | // Convert case 1 into case 2, 3, or 4 645 | // logger.Printf("\t\t\tR> case 1\n") 646 | w.color = BLACK 647 | x.parent.color = RED 648 | t.RotateRight(x.parent) 649 | w = x.parent.left 650 | } 651 | if w != nil { 652 | switch { 653 | case !isRed(w.left) && !isRed(w.right): 654 | // case 2 - both children of w are BLACK 655 | // logger.Printf("\t\t\tR> case 2\n") 656 | w.color = RED 657 | x = x.parent // recurse up tree 658 | case isRed(w.right) && !isRed(w.left): 659 | // case 3 - right child RED & left child BLACK 660 | // convert to case 4 661 | // logger.Printf("\t\t\tR> case 3\n") 662 | w.right.color = BLACK 663 | w.color = RED 664 | t.RotateLeft(w) 665 | w = x.parent.left 666 | } 667 | if isRed(w.left) { 668 | // case 4 - left child is RED 669 | // logger.Printf("\t\t\tR> case 4\n") 670 | w.color = x.parent.color 671 | x.parent.color = BLACK 672 | w.left.color = BLACK 673 | t.RotateRight(x.parent) 674 | x = t.root 675 | } 676 | } 677 | case x == x.parent.left: 678 | // logger.Printf("\t\tBRANCH: x is left child of parent\n") 679 | w := x.parent.right // is nillable 680 | if isRed(w) { 681 | // Convert case 1 into case 2, 3, or 4 682 | // logger.Printf("\t\t\tL> case 1\n") 683 | w.color = BLACK 684 | x.parent.color = RED 685 | t.RotateLeft(x.parent) 686 | w = x.parent.right 687 | } 688 | if w != nil { 689 | switch { 690 | case !isRed(w.left) && !isRed(w.right): 691 | // case 2 - both children of w are BLACK 692 | // logger.Printf("\t\t\tL> case 2\n") 693 | w.color = RED 694 | x = x.parent // recurse up tree 695 | case isRed(w.left) && !isRed(w.right): 696 | // case 3 - left child RED & right child BLACK 697 | // convert to case 4 698 | // logger.Printf("\t\t\tL> case 3\n") 699 | w.left.color = BLACK 700 | w.color = RED 701 | t.RotateRight(w) 702 | w = x.parent.right 703 | } 704 | if isRed(w.right) { 705 | // case 4 - right child is RED 706 | // logger.Printf("\t\t\tL> case 4\n") 707 | w.color = x.parent.color 708 | x.parent.color = BLACK 709 | w.right.color = BLACK 710 | t.RotateLeft(x.parent) 711 | x = t.root 712 | } 713 | } 714 | } 715 | } 716 | x.color = BLACK 717 | } 718 | 719 | // Walk accepts a Visitor 720 | func (t *Tree) Walk(visitor Visitor) { 721 | visitor.Visit(t.root) 722 | } 723 | 724 | // GetRoot get root node 725 | func (t *Tree) GetRoot() *NodeEntry { 726 | return t.root 727 | } 728 | 729 | // countingVisitor counts the number 730 | // of nodes in the tree. 731 | type countingVisitor struct { 732 | Count uint64 733 | } 734 | 735 | // Visit . 736 | func (v *countingVisitor) Visit(node *NodeEntry) { 737 | if node == nil { 738 | return 739 | } 740 | 741 | v.Visit(node.left) 742 | v.Count = v.Count + 1 743 | v.Visit(node.right) 744 | } 745 | 746 | // InorderVisitor walks the tree in inorder fashion. 747 | // This visitor maintains internal state; thus do not 748 | // reuse after the completion of a walk. 749 | type InorderVisitor struct { 750 | buffer bytes.Buffer 751 | } 752 | 753 | // Eq 比较 754 | func (v *InorderVisitor) Eq(other *InorderVisitor) bool { 755 | if other == nil { 756 | return false 757 | } 758 | return v.String() == other.String() 759 | } 760 | 761 | func (v *InorderVisitor) trim(s string) string { 762 | return strings.TrimRight(strings.TrimRight(s, "ed"), "lack") 763 | } 764 | 765 | // String returns string 766 | func (v *InorderVisitor) String() string { 767 | return v.buffer.String() 768 | } 769 | 770 | // Visit . 771 | func (v *InorderVisitor) Visit(node *NodeEntry) { 772 | if node == nil { 773 | v.buffer.Write([]byte(".")) 774 | return 775 | } 776 | v.buffer.Write([]byte("(")) 777 | v.Visit(node.left) 778 | v.buffer.Write([]byte(fmt.Sprintf("%s", node.Key))) // @TODO 779 | // v.buffer.Write([]byte(fmt.Sprintf("%d{%s}", node.Key, v.trim(node.color.String())))) 780 | v.Visit(node.right) 781 | v.buffer.Write([]byte(")")) 782 | } 783 | 784 | // HookVisitor . 785 | type HookVisitor struct { 786 | Hook func(node *NodeEntry) 787 | } 788 | 789 | // Visit . 790 | func (v *HookVisitor) Visit(node *NodeEntry) { 791 | if node == nil { 792 | return 793 | } 794 | v.Hook(node) 795 | v.Visit(node.left) 796 | v.Visit(node.right) 797 | } 798 | 799 | var ( 800 | // ErrorKeyIsNil the literal nil not allowed as keys 801 | ErrorKeyIsNil = errors.New("the literal nil not allowed as keys") 802 | // ErrorKeyDisallowed disallowed key type 803 | ErrorKeyDisallowed = errors.New("disallowed key type") 804 | ) 805 | 806 | // Allowed key types are: Boolean, Integer, Floating point, Complex, String values 807 | // And structs containing these. 808 | // @TODO Should pointer type be allowed ? 809 | func mustBeValidKey(key interface{}) error { 810 | if key == nil { 811 | return ErrorKeyIsNil 812 | } 813 | 814 | /*keyValue := reflect.ValueOf(key) 815 | switch keyValue.Kind() { 816 | case reflect.Chan: 817 | fallthrough 818 | case reflect.Func: 819 | fallthrough 820 | case reflect.Interface: 821 | fallthrough 822 | case reflect.Map: 823 | fallthrough 824 | case reflect.Ptr: 825 | return ErrorKeyDisallowed 826 | default: 827 | return nil 828 | }*/ 829 | return nil 830 | } 831 | -------------------------------------------------------------------------------- /xmap/concurrent_raw_hash_map.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 XDS project Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // XDS Project Site: https://github.com/heiyeluren 16 | // XDS URL: https://github.com/heiyeluren/xds 17 | // 18 | 19 | package xmap 20 | 21 | import ( 22 | "bytes" 23 | "errors" 24 | "github.com/heiyeluren/xds/xmap/entry" 25 | "github.com/heiyeluren/xmm" 26 | "log" 27 | "reflect" 28 | "runtime" 29 | "sync" 30 | "sync/atomic" 31 | "unsafe" 32 | ) 33 | 34 | // MinTransferStride is the minimum number of entries to transfer between 35 | // 1、步长resize 36 | // 2、sizeCtl增加cap的cas，不允许提前resize。 37 | // 考虑 数组+ 链表方式 38 | const MinTransferStride = 16 39 | const maximumCapacity = 1 << 30 40 | 41 | // KeyExists 标识key已经存在 42 | const KeyExists = true 43 | 44 | // KeyNotExists 标识key不存在 45 | const KeyNotExists = false 46 | 47 | var NotFound = errors.New("not found") 48 | var _BucketSize = unsafe.Sizeof(Bucket{}) 49 | var _ForwardingBucketSize = unsafe.Sizeof(ForwardingBucket{}) 50 | var _NodeEntrySize = unsafe.Sizeof(entry.NodeEntry{}) 51 | var _TreeSize = unsafe.Sizeof(entry.Tree{}) 52 | var uintPtrSize = uintptr(8) 53 | 54 | // ConcurrentRawHashMap is a concurrent hash map with a fixed number of buckets. 55 | // 清理方式：1、del清除entry(简单) 56 | // 2、buckets清除旧的（resize结束，并无get读引用【引入重试来解决该问题】） 57 | // 3、Bucket清除 58 | // 4、ForwardingBucket清除 59 | // 5、Tree 清除（spliceEntry2时候） 60 | type ConcurrentRawHashMap struct { 61 | size uint64 62 | threshold uint64 63 | initCap uint64 64 | // off-heap 65 | buckets *[]uintptr 66 | mm xmm.XMemory 67 | lock sync.RWMutex 68 | 69 | treeSize uint64 70 | 71 | // resize 72 | sizeCtl int64 // -1 正在扩容 73 | reSizeGen uint64 74 | 75 | nextBuckets *[]uintptr 76 | transferIndex uint64 77 | 78 | destroyed uint32 // 0:未销毁 1:已经销毁 79 | 80 | destroyLock sync.RWMutex 81 | } 82 | 83 | // Bucket is a hash bucket. 84 | type Bucket struct { 85 | forwarding bool // 已经迁移完成 86 | rwLock sync.RWMutex 87 | index uint64 88 | newBuckets *[]uintptr 89 | Head *entry.NodeEntry 90 | Tree *entry.Tree 91 | isTree bool 92 | size uint64 93 | } 94 | 95 | // ForwardingBucket is a hash bucket that has been forwarded to a new table. 96 | type ForwardingBucket struct { 97 | forwarding bool // 已经迁移完成 98 | rwLock sync.RWMutex 99 | index uint64 100 | newBuckets *[]uintptr 101 | } 102 | 103 | // Snapshot 利用快照比对产生 104 | type Snapshot struct { 105 | buckets *[]uintptr 106 | sizeCtl int64 // -1 正在扩容 107 | nextBuckets *[]uintptr 108 | } 109 | 110 | // NewDefaultConcurrentRawHashMap returns a new ConcurrentRawHashMap with the default 111 | // mm: xmm 112 | func NewDefaultConcurrentRawHashMap(mm xmm.XMemory) (*ConcurrentRawHashMap, error) { 113 | return NewConcurrentRawHashMap(mm, 16, 0.75, 8) 114 | } 115 | 116 | // NewConcurrentRawHashMap will create a new ConcurrentRawHashMap with the given 117 | // mm: xmm 内存对象 118 | // cap:初始化bucket长度 （可以理解为 map 元素预计最大个数~，如果知道这个值可以提前传递） 119 | // fact:负责因子，当存放的元素超过该百分比，就会触发扩容。 120 | // treeSize：bucket中的链表长度达到该值后，会转换为红黑树。 121 | func NewConcurrentRawHashMap(mm xmm.XMemory, cap uintptr, fact float64, treeSize uint64) (*ConcurrentRawHashMap, error) { 122 | if cap < 1 { 123 | return nil, errors.New("cap < 1") 124 | } 125 | var alignCap uintptr 126 | for i := 1; i < 64 && cap > alignCap; i++ { 127 | alignCap = 1 << uint(i) 128 | } 129 | cap = alignCap 130 | bucketsPtr, err := mm.AllocSlice(uintPtrSize, cap, cap) 131 | if err != nil { 132 | return nil, err 133 | } 134 | buckets := (*[]uintptr)(bucketsPtr) 135 | return &ConcurrentRawHashMap{buckets: buckets, initCap: uint64(cap), threshold: uint64(float64(cap) * fact), 136 | mm: mm, treeSize: treeSize}, nil 137 | } 138 | 139 | func (chm *ConcurrentRawHashMap) getBucket(h uint64, tab *[]uintptr) *Bucket { 140 | mask := uint64(cap(*tab) - 1) 141 | idx := h & mask 142 | _, _, bucket := chm.tabAt(tab, idx) 143 | if bucket != nil && bucket.forwarding && chm.transferIndex >= 0 { 144 | return chm.getBucket(h, bucket.newBuckets) 145 | } 146 | return bucket 147 | } 148 | 149 | // Get Fetch key from hashmap 150 | func (chm *ConcurrentRawHashMap) Get(key []byte) (val []byte, keyExists bool, err error) { 151 | h := BKDRHashWithSpread(key) 152 | bucket := chm.getBucket(h, chm.buckets) 153 | if bucket == nil { 154 | return nil, KeyNotExists, NotFound 155 | } 156 | if bucket.isTree { 157 | exist, value := bucket.Tree.Get(key) 158 | if !exist { 159 | return nil, KeyNotExists, NotFound 160 | } 161 | return value, KeyExists, nil 162 | } 163 | keySize := len(key) 164 | for cNode := bucket.Head; cNode != nil; cNode = cNode.Next { 165 | if keySize == len(cNode.Key) && bytes.Compare(key, cNode.Key) == 0 { 166 | return cNode.Value, KeyExists, nil 167 | } 168 | } 169 | return nil, KeyNotExists, NotFound 170 | } 171 | func (chm *ConcurrentRawHashMap) initForwardingEntries(newBuckets *[]uintptr, index uint64) (*ForwardingBucket, error) { 172 | entriesPtr, err := chm.mm.Alloc(_ForwardingBucketSize) 173 | if err != nil { 174 | return nil, err 175 | } 176 | entries := (*ForwardingBucket)(entriesPtr) 177 | chm.assignmentForwardingEntries(newBuckets, entries, index) 178 | return entries, nil 179 | } 180 | 181 | func (chm *ConcurrentRawHashMap) assignmentForwardingEntries(newBuckets *[]uintptr, entries *ForwardingBucket, index uint64) { 182 | entries.newBuckets = newBuckets 183 | entries.index = index 184 | entries.forwarding = true 185 | } 186 | 187 | func (chm *ConcurrentRawHashMap) initEntries(entry *entry.NodeEntry, idx uint64) (*Bucket, error) { 188 | ptr, err := chm.mm.Alloc(_BucketSize) 189 | if err != nil { 190 | return nil, err 191 | } 192 | entries := (*Bucket)(ptr) 193 | chm.assignmentEntries(entries, entry, idx) 194 | return entries, nil 195 | } 196 | 197 | func (chm *ConcurrentRawHashMap) assignmentEntries(entries *Bucket, entry *entry.NodeEntry, index uint64) { 198 | entries.index = index 199 | entries.Head = entry 200 | } 201 | 202 | func (chm *ConcurrentRawHashMap) getStride(length uint64) (stride uint64) { 203 | cpuNum := uint64(runtime.NumCPU()) 204 | if cpuNum > 1 { 205 | stride = (length >> 3) / cpuNum 206 | } else { 207 | stride = length 208 | } 209 | if stride < MinTransferStride { 210 | stride = MinTransferStride 211 | } 212 | return stride 213 | } 214 | 215 | func (chm *ConcurrentRawHashMap) resizeStamp(length uint64) (stride int64) { 216 | return -10000 - int64(length) 217 | } 218 | 219 | func (chm *ConcurrentRawHashMap) helpTransform(entry *entry.NodeEntry, bucket *Bucket, tab *[]uintptr) (currentBucket *Bucket, init bool, currentTab *[]uintptr, err error) { 220 | if bucket != nil && bucket.forwarding { 221 | if err := chm.reHashSize(tab); err != nil { 222 | return nil, init, nil, err 223 | } 224 | tabPtr := bucket.newBuckets 225 | bucket, swapped, err := chm.getAndInitBucket(entry, tabPtr) 226 | if err != nil { 227 | return nil, init, nil, err 228 | } 229 | if swapped { 230 | return nil, true, nil, err 231 | } 232 | return bucket, init, tabPtr, nil 233 | } 234 | return bucket, init, tab, nil 235 | } 236 | 237 | func (chm *ConcurrentRawHashMap) indexAndInitBucket(entry *entry.NodeEntry) (entries *Bucket, init bool, tabPtr *[]uintptr, err error) { 238 | tabPtr = chm.buckets 239 | bucket, swapped, err := chm.getAndInitBucket(entry, tabPtr) 240 | if err != nil { 241 | return nil, init, nil, err 242 | } 243 | if swapped { 244 | return bucket, true, tabPtr, nil 245 | } 246 | if bucket != nil && bucket.forwarding && chm.transferIndex >= 0 { 247 | bucket, swapped, tabPtr, err = chm.helpTransform(entry, bucket, tabPtr) 248 | if err != nil { 249 | return nil, init, nil, err 250 | } 251 | if swapped { 252 | return bucket, true, tabPtr, nil 253 | } 254 | } 255 | return bucket, init, tabPtr, nil 256 | } 257 | 258 | func (chm *ConcurrentRawHashMap) index(h uint64, length int) uint64 { 259 | idx := h & uint64(length-1) 260 | return idx 261 | } 262 | 263 | func (chm *ConcurrentRawHashMap) tabAt(buckets *[]uintptr, idx uint64) (*uintptr, uintptr, *Bucket) { 264 | addr := &((*buckets)[idx]) 265 | ptr := atomic.LoadUintptr(addr) 266 | if ptr == 0 { 267 | return addr, ptr, nil 268 | } 269 | bucket := (*Bucket)(unsafe.Pointer(ptr)) 270 | return addr, ptr, bucket 271 | } 272 | 273 | // cas 设置bucket 274 | func (chm *ConcurrentRawHashMap) getAndInitBucket(entry *entry.NodeEntry, tabPtr *[]uintptr) (bucket *Bucket, swapped bool, err error) { 275 | h := entry.Hash 276 | idx := chm.index(h, cap(*tabPtr)) 277 | // retry := 10改小后，出现该问题。 278 | addr, _, bucket := chm.tabAt(tabPtr, idx) 279 | if bucket != nil { 280 | return bucket, false, nil 281 | } 282 | entity, err := chm.initEntries(entry, idx) 283 | if err != nil { 284 | return nil, false, err 285 | } 286 | ptr := uintptr(unsafe.Pointer(entity)) 287 | atomic.CompareAndSwapUintptr(addr, 0, ptr) 288 | bucket = (*Bucket)(unsafe.Pointer(atomic.LoadUintptr(addr))) 289 | return bucket, swapped, nil 290 | } 291 | 292 | func (chm *ConcurrentRawHashMap) increaseSize() (newSize uint64) { 293 | retry := 30 294 | var swapped bool 295 | for !swapped && retry > 0 { 296 | retry-- 297 | oldVal := atomic.LoadUint64(&chm.size) 298 | newVal := oldVal + 1 299 | swapped = atomic.CompareAndSwapUint64(&chm.size, oldVal, newVal) 300 | if swapped { 301 | return newVal 302 | } 303 | } 304 | chm.lock.Lock() 305 | defer chm.lock.Unlock() 306 | size := chm.size + 1 307 | chm.size = size 308 | return size 309 | } 310 | 311 | func (chm *ConcurrentRawHashMap) createEntry(key []byte, val []byte, hash uint64) (*entry.NodeEntry, error) { 312 | entryPtr, err := chm.mm.Alloc(_NodeEntrySize) 313 | if err != nil { 314 | return nil, err 315 | } 316 | node := (*entry.NodeEntry)(entryPtr) 317 | keyPtr, valPtr, err := chm.mm.Copy2(key, val) 318 | if err != nil { 319 | return nil, err 320 | } 321 | chm.entryAssignment(keyPtr, valPtr, hash, node) 322 | return node, nil 323 | } 324 | 325 | func (chm *ConcurrentRawHashMap) entryAssignment(keyPtr []byte, valPtr []byte, hash uint64, entry *entry.NodeEntry) { 326 | entry.Key = keyPtr 327 | entry.Value = valPtr 328 | entry.Hash = hash 329 | } 330 | 331 | func (chm *ConcurrentRawHashMap) entryAssignmentCpy(keyPtr []byte, valPtr []byte, hash uint64, nodePtr unsafe.Pointer) error { 332 | source := entry.NodeEntry{Value: keyPtr, Key: valPtr, Hash: hash} 333 | offset := 40 // unsafe.Offsetof(source.Next) //40 334 | srcData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&source)), Len: offset, Cap: offset})) 335 | dstData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(nodePtr), Len: offset, Cap: offset})) 336 | if offset != copy(*dstData, *srcData) { 337 | return errors.New("incorrect copy length") // 拷贝长度不正确 338 | } 339 | return nil 340 | } 341 | 342 | func (chm *ConcurrentRawHashMap) putVal(key []byte, val []byte, h uint64) (*[]uintptr, error) { 343 | node, err := chm.createEntry(key, val, h) 344 | if err != nil { 345 | return nil, err 346 | } 347 | loop := true 348 | var tabPtr *[]uintptr 349 | for loop { 350 | bucket, init, newTabPtr, err := chm.indexAndInitBucket(node) 351 | if err != nil { 352 | return nil, err 353 | } 354 | tabPtr = newTabPtr 355 | if init { 356 | break 357 | } 358 | if loop, err = chm.PutBucketValue(bucket, node, tabPtr); err != nil { 359 | return nil, err 360 | } 361 | } 362 | return tabPtr, nil 363 | } 364 | 365 | // Put 将键值对添加到map中 366 | func (chm *ConcurrentRawHashMap) Put(key []byte, val []byte) error { 367 | h := BKDRHashWithSpread(key) 368 | tabPtr, err := chm.putVal(key, val, h) 369 | if err != nil { 370 | return err 371 | } 372 | size := chm.increaseSize() 373 | threshold := atomic.LoadUint64(&chm.threshold) 374 | if size >= threshold && size < maximumCapacity { 375 | return chm.reHashSize(tabPtr) 376 | } 377 | return nil 378 | } 379 | 380 | // Del 删除键值对 381 | func (chm *ConcurrentRawHashMap) Del(key []byte) error { 382 | h := BKDRHashWithSpread(key) 383 | return chm.delVal(key, h) 384 | } 385 | 386 | func (chm *ConcurrentRawHashMap) delVal(key []byte, h uint64) error { 387 | loop := true 388 | var tabPtr *[]uintptr 389 | tabPtr = chm.buckets 390 | for loop { 391 | idx := chm.index(h, cap(*tabPtr)) 392 | _, _, bucket := chm.tabAt(tabPtr, idx) 393 | if bucket == nil { 394 | return nil 395 | } 396 | if bucket != nil && bucket.forwarding { 397 | if err := chm.reHashSize(tabPtr); err != nil { 398 | return err 399 | } 400 | tabPtr = bucket.newBuckets 401 | idx = chm.index(h, cap(*tabPtr)) 402 | _, _, bucket = chm.tabAt(tabPtr, idx) 403 | continue 404 | } 405 | var removeNode *entry.NodeEntry 406 | func() { 407 | // 删除bucket中的数目 408 | bucket.rwLock.Lock() 409 | defer bucket.rwLock.Unlock() 410 | _, _, newBucket := chm.tabAt(tabPtr, chm.index(h, cap(*tabPtr))) 411 | if newBucket != bucket || (bucket != nil && bucket.forwarding) { 412 | return 413 | } 414 | if bucket.isTree { 415 | if node := bucket.Tree.Delete(key); node != nil { 416 | removeNode = node 417 | } 418 | } else { 419 | keySize := len(key) 420 | var pre *entry.NodeEntry 421 | for cNode := bucket.Head; cNode != nil; cNode = cNode.Next { 422 | if keySize == len(cNode.Key) && bytes.Compare(key, cNode.Key) == 0 { 423 | removeNode = cNode 424 | if pre == nil { 425 | bucket.Head = cNode.Next 426 | } else { 427 | pre.Next = cNode.Next 428 | } 429 | break 430 | } 431 | pre = cNode 432 | } 433 | } 434 | loop = false 435 | return 436 | }() 437 | if err := chm.freeEntry(removeNode); err != nil { 438 | return err 439 | } 440 | } 441 | return nil 442 | } 443 | 444 | // freeEntry todo 异步free 445 | func (chm *ConcurrentRawHashMap) freeEntry(removeNode *entry.NodeEntry) error { 446 | if removeNode == nil { 447 | return nil 448 | } 449 | if err := chm.mm.Free(uintptr(unsafe.Pointer(removeNode))); err != nil { 450 | return err 451 | } 452 | keySlice, valSlice := (*reflect.SliceHeader)(unsafe.Pointer(&removeNode.Key)), (*reflect.SliceHeader)(unsafe.Pointer(&removeNode.Value)) 453 | if err := chm.mm.Free(keySlice.Data); err != nil { 454 | return err 455 | } 456 | if err := chm.mm.Free(valSlice.Data); err != nil { 457 | return err 458 | } 459 | return nil 460 | } 461 | 462 | func (chm *ConcurrentRawHashMap) growTree(bucket *Bucket) error { 463 | if bucket.isTree || bucket.size < chm.treeSize { 464 | return nil 465 | } 466 | treePtr, err := chm.mm.Alloc(_TreeSize) 467 | if err != nil { 468 | return err 469 | } 470 | bucket.Tree = (*entry.Tree)(treePtr) 471 | bucket.Tree.SetComparator(entry.BytesAscSort) 472 | for node := bucket.Head; node != nil; { 473 | next := node.Next 474 | if err := bucket.Tree.Put(node); err != nil { 475 | return err 476 | } 477 | node = next 478 | } 479 | bucket.Head = nil 480 | bucket.isTree = true 481 | return nil 482 | } 483 | 484 | // PutBucketValue table的可见性问题，并发问题。 485 | func (chm *ConcurrentRawHashMap) PutBucketValue(bucket *Bucket, node *entry.NodeEntry, tab *[]uintptr) (loop bool, err error) { 486 | bucket.rwLock.Lock() 487 | defer bucket.rwLock.Unlock() 488 | idx := chm.index(node.Hash, cap(*tab)) 489 | _, _, newBucket := chm.tabAt(tab, idx) 490 | if newBucket != bucket || newBucket.forwarding { 491 | return true, nil 492 | } 493 | bucket = newBucket 494 | // 树化 495 | if err := chm.growTree(bucket); err != nil { 496 | return false, err 497 | } 498 | if bucket.isTree { 499 | if err := bucket.Tree.Put(node); err != nil { 500 | return false, err 501 | } 502 | return false, nil 503 | } 504 | key, val := node.Key, node.Value 505 | var last *entry.NodeEntry 506 | for node := bucket.Head; node != nil; node = node.Next { 507 | if len(key) == len(node.Key) && bytes.Compare(node.Key, key) == 0 { 508 | node.Value = val 509 | return false, nil 510 | } 511 | if node.Next == nil { 512 | last = node 513 | } 514 | } 515 | bucket.size += 1 516 | // 加入 517 | if last == nil { 518 | bucket.Head = node 519 | return false, nil 520 | } 521 | last.Next = node 522 | return false, nil 523 | } 524 | 525 | func (chm *ConcurrentRawHashMap) expandCap(tab *[]uintptr) (s *Snapshot, need bool) { 526 | old, size, transferIndex, threshold := uint64(cap(*chm.buckets)), atomic.LoadUint64(&chm.size), 527 | atomic.LoadUint64(&chm.transferIndex), atomic.LoadUint64(&chm.threshold) 528 | if size < threshold || transferIndex >= old { 529 | return nil, false 530 | } 531 | nextBuckets, buckets, sizeCtl := chm.nextBuckets, chm.buckets, atomic.LoadInt64(&chm.sizeCtl) 532 | // 正在扩容 533 | if sizeCtl < 0 && nextBuckets != nil && cap(*nextBuckets) == int(old)*2 { 534 | if sizeCtl >= 0 { 535 | return nil, false 536 | } 537 | // 当前扩容状态正确，开始扩容 538 | if unsafe.Pointer(tab) == unsafe.Pointer(buckets) && nextBuckets != nil { 539 | if atomic.CompareAndSwapInt64(&chm.sizeCtl, sizeCtl, sizeCtl+1) { 540 | return &Snapshot{buckets: buckets, sizeCtl: sizeCtl, nextBuckets: nextBuckets}, true 541 | } 542 | } 543 | return nil, false 544 | } 545 | // 未开始扩容的判断 546 | if sizeCtl >= 0 && (old<<1) > uint64(sizeCtl) && unsafe.Pointer(tab) == unsafe.Pointer(buckets) { 547 | newSizeCtl := chm.resizeStamp(old) + 2 548 | swapped := atomic.CompareAndSwapInt64(&chm.sizeCtl, sizeCtl, newSizeCtl) 549 | // 开始扩容 550 | if swapped { 551 | newCap := old << 1 552 | bucketsPtr, err := chm.mm.AllocSlice(uintPtrSize, uintptr(newCap), uintptr(newCap)) 553 | if err != nil { 554 | log.Printf("ConcurrentRawHashMap chm.mm.Alloc newCap:%d err:%s \n", newCap, err) 555 | return nil, false 556 | } 557 | nextBuckets = (*[]uintptr)(bucketsPtr) 558 | chm.nextBuckets = nextBuckets 559 | chm.transferIndex = 0 560 | return &Snapshot{buckets: buckets, sizeCtl: sizeCtl, nextBuckets: nextBuckets}, true 561 | } 562 | return nil, false 563 | } 564 | return nil, false 565 | } 566 | 567 | func (chm *ConcurrentRawHashMap) reHashSize(tab *[]uintptr) error { 568 | // cas锁 569 | snapshot, need := chm.expandCap(tab) 570 | if !need { 571 | return nil 572 | } 573 | currentCap := uint64(cap(*snapshot.buckets)) 574 | buckets := snapshot.nextBuckets 575 | // 取当前内容oldBuckets，在oldBuckets中利用bucket的lock来避免同时操作。 576 | err := chm.reSizeBuckets(snapshot) 577 | if err != nil { 578 | return err 579 | } 580 | for { 581 | sizeCtl := atomic.LoadInt64(&chm.sizeCtl) 582 | if sizeCtl >= 0 { 583 | return nil 584 | } 585 | if atomic.CompareAndSwapInt64(&chm.sizeCtl, sizeCtl, sizeCtl-1) { 586 | if sizeCtl != chm.resizeStamp(currentCap)+2 { 587 | return nil 588 | } 589 | break 590 | } 591 | } 592 | return func() error { 593 | oldBuckets, tabPtr := chm.buckets, unsafe.Pointer(&chm.buckets) 594 | bucketsAddr := (*unsafe.Pointer)(tabPtr) 595 | old := (*reflect.SliceHeader)(atomic.LoadPointer(bucketsAddr)) 596 | if old.Cap != int(currentCap) { 597 | return nil 598 | } 599 | if atomic.CompareAndSwapPointer(bucketsAddr, unsafe.Pointer(old), unsafe.Pointer(buckets)) { 600 | // 更换内容赋值 601 | chm.nextBuckets = nil 602 | newCap := currentCap << 1 603 | atomic.StoreInt64(&chm.sizeCtl, int64(newCap)) 604 | chm.threshold = chm.threshold << 1 605 | chm.reSizeGen += 1 606 | if err := chm.freeBuckets(oldBuckets); err != nil { 607 | log.Printf("freeBuckets err:%s\n", err) 608 | } 609 | return err 610 | } 611 | return nil 612 | }() 613 | } 614 | 615 | /* 616 | todo 异步free 清除内存 617 | buckets清除旧的（resize结束，并无get读引用【引入重试来解决该问题】） 618 | 3、Bucket清除 619 | 4、元素清除*/ 620 | func (chm *ConcurrentRawHashMap) freeBuckets(buckets *[]uintptr) error { 621 | for _, ptr := range *buckets { 622 | if ptr < 1 { 623 | continue 624 | } 625 | if err := chm.mm.Free(ptr); err != nil { 626 | log.Printf("freeBuckets Free element(%d) err:%s\n", ptr, err) 627 | } 628 | } 629 | if err := chm.mm.Free(uintptr(unsafe.Pointer(buckets))); err != nil { 630 | return err 631 | } 632 | return nil 633 | } 634 | 635 | func (chm *ConcurrentRawHashMap) increaseTransferIndex(cap uint64, stride uint64) (offset uint64, over bool) { 636 | for { 637 | transferIndex := atomic.LoadUint64(&chm.transferIndex) 638 | if transferIndex >= cap { 639 | return 0, true 640 | } 641 | swapped := atomic.CompareAndSwapUint64(&chm.transferIndex, transferIndex, transferIndex+stride) 642 | if swapped { 643 | return transferIndex, false 644 | } 645 | } 646 | } 647 | 648 | func (chm *ConcurrentRawHashMap) reSizeBuckets(s *Snapshot) error { 649 | newBuckets, oldBuckets, currentCap := s.nextBuckets, s.buckets, uint64(cap(*s.buckets)) 650 | for { 651 | stride := chm.getStride(currentCap) 652 | offset, over := chm.increaseTransferIndex(currentCap, stride) 653 | if over { 654 | return nil 655 | } 656 | maxIndex := offset + stride 657 | if maxIndex > currentCap { 658 | maxIndex = currentCap 659 | } 660 | for index := offset; index < maxIndex; index++ { 661 | var err error 662 | loop := true 663 | for loop { 664 | addr := &((*oldBuckets)[index]) 665 | entries := (*Bucket)(unsafe.Pointer(atomic.LoadUintptr(addr))) 666 | if entries == nil { 667 | forwardingEntries, err := chm.initForwardingEntries(newBuckets, index) 668 | if err != nil { 669 | log.Printf("ERROR reSizeBucket initForwardingEntries err:%s\n", err) 670 | return err 671 | } 672 | ptr := uintptr(unsafe.Pointer(forwardingEntries)) 673 | if atomic.CompareAndSwapUintptr(addr, 0, ptr) { 674 | break 675 | } else { 676 | entries = (*Bucket)(unsafe.Pointer(atomic.LoadUintptr(addr))) 677 | } 678 | } 679 | if loop, err = chm.reSizeBucket(entries, s); err != nil { 680 | log.Printf("ERROR reSizeBucket oldEntries err:%s\n", err) 681 | } 682 | } 683 | } 684 | } 685 | } 686 | 687 | // Chain .链接 688 | type Chain struct { 689 | Head *entry.NodeEntry 690 | Tail *entry.NodeEntry 691 | } 692 | 693 | // Add 添加节点 694 | func (c *Chain) Add(node *entry.NodeEntry) { 695 | if c.Head == nil { 696 | c.Head = node 697 | } 698 | if c.Tail != nil { 699 | c.Tail.Next = node 700 | } 701 | c.Tail = node 702 | } 703 | 704 | // GetHead 获取head节点 705 | func (c *Chain) GetHead() *entry.NodeEntry { 706 | if c.Tail != nil { 707 | c.Tail.Next = nil 708 | } 709 | return c.Head 710 | } 711 | 712 | func (chm *ConcurrentRawHashMap) reSizeBucket(entries *Bucket, s *Snapshot) (loop bool, err error) { 713 | entries.rwLock.Lock() 714 | defer entries.rwLock.Unlock() 715 | oldBuckets, newBuckets := s.buckets, s.nextBuckets 716 | currentCap := uint64(cap(*s.buckets)) 717 | tabPtr, nextTabPtr := unsafe.Pointer(s.buckets), unsafe.Pointer(s.nextBuckets) 718 | _, _, currentEntries := chm.tabAt(oldBuckets, entries.index) 719 | // 判断newBuckets为chm.newBuckets,判断newBuckets为chm.newBuckets 720 | if entries.forwarding || tabPtr != unsafe.Pointer(chm.buckets) || 721 | nextTabPtr != unsafe.Pointer(chm.nextBuckets) || entries != currentEntries { 722 | return false, nil 723 | } 724 | if entries != nil && ((!entries.isTree && entries.Head == nil) || (entries.isTree && entries.Tree == nil)) { 725 | entries.newBuckets = newBuckets 726 | entries.forwarding = true 727 | return false, nil 728 | } 729 | mask := (currentCap << 1) - 1 730 | index := entries.index 731 | var oldIndex, newIndex = index, currentCap + index 732 | oldChains, newChains, err := chm.spliceEntry2(entries, mask) 733 | if err != nil { 734 | return true, err 735 | } 736 | oldBucket, newBucket, err := chm.spliceBucket2(oldChains, newChains) 737 | if err != nil { 738 | return false, err 739 | } 740 | if oldBucket != nil { 741 | oldBucket.index = oldIndex 742 | (*newBuckets)[oldIndex] = uintptr(unsafe.Pointer(oldBucket)) 743 | } 744 | if newBucket != nil { 745 | newBucket.index = newIndex 746 | (*newBuckets)[newIndex] = uintptr(unsafe.Pointer(newBucket)) 747 | } 748 | entries.newBuckets = newBuckets 749 | entries.forwarding = true 750 | entries.Head = nil 751 | return false, nil 752 | } 753 | 754 | // TreeSplice . 755 | func (chm *ConcurrentRawHashMap) TreeSplice(node *entry.NodeEntry, index uint64, mask uint64, oldChains, newChains *Chain) { 756 | if node == nil { 757 | return 758 | } 759 | if node.Hash&mask == index { 760 | oldChains.Add(node) 761 | } else { 762 | newChains.Add(node) 763 | } 764 | chm.TreeSplice(node.Left(), index, mask, oldChains, newChains) 765 | chm.TreeSplice(node.Right(), index, mask, oldChains, newChains) 766 | } 767 | 768 | // spliceEntry2 将Entry拆分为两个列表 769 | func (chm *ConcurrentRawHashMap) spliceEntry2(entries *Bucket, mask uint64) (old *entry.NodeEntry, new *entry.NodeEntry, err error) { 770 | var oldHead, oldTail, newHead, newTail *entry.NodeEntry 771 | index := entries.index 772 | if entries.isTree { 773 | var oldChains, newChains Chain 774 | chm.TreeSplice(entries.Tree.GetRoot(), index, mask, &oldChains, &newChains) 775 | oldHead, newHead = oldChains.GetHead(), newChains.GetHead() 776 | // 删除现在的tree内容 777 | if err := chm.mm.Free(uintptr(unsafe.Pointer(entries.Tree))); err != nil { 778 | return nil, nil, err 779 | } 780 | } else { 781 | for nodeEntry := entries.Head; nodeEntry != nil; nodeEntry = nodeEntry.Next { 782 | idx := nodeEntry.Hash & mask 783 | if idx == index { 784 | if oldHead == nil { 785 | oldHead = nodeEntry 786 | } else if oldTail != nil { 787 | oldTail.Next = nodeEntry 788 | } 789 | oldTail = nodeEntry 790 | } else { 791 | if newHead == nil { 792 | newHead = nodeEntry 793 | } else if newTail != nil { 794 | newTail.Next = nodeEntry 795 | } 796 | newTail = nodeEntry 797 | } 798 | } 799 | } 800 | if newTail != nil { 801 | newTail.Next = nil 802 | } 803 | if oldTail != nil { 804 | oldTail.Next = nil 805 | } 806 | return oldHead, newHead, nil 807 | } 808 | 809 | // 性能更好 810 | func (chm *ConcurrentRawHashMap) spliceBucket2(old *entry.NodeEntry, new *entry.NodeEntry) (oldBucket *Bucket, newBucket *Bucket, err error) { 811 | if old != nil { 812 | head, index := old, uint64(0) 813 | item, err := chm.initEntries(head, index) 814 | if err != nil { 815 | return nil, nil, err 816 | } 817 | oldBucket = item 818 | } 819 | 820 | if new != nil { 821 | head, index := new, uint64(0) 822 | item, err := chm.initEntries(head, index) 823 | if err != nil { 824 | return nil, nil, err 825 | } 826 | newBucket = item 827 | } 828 | return 829 | } 830 | 831 | func (chm *ConcurrentRawHashMap) indexBucket(idx int, tab *[]uintptr) (bucket *Bucket, outIndex bool) { 832 | if idx >= cap(*tab) { 833 | return nil, false 834 | } 835 | _, _, bucket = chm.tabAt(tab, uint64(idx)) 836 | if bucket != nil && bucket.forwarding && chm.transferIndex >= 0 { 837 | return chm.indexBucket(idx, bucket.newBuckets) 838 | } 839 | return bucket, true 840 | } 841 | 842 | func (chm *ConcurrentRawHashMap) ForEach(fun func(key, val []byte) error) error { 843 | bucket, exist := chm.indexBucket(0, chm.buckets) 844 | for i := 1; exist; i++ { 845 | if bucket != nil { 846 | if tree := bucket.Tree; bucket.isTree && tree != nil { 847 | if err := chm.TreeForEach(fun, tree.GetRoot()); err != nil { 848 | return err 849 | } 850 | } else { 851 | for cNode := bucket.Head; cNode != nil; cNode = cNode.Next { 852 | if err := fun(cNode.Key, cNode.Value); err != nil { 853 | return err 854 | } 855 | } 856 | } 857 | } 858 | bucket, exist = chm.indexBucket(i, chm.buckets) 859 | } 860 | return nil 861 | } 862 | 863 | func (chm *ConcurrentRawHashMap) TreeForEach(fun func(key, val []byte) error, node *entry.NodeEntry) error { 864 | if node == nil { 865 | return nil 866 | } 867 | if err := fun(node.Key, node.Value); err != nil { 868 | return err 869 | } 870 | chm.TreeForEach(fun, node.Left()) 871 | chm.TreeForEach(fun, node.Right()) 872 | return nil 873 | } 874 | 875 | // BKDRHashWithSpread . 31 131 1313 13131 131313 etc.. 876 | func BKDRHashWithSpread(str []byte) uint64 { 877 | seed := uint64(131) 878 | hash := uint64(0) 879 | for i := 0; i < len(str); i++ { 880 | hash = (hash * seed) + uint64(str[i]) 881 | } 882 | return hash ^ (hash>>16)&0x7FFFFFFF 883 | } 884 | --------------------------------------------------------------------------------