├── go.mod ├── .travis.yml ├── .gitignore ├── .github └── workflows │ └── go.yml ├── LICENSE ├── README.md ├── agg.go └── agg_test.go /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ldkingvivi/go-aggregate 2 | 3 | go 1.21 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.13.x 5 | - tip 6 | 7 | before_install: 8 | - go get -t -v ./... 9 | 10 | script: 11 | - go test -race -coverprofile=coverage.txt -covermode=atomic 12 | 13 | after_success: 14 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.21 17 | uses: actions/setup-go@v1 18 | with: 19 | go-version: 1.21 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go mod vendor 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: Build 34 | run: go test -v . 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Di 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-aggregate 2 | [![License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT) 3 | [![Actions Status](https://github.com/ldkingvivi/go-aggregate/workflows/Go/badge.svg)](https://github.com/ldkingvivi/go-aggregate/actions) 4 | [![Build Status](https://travis-ci.org/ldkingvivi/go-aggregate.png?branch=master)](https://travis-ci.org/ldkingvivi/go-aggregate) 5 | [![codecov](https://codecov.io/gh/ldkingvivi/go-aggregate/branch/master/graph/badge.svg)](https://codecov.io/gh/ldkingvivi/go-aggregate) 6 | 7 | # What is this 8 | This is the go implementation of the original aggregate from [@horms]( https://github.com/horms) on linux back in 2002, but more generic, you can implement the interface and make it very flexible 9 | 10 | ### Basic Example 11 | 12 | ``` 13 | package main 14 | 15 | import ( 16 | agg "github.com/ldkingvivi/go-aggregate" 17 | "log" 18 | "net" 19 | ) 20 | 21 | func main() { 22 | 23 | // example use NewBasicCidrEntry for basic aggregate 24 | _, aNet, _ := net.ParseCIDR("8.8.8.0/25") 25 | a := agg.NewBasicCidrEntry(aNet) 26 | 27 | _, bNet, _ := net.ParseCIDR("9.9.9.0/25") 28 | b := agg.NewBasicCidrEntry(bNet) 29 | 30 | _, cNet, _ := net.ParseCIDR("8.8.8.128/25") 31 | c := agg.NewBasicCidrEntry(cNet) 32 | 33 | // empty merge func will do the basic merge 34 | result := agg.Aggregate([]agg.CidrEntry{a, b, c}, func(_, _ agg.CidrEntry) {}) 35 | for _, cidr := range result { 36 | log.Printf("%s", cidr.GetNetwork()) 37 | //2020/03/29 22:02:12 8.8.8.0/24 38 | //2020/03/29 22:02:12 9.9.9.0/25 39 | } 40 | } 41 | ``` 42 | 43 | ### Custom Struct Example 44 | 45 | ``` 46 | package main 47 | 48 | import ( 49 | agg "github.com/ldkingvivi/go-aggregate" 50 | "log" 51 | "net" 52 | ) 53 | 54 | type customCidrEntry struct { 55 | ipNet *net.IPNet 56 | count int 57 | note string 58 | } 59 | 60 | func (c *customCidrEntry) GetNetwork() *net.IPNet { 61 | return c.ipNet 62 | } 63 | 64 | func (c *customCidrEntry) SetNetwork(ipNet *net.IPNet) { 65 | c.ipNet = ipNet 66 | } 67 | 68 | func NewCustomCidrEntry(ipNet *net.IPNet, count int, note string) agg.CidrEntry { 69 | return &customCidrEntry{ 70 | ipNet: ipNet, 71 | count: count, 72 | note: note, 73 | } 74 | } 75 | 76 | func main() { 77 | // example use custom interface with client's own merge logic 78 | _, xNet, _ := net.ParseCIDR("8.8.8.128/25") 79 | _, yNet, _ := net.ParseCIDR("8.8.8.0/25") 80 | 81 | x := NewCustomCidrEntry(xNet, 10, "US") 82 | y := NewCustomCidrEntry(yNet, 20, "US") 83 | 84 | // add CIDR's count when merged 85 | result := agg.Aggregate([]agg.CidrEntry{x, y}, func(keep, delete agg.CidrEntry) { 86 | specificKeep, _ := keep.(*customCidrEntry) 87 | specificDelete, _ := delete.(*customCidrEntry) 88 | specificKeep.count += specificDelete.count 89 | }) 90 | 91 | for _, cidr := range result { 92 | custom, ok := cidr.(*customCidrEntry) 93 | if ok { 94 | log.Printf("%s count : %d with note: %s", 95 | custom.GetNetwork(), custom.count, custom.note) 96 | //2020/03/29 22:25:10 8.8.8.0/24 count : 30 with note: US 97 | } 98 | } 99 | } 100 | 101 | ``` 102 | 103 | ### BenchMark with following string 104 | ``` 105 | input := []string{ 106 | "192.0.2.160/29", "192.0.2.176/29", "192.0.2.184/29", "192.0.2.168/32", 107 | "192.0.2.0/29", "192.0.2.8/29", "192.0.2.16/29", "192.0.2.24/29", 108 | "192.0.2.32/29", "192.0.2.40/29", "192.0.2.48/29", "192.0.2.56/29", 109 | "192.0.2.64/29", "192.0.2.72/29", "192.0.2.80/29", "192.0.2.88/29", 110 | "2001:db8::/64", "2001:db8:0:2::/64", "2001:db8:0:3::/64", "2001:db8:0:1::/64", 111 | "192.0.2.128/29", "192.0.2.136/29", "192.0.2.144/29", "192.0.2.152/29", 112 | "192.0.2.192/29", "192.0.2.200/29", "192.0.2.208/29", "192.0.2.216/29", 113 | "192.0.2.224/29", "192.0.2.232/29", "192.0.2.240/29", "192.0.2.248/29", 114 | "2001:db8:0:4::/64", "192.0.2.171/32", "192.0.2.172/32", "192.0.2.174/32", 115 | "192.0.2.169/32", "192.0.2.170/32", "192.0.2.173/32", "192.0.2.175/32", 116 | "192.0.2.96/29", "192.0.2.104/29", "192.0.2.112/29", "192.0.2.120/29", 117 | } 118 | ``` 119 | 120 | ``` 121 | goos: darwin 122 | goarch: amd64 123 | pkg: github.com/ldkingvivi/go-aggregate 124 | BenchmarkAggregateMergeAddCount-12 65290 17640 ns/op 18056 B/op 204 allocs/op 125 | BenchmarkAggregateMergeUseDeletNote-12 66913 17600 ns/op 18056 B/op 204 allocs/op 126 | BenchmarkAggregateMergeDoNothing-12 67716 17702 ns/op 18056 B/op 204 allocs/op 127 | ``` -------------------------------------------------------------------------------- /agg.go: -------------------------------------------------------------------------------- 1 | package Agg 2 | 3 | import ( 4 | "math/big" 5 | "net" 6 | "net/netip" 7 | "sort" 8 | ) 9 | 10 | type cidr struct { 11 | netIP netip.Addr 12 | startIP *big.Int 13 | nextStartIP *big.Int // this is end IP + 1 14 | ones int 15 | bits int 16 | 17 | prev *cidr 18 | next *cidr 19 | 20 | entry CidrEntry 21 | } 22 | 23 | type CidrEntry interface { 24 | GetNetwork() netip.Prefix 25 | SetNetwork(netip.Prefix) 26 | } 27 | 28 | type basicCidrEntry struct { 29 | ipNet netip.Prefix 30 | } 31 | 32 | func (b *basicCidrEntry) GetNetwork() netip.Prefix { 33 | return b.ipNet 34 | } 35 | 36 | func (b *basicCidrEntry) SetNetwork(ipNet netip.Prefix) { 37 | b.ipNet = ipNet 38 | } 39 | 40 | func NewBasicCidrEntry(ipNet netip.Prefix) CidrEntry { 41 | return &basicCidrEntry{ 42 | ipNet: ipNet, 43 | } 44 | } 45 | 46 | type Merge func(keep, delete CidrEntry) 47 | 48 | func Aggregate(cidrEntries []CidrEntry, mergeFn Merge) []CidrEntry { 49 | if len(cidrEntries) < 2 { 50 | return cidrEntries 51 | } 52 | cidrs := convertToCidr(cidrEntries) 53 | // sort it 54 | sortIt(cidrs) 55 | // add pointer 56 | addPointer(cidrs) 57 | // unlink the smaller ones that already in bigger ones 58 | unlinkCovered(cidrs, mergeFn) 59 | // do the aggregate 60 | aggregateAdj(cidrs, mergeFn) 61 | 62 | return getEntries(cidrs) 63 | } 64 | 65 | func convertToCidr(cidrEntries []CidrEntry) []cidr { 66 | var cidrs []cidr 67 | bigOne := big.NewInt(1) 68 | var ipnet netip.Prefix 69 | // convert 70 | for _, cidrEntry := range cidrEntries { 71 | ipnet = cidrEntry.GetNetwork().Masked() 72 | 73 | // cover IPv6 74 | startIP := big.NewInt(0) 75 | startIP.SetBytes(ipnet.Addr().AsSlice()) 76 | 77 | nextStartIP := big.NewInt(0) 78 | 79 | ones := ipnet.Bits() 80 | bits := ipnet.Addr().BitLen() 81 | diff := uint(bits) - uint(ones) 82 | 83 | nextStartIP.Lsh(bigOne, diff) 84 | nextStartIP.Add(nextStartIP, startIP) 85 | 86 | cidrs = append(cidrs, cidr{ 87 | netIP: ipnet.Addr(), 88 | startIP: startIP, 89 | nextStartIP: nextStartIP, 90 | ones: ones, 91 | bits: bits, 92 | entry: cidrEntry, 93 | }) 94 | } 95 | 96 | return cidrs 97 | } 98 | 99 | func sortIt(cidrs []cidr) { 100 | sort.Slice(cidrs, func(i, j int) bool { 101 | startIPCmp := cidrs[i].startIP.Cmp(cidrs[j].startIP) 102 | if startIPCmp < 0 { 103 | return true 104 | } else if startIPCmp == 0 && cidrs[i].ones < cidrs[j].ones { 105 | return true 106 | } 107 | return false 108 | }) 109 | } 110 | 111 | func addPointer(cidrs []cidr) { 112 | s := 0 113 | e := 1 114 | for e < len(cidrs) { 115 | cidrs[s].next = &cidrs[e] 116 | cidrs[e].prev = &cidrs[s] 117 | s++ 118 | e++ 119 | } 120 | } 121 | 122 | func unlinkCovered(cidrs []cidr, mergeFn Merge) { 123 | // check already done from Aggregate() 124 | currentP := &cidrs[0] 125 | nextP := currentP.next 126 | 127 | for nextP != nil { 128 | if currentP.nextStartIP.Cmp(nextP.nextStartIP) >= 0 { 129 | // run the merge func 130 | mergeFn(currentP.entry, nextP.entry) 131 | // skip the next 132 | currentP.next = nextP.next 133 | if nextP.next != nil { 134 | nextP.next.prev = currentP 135 | } 136 | } else { 137 | // only move current forward if current endIP not cover next endIP 138 | currentP = nextP 139 | } 140 | nextP = currentP.next 141 | } 142 | } 143 | 144 | func aggregateAdj(cidrs []cidr, mergeFn Merge) { 145 | // check already done from Aggregate() 146 | currentP := &cidrs[0] 147 | nextP := currentP.next 148 | 149 | for nextP != nil { 150 | 151 | if currentP.ones == nextP.ones && 152 | currentP.nextStartIP.Cmp(nextP.startIP) == 0 && 153 | getIPPrefix(currentP.netIP) < currentP.ones { 154 | // change current endIP and prefix 155 | // no need to change the netIP 156 | currentP.nextStartIP = nextP.nextStartIP 157 | currentP.ones = currentP.ones - 1 158 | // run the merge func 159 | mergeFn(currentP.entry, nextP.entry) 160 | 161 | // redo the link 162 | currentP.next = nextP.next 163 | if nextP.next != nil { 164 | nextP.next.prev = currentP 165 | } 166 | 167 | // try to move up if possible 168 | if currentP.prev != nil { 169 | nextP = currentP 170 | currentP = currentP.prev 171 | } else { 172 | nextP = nextP.next 173 | } 174 | continue 175 | } 176 | 177 | // move forward 178 | currentP = nextP 179 | nextP = currentP.next 180 | } 181 | } 182 | 183 | func getEntries(cidrs []cidr) []CidrEntry { 184 | var r []CidrEntry 185 | currentP := &cidrs[0] 186 | for currentP != nil { 187 | // update the entry network 188 | prefix := netip.PrefixFrom(currentP.netIP, currentP.ones) 189 | currentP.entry.SetNetwork(prefix) 190 | // added to results 191 | r = append(r, currentP.entry) 192 | // move to next 193 | currentP = currentP.next 194 | } 195 | return r 196 | } 197 | 198 | func getIPPrefix(ip netip.Addr) int { 199 | 200 | if ip.Is4() { 201 | // ipv4 202 | return net.IPv4len*8 - getTrailingZeroV4(ip.As4()) 203 | } else { 204 | // ipv6 205 | return net.IPv6len*8 - getTrailingZeroV6(ip.As16()) 206 | } 207 | } 208 | 209 | func getTrailingZeroV4(ip [4]byte) int { 210 | var n int 211 | var v byte 212 | 213 | i := len(ip) - 1 214 | for i >= 0 { 215 | v = ip[i] 216 | if v == 0x00 { 217 | n += 8 218 | i-- 219 | continue 220 | } 221 | // found non-00 byte 222 | // count 0 bits 223 | for v&0x01 != 1 { 224 | n++ 225 | v >>= 1 226 | } 227 | break 228 | } 229 | return n 230 | } 231 | 232 | func getTrailingZeroV6(ip [16]byte) int { 233 | var n int 234 | var v byte 235 | 236 | i := len(ip) - 1 237 | for i >= 0 { 238 | v = ip[i] 239 | if v == 0x00 { 240 | n += 8 241 | i-- 242 | continue 243 | } 244 | // found non-00 byte 245 | // count 0 bits 246 | for v&0x01 != 1 { 247 | n++ 248 | v >>= 1 249 | } 250 | break 251 | } 252 | return n 253 | } 254 | -------------------------------------------------------------------------------- /agg_test.go: -------------------------------------------------------------------------------- 1 | package Agg 2 | 3 | import ( 4 | "net/netip" 5 | "reflect" 6 | "strconv" 7 | "testing" 8 | ) 9 | 10 | func TestGetIPPrefix(t *testing.T) { 11 | ip1 := netip.MustParseAddr("0.0.0.0") 12 | 13 | got1 := getIPPrefix(ip1) 14 | if got1 != 0 { 15 | t.Errorf("expect 0 but got %+v", got1) 16 | } 17 | 18 | ip2 := netip.MustParseAddr("8.8.8.0") 19 | got2 := getIPPrefix(ip2) 20 | if got2 != 21 { 21 | t.Errorf("expect 21 but got %+v", got2) 22 | } 23 | 24 | ip3 := netip.MustParseAddr("8.8.8.8") 25 | got3 := getIPPrefix(ip3) 26 | if got3 != 29 { 27 | t.Errorf("expect 29 but got %+v", got3) 28 | } 29 | 30 | ip4 := netip.MustParseAddr("2620:108:700f::3645:f643") 31 | got4 := getIPPrefix(ip4) 32 | if got4 != 128 { 33 | t.Errorf("expect 128 but got %+v", got4) 34 | } 35 | 36 | ipnet4 := netip.MustParsePrefix("2620:108:700f::3645:f643/64") 37 | got5 := getIPPrefix(ipnet4.Masked().Addr()) 38 | if got5 != 48 { 39 | t.Errorf("expect 48 but got %+v", got5) 40 | } 41 | } 42 | 43 | type testResults struct { 44 | ipnetString string 45 | count int 46 | } 47 | 48 | type customCidrEntry struct { 49 | ipNet netip.Prefix 50 | count int 51 | note string 52 | } 53 | 54 | func (c *customCidrEntry) GetNetwork() netip.Prefix { 55 | return c.ipNet 56 | } 57 | 58 | func (c *customCidrEntry) SetNetwork(ipNet netip.Prefix) { 59 | c.ipNet = ipNet 60 | } 61 | 62 | func (c *customCidrEntry) GetCount() int { 63 | return c.count 64 | } 65 | 66 | func (c *customCidrEntry) SetCount(count int) { 67 | c.count = count 68 | } 69 | 70 | func NewCustomCidrEntry(ipNet netip.Prefix, count int, note string) CidrEntry { 71 | return &customCidrEntry{ 72 | ipNet: ipNet, 73 | count: count, 74 | note: note, 75 | } 76 | } 77 | 78 | func mergeAddCount(k, d CidrEntry) { 79 | sk, _ := k.(*customCidrEntry) 80 | sd, _ := d.(*customCidrEntry) 81 | 82 | sk.count += sd.count 83 | } 84 | 85 | func mergeUseDeleteNote(k, d CidrEntry) { 86 | 87 | sk, _ := k.(*customCidrEntry) 88 | sd, _ := d.(*customCidrEntry) 89 | 90 | sk.note = sd.note 91 | } 92 | 93 | func mergeDoNothing(_, _ CidrEntry) { 94 | } 95 | 96 | func TestAggregateAddCount(t *testing.T) { 97 | 98 | var got []CidrEntry 99 | 100 | for i, c := range []struct { 101 | in []string 102 | want []testResults 103 | }{ 104 | // Empty 105 | { 106 | []string{}, 107 | []testResults{}, 108 | }, 109 | // Single 110 | { 111 | []string{"8.8.8.0/24"}, 112 | []testResults{ 113 | { 114 | "8.8.8.0/24", 115 | 1, 116 | }, 117 | }, 118 | }, 119 | // IPv4 prefixes 120 | { 121 | []string{ 122 | "8.8.8.8/29", "8.8.8.0/24", 123 | }, 124 | []testResults{ 125 | { 126 | "8.8.8.0/24", 127 | 2, 128 | }, 129 | }, 130 | }, 131 | { 132 | []string{ 133 | "8.8.8.8/29", "8.8.8.0/29", 134 | }, 135 | []testResults{ 136 | { 137 | "8.8.8.0/28", 138 | 2, 139 | }, 140 | }, 141 | }, 142 | { 143 | []string{ 144 | "8.8.8.8/29", "8.8.8.16/29", 145 | }, 146 | []testResults{ 147 | { 148 | "8.8.8.8/29", 149 | 1, 150 | }, 151 | { 152 | "8.8.8.16/29", 153 | 1, 154 | }, 155 | }, 156 | }, 157 | { 158 | []string{ 159 | "8.8.8.0/25", "9.9.9.0/25", "8.8.8.128/25", 160 | }, 161 | []testResults{ 162 | { 163 | "8.8.8.0/24", 164 | 2, 165 | }, 166 | { 167 | "9.9.9.0/25", 168 | 1, 169 | }, 170 | }, 171 | }, 172 | { 173 | []string{ 174 | "192.0.2.0/25", "192.0.2.128/25", 175 | }, 176 | []testResults{ 177 | { 178 | "192.0.2.0/24", 179 | 2, 180 | }, 181 | }, 182 | }, 183 | { 184 | []string{ 185 | "192.0.2.0/26", "192.0.2.64/26", "192.0.2.128/26", "192.0.2.192/26", 186 | }, 187 | []testResults{ 188 | { 189 | "192.0.2.0/24", 190 | 4, 191 | }, 192 | }, 193 | }, 194 | { 195 | []string{ 196 | "192.0.2.0/27", "192.0.2.32/27", "192.0.2.64/27", "192.0.2.96/27", 197 | "192.0.2.128/27", "192.0.2.160/27", "192.0.2.192/27", "192.0.2.224/27", 198 | }, 199 | []testResults{ 200 | { 201 | "192.0.2.0/24", 202 | 8, 203 | }, 204 | }, 205 | }, 206 | { 207 | []string{ 208 | "192.0.2.0/28", "192.0.2.16/28", "192.0.2.32/28", "192.0.2.48/28", 209 | "192.0.2.64/28", "192.0.2.80/28", "192.0.2.96/28", "192.0.2.112/28", 210 | "192.0.2.128/28", "192.0.2.144/28", "192.0.2.160/28", "192.0.2.176/28", 211 | "192.0.2.192/28", "192.0.2.208/28", "192.0.2.224/28", "192.0.2.240/28", 212 | }, 213 | []testResults{ 214 | { 215 | "192.0.2.0/24", 216 | 16, 217 | }, 218 | }, 219 | }, 220 | { 221 | []string{ 222 | "192.0.2.0/29", "192.0.2.8/29", "192.0.2.16/29", "192.0.2.24/29", 223 | "192.0.2.32/29", "192.0.2.40/29", "192.0.2.48/29", "192.0.2.56/29", 224 | "192.0.2.64/29", "192.0.2.72/29", "192.0.2.80/29", "192.0.2.88/29", 225 | "192.0.2.96/29", "192.0.2.104/29", "192.0.2.112/29", "192.0.2.120/29", 226 | "192.0.2.128/29", "192.0.2.136/29", "192.0.2.144/29", "192.0.2.152/29", 227 | "192.0.2.160/29", "192.0.2.168/29", "192.0.2.176/29", "192.0.2.184/29", 228 | "192.0.2.192/29", "192.0.2.200/29", "192.0.2.208/29", "192.0.2.216/29", 229 | "192.0.2.224/29", "192.0.2.232/29", "192.0.2.240/29", "192.0.2.248/29", 230 | }, 231 | []testResults{ 232 | { 233 | "192.0.2.0/24", 234 | 32, 235 | }, 236 | }, 237 | }, 238 | { 239 | []string{ 240 | "192.0.2.0/26", "192.0.2.64/26", "192.0.2.192/26", 241 | "192.0.2.128/28", "192.0.2.144/28", "192.0.2.160/28", "192.0.2.176/28", 242 | }, 243 | []testResults{ 244 | { 245 | "192.0.2.0/24", 246 | 7, 247 | }, 248 | }, 249 | }, 250 | { 251 | []string{ 252 | "192.0.2.1/32", "192.0.2.1/32", 253 | }, 254 | []testResults{ 255 | { 256 | "192.0.2.1/32", 257 | 2, 258 | }, 259 | }, 260 | }, 261 | { 262 | []string{ 263 | "192.0.2.0/25", "192.0.2.128/25", 264 | "192.0.2.248/29", 265 | }, 266 | []testResults{ 267 | { 268 | "192.0.2.0/24", 269 | 3, 270 | }, 271 | }, 272 | }, 273 | { 274 | []string{ 275 | "192.0.2.0/24", 276 | "198.51.100.0/24", 277 | "203.0.113.0/24", 278 | }, 279 | []testResults{ 280 | { 281 | "192.0.2.0/24", 282 | 1, 283 | }, 284 | { 285 | "198.51.100.0/24", 286 | 1, 287 | }, 288 | { 289 | "203.0.113.0/24", 290 | 1, 291 | }, 292 | }, 293 | }, 294 | { 295 | []string{ 296 | "192.0.2.0/25", 297 | "192.0.2.0/26", 298 | "192.0.2.0/27", 299 | "192.0.2.0/28", 300 | "192.0.2.0/29", 301 | "192.0.2.0/30", 302 | }, 303 | []testResults{ 304 | { 305 | "192.0.2.0/25", 306 | 6, 307 | }, 308 | }, 309 | }, 310 | { 311 | []string{ 312 | "0.0.0.0/0", 313 | "192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24", 314 | "255.255.255.255/32", 315 | }, 316 | []testResults{ 317 | { 318 | "0.0.0.0/0", 319 | 5, 320 | }, 321 | }, 322 | }, 323 | { 324 | []string{ 325 | "0.0.0.0/0", "0.0.0.0/0", 326 | "255.255.255.255/32", "255.255.255.255/32", 327 | }, 328 | []testResults{ 329 | { 330 | "0.0.0.0/0", 331 | 4, 332 | }, 333 | }, 334 | }, 335 | { 336 | []string{ 337 | "192.168.0.0/25", "192.168.0.128/25", 338 | "192.168.1.0/24", "192.168.3.0/24", "192.168.4.0/24", 339 | "192.168.5.0/26", 340 | "192.168.128.0/22", "192.168.132.0/22", 341 | "192.168.128.0/21", 342 | }, 343 | []testResults{ 344 | { 345 | "192.168.0.0/23", 346 | 3, 347 | }, 348 | { 349 | "192.168.3.0/24", 350 | 1, 351 | }, 352 | { 353 | "192.168.4.0/24", 354 | 1, 355 | }, 356 | { 357 | "192.168.5.0/26", 358 | 1, 359 | }, 360 | { 361 | "192.168.128.0/21", 362 | 3, 363 | }, 364 | }, 365 | }, 366 | { 367 | []string{ 368 | "192.168.0.0/25", "192.168.0.128/25", 369 | "192.168.1.0/24", "192.168.3.0/24", "192.168.4.0/24", 370 | "192.168.5.0/26", 371 | }, 372 | []testResults{ 373 | { 374 | "192.168.0.0/23", 375 | 3, 376 | }, 377 | { 378 | "192.168.3.0/24", 379 | 1, 380 | }, 381 | { 382 | "192.168.4.0/24", 383 | 1, 384 | }, 385 | { 386 | "192.168.5.0/26", 387 | 1, 388 | }, 389 | }, 390 | }, 391 | { 392 | []string{ 393 | "192.0.2.0/25", "198.51.100.0/25", "192.0.2.128/25", 394 | }, 395 | []testResults{ 396 | { 397 | "192.0.2.0/24", 398 | 2, 399 | }, 400 | { 401 | "198.51.100.0/25", 402 | 1, 403 | }, 404 | }, 405 | }, 406 | // IPv6 prefixes 407 | { 408 | []string{ 409 | "2001:db8::/64", "2001:db8:0:1::/64", "2001:db8:0:2::/64", "2001:db8:0:3::/64", 410 | "2001:db8:0:4::/64", 411 | }, 412 | []testResults{ 413 | { 414 | "2001:db8::/62", 415 | 4, 416 | }, 417 | { 418 | "2001:db8:0:4::/64", 419 | 1, 420 | }, 421 | }, 422 | }, 423 | { 424 | []string{ 425 | "::/0", 426 | "2001:db8::/32", 427 | "2001:db8::/126", 428 | "2001:db8::/127", 429 | "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128", 430 | }, 431 | []testResults{ 432 | { 433 | "::/0", 434 | 5, 435 | }, 436 | }, 437 | }, 438 | { 439 | []string{ 440 | "::/0", "::/0", 441 | "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128", 442 | }, 443 | []testResults{ 444 | { 445 | "::/0", 446 | 4, 447 | }, 448 | }, 449 | }, 450 | // Mix IPv4 and IPv6 451 | { 452 | []string{ 453 | "192.0.2.0/29", "192.0.2.8/29", "192.0.2.16/29", "192.0.2.24/29", 454 | "192.0.2.32/29", "192.0.2.40/29", "192.0.2.48/29", "192.0.2.56/29", 455 | "192.0.2.64/29", "192.0.2.72/29", "192.0.2.80/29", "192.0.2.88/29", 456 | "192.0.2.96/29", "192.0.2.104/29", "192.0.2.112/29", "192.0.2.120/29", 457 | "2001:db8::/64", "2001:db8:0:2::/64", "2001:db8:0:3::/64", "2001:db8:0:1::/64", 458 | "192.0.2.128/29", "192.0.2.136/29", "192.0.2.144/29", "192.0.2.152/29", 459 | "192.0.2.160/29", "192.0.2.176/29", "192.0.2.184/29", "192.0.2.168/32", 460 | "192.0.2.192/29", "192.0.2.200/29", "192.0.2.208/29", "192.0.2.216/29", 461 | "192.0.2.224/29", "192.0.2.232/29", "192.0.2.240/29", "192.0.2.248/29", 462 | "2001:db8:0:4::/64", "192.0.2.171/32", "192.0.2.172/32", "192.0.2.174/32", 463 | "192.0.2.169/32", "192.0.2.170/32", "192.0.2.173/32", "192.0.2.175/32", 464 | }, 465 | []testResults{ 466 | { 467 | "192.0.2.0/24", 468 | 39, 469 | }, 470 | { 471 | "2001:db8::/62", 472 | 4, 473 | }, 474 | { 475 | "2001:db8:0:4::/64", 476 | 1, 477 | }, 478 | }, 479 | }, 480 | } { 481 | 482 | var cidrEntries []CidrEntry 483 | var cidrWant []CidrEntry 484 | 485 | for _, s := range c.in { 486 | ipnet := netip.MustParsePrefix(s) 487 | cidrEntries = append(cidrEntries, NewCustomCidrEntry(ipnet, 1, "US")) 488 | } 489 | 490 | for _, s := range c.want { 491 | ipnet := netip.MustParsePrefix(s.ipnetString) 492 | cidrWant = append(cidrWant, NewCustomCidrEntry(ipnet, s.count, "US")) 493 | } 494 | 495 | got = Aggregate(cidrEntries, mergeAddCount) 496 | 497 | if !reflect.DeepEqual(got, cidrWant) { 498 | t.Errorf("#%d: expect: %+v , but got %+v", i, cidrWant, got) 499 | } 500 | } 501 | } 502 | 503 | func TestAggregateWithGivenCount(t *testing.T) { 504 | 505 | var input = []testResults{ 506 | { 507 | "8.8.9.128/25", 508 | 1, 509 | }, 510 | { 511 | "8.8.8.0/24", 512 | 39, 513 | }, 514 | { 515 | "8.8.9.0/25", 516 | 4, 517 | }, 518 | } 519 | 520 | var want = []testResults{ 521 | { 522 | "8.8.8.0/23", 523 | 44, 524 | }, 525 | } 526 | 527 | var inputCidrs []CidrEntry 528 | for _, s := range input { 529 | ipnet := netip.MustParsePrefix(s.ipnetString) 530 | inputCidrs = append(inputCidrs, NewCustomCidrEntry(ipnet, s.count, "US")) 531 | } 532 | 533 | got := Aggregate(inputCidrs, mergeAddCount) 534 | 535 | var cidrWant []CidrEntry 536 | for _, s := range want { 537 | ipnet := netip.MustParsePrefix(s.ipnetString) 538 | cidrWant = append(cidrWant, NewCustomCidrEntry(ipnet, s.count, "US")) 539 | } 540 | 541 | if !reflect.DeepEqual(got, cidrWant) { 542 | t.Errorf("expect: %+v , but got %+v", cidrWant, got) 543 | } 544 | } 545 | 546 | func TestAggregateWithMergeDeleteNote(t *testing.T) { 547 | 548 | // example use custom interface 549 | xNet := netip.MustParsePrefix("8.8.8.128/25") 550 | yNet := netip.MustParsePrefix("8.8.8.0/25") 551 | 552 | x := NewCustomCidrEntry(xNet, 10, "US") 553 | y := NewCustomCidrEntry(yNet, 20, "CA") 554 | 555 | got := Aggregate([]CidrEntry{x, y}, mergeUseDeleteNote) 556 | 557 | if len(got) != 1 { 558 | t.Errorf("expect single results") 559 | } 560 | 561 | gotS, ok := got[0].(*customCidrEntry) 562 | if !ok { 563 | t.Errorf("error to map type back") 564 | } 565 | 566 | expect := "US" 567 | 568 | if gotS.note != expect { 569 | t.Errorf("expect %s, but got %s", expect, gotS.note) 570 | } 571 | } 572 | 573 | func TestAggregateWithMergeDoNothing(t *testing.T) { 574 | 575 | var input = []string{ 576 | "8.8.9.128/25", 577 | "8.8.8.0/24", 578 | "8.8.9.0/25", 579 | } 580 | 581 | var want = []string{ 582 | "8.8.8.0/23", 583 | } 584 | 585 | var inputCidrs []CidrEntry 586 | for _, s := range input { 587 | ipnet := netip.MustParsePrefix(s) 588 | inputCidrs = append(inputCidrs, NewBasicCidrEntry(ipnet)) 589 | } 590 | 591 | got := Aggregate(inputCidrs, mergeDoNothing) 592 | 593 | var cidrWant []CidrEntry 594 | for _, s := range want { 595 | ipnet := netip.MustParsePrefix(s) 596 | cidrWant = append(cidrWant, NewBasicCidrEntry(ipnet)) 597 | } 598 | 599 | if !reflect.DeepEqual(got, cidrWant) { 600 | t.Errorf("expect: %+v , but got %+v", cidrWant, got) 601 | } 602 | } 603 | 604 | func TestAggregateWithMergeDoNothing65K(t *testing.T) { 605 | var inputCidrs []CidrEntry 606 | 607 | var c, d int 608 | var cStr, dStr string 609 | 610 | for c = 0; c < 256; c++ { 611 | cStr = strconv.Itoa(c) 612 | for d = 0; d < 256; d++ { 613 | dStr = strconv.Itoa(d) 614 | ipnet := netip.MustParsePrefix("1.1." + cStr + "." + dStr + "/32") 615 | inputCidrs = append(inputCidrs, NewBasicCidrEntry(ipnet)) 616 | } 617 | } 618 | 619 | got := Aggregate(inputCidrs, mergeDoNothing) 620 | 621 | var cidrWant []CidrEntry 622 | ipnet := netip.MustParsePrefix("1.1.0.0/16") 623 | cidrWant = append(cidrWant, NewBasicCidrEntry(ipnet)) 624 | 625 | if !reflect.DeepEqual(got, cidrWant) { 626 | t.Errorf("expect: %+v , but got %+v", cidrWant, got) 627 | } 628 | 629 | } 630 | 631 | func BenchmarkAggregateMergeAddCount(b *testing.B) { 632 | input := []string{ 633 | "192.0.2.160/29", "192.0.2.176/29", "192.0.2.184/29", "192.0.2.168/32", 634 | "192.0.2.0/29", "192.0.2.8/29", "192.0.2.16/29", "192.0.2.24/29", 635 | "192.0.2.32/29", "192.0.2.40/29", "192.0.2.48/29", "192.0.2.56/29", 636 | "192.0.2.64/29", "192.0.2.72/29", "192.0.2.80/29", "192.0.2.88/29", 637 | "2001:db8::/64", "2001:db8:0:2::/64", "2001:db8:0:3::/64", "2001:db8:0:1::/64", 638 | "192.0.2.128/29", "192.0.2.136/29", "192.0.2.144/29", "192.0.2.152/29", 639 | "192.0.2.192/29", "192.0.2.200/29", "192.0.2.208/29", "192.0.2.216/29", 640 | "192.0.2.224/29", "192.0.2.232/29", "192.0.2.240/29", "192.0.2.248/29", 641 | "2001:db8:0:4::/64", "192.0.2.171/32", "192.0.2.172/32", "192.0.2.174/32", 642 | "192.0.2.169/32", "192.0.2.170/32", "192.0.2.173/32", "192.0.2.175/32", 643 | "192.0.2.96/29", "192.0.2.104/29", "192.0.2.112/29", "192.0.2.120/29", 644 | } 645 | 646 | var cidrEntries []CidrEntry 647 | for _, s := range input { 648 | ipnet := netip.MustParsePrefix(s) 649 | cidrEntries = append(cidrEntries, NewCustomCidrEntry(ipnet, 1, "US")) 650 | } 651 | 652 | b.ResetTimer() 653 | for i := 0; i < b.N; i++ { 654 | _ = Aggregate(cidrEntries, mergeAddCount) 655 | } 656 | } 657 | 658 | func BenchmarkAggregateMergeUseDeletNote(b *testing.B) { 659 | input := []string{ 660 | "192.0.2.160/29", "192.0.2.176/29", "192.0.2.184/29", "192.0.2.168/32", 661 | "192.0.2.0/29", "192.0.2.8/29", "192.0.2.16/29", "192.0.2.24/29", 662 | "192.0.2.32/29", "192.0.2.40/29", "192.0.2.48/29", "192.0.2.56/29", 663 | "192.0.2.64/29", "192.0.2.72/29", "192.0.2.80/29", "192.0.2.88/29", 664 | "2001:db8::/64", "2001:db8:0:2::/64", "2001:db8:0:3::/64", "2001:db8:0:1::/64", 665 | "192.0.2.128/29", "192.0.2.136/29", "192.0.2.144/29", "192.0.2.152/29", 666 | "192.0.2.192/29", "192.0.2.200/29", "192.0.2.208/29", "192.0.2.216/29", 667 | "192.0.2.224/29", "192.0.2.232/29", "192.0.2.240/29", "192.0.2.248/29", 668 | "2001:db8:0:4::/64", "192.0.2.171/32", "192.0.2.172/32", "192.0.2.174/32", 669 | "192.0.2.169/32", "192.0.2.170/32", "192.0.2.173/32", "192.0.2.175/32", 670 | "192.0.2.96/29", "192.0.2.104/29", "192.0.2.112/29", "192.0.2.120/29", 671 | } 672 | 673 | var cidrEntries []CidrEntry 674 | for _, s := range input { 675 | ipnet := netip.MustParsePrefix(s) 676 | cidrEntries = append(cidrEntries, NewCustomCidrEntry(ipnet, 10, "US")) 677 | } 678 | 679 | b.ResetTimer() 680 | for i := 0; i < b.N; i++ { 681 | _ = Aggregate(cidrEntries, mergeUseDeleteNote) 682 | } 683 | } 684 | 685 | func BenchmarkAggregateMergeDoNothing(b *testing.B) { 686 | input := []string{ 687 | "192.0.2.160/29", "192.0.2.176/29", "192.0.2.184/29", "192.0.2.168/32", 688 | "192.0.2.0/29", "192.0.2.8/29", "192.0.2.16/29", "192.0.2.24/29", 689 | "192.0.2.32/29", "192.0.2.40/29", "192.0.2.48/29", "192.0.2.56/29", 690 | "192.0.2.64/29", "192.0.2.72/29", "192.0.2.80/29", "192.0.2.88/29", 691 | "2001:db8::/64", "2001:db8:0:2::/64", "2001:db8:0:3::/64", "2001:db8:0:1::/64", 692 | "192.0.2.128/29", "192.0.2.136/29", "192.0.2.144/29", "192.0.2.152/29", 693 | "192.0.2.192/29", "192.0.2.200/29", "192.0.2.208/29", "192.0.2.216/29", 694 | "192.0.2.224/29", "192.0.2.232/29", "192.0.2.240/29", "192.0.2.248/29", 695 | "2001:db8:0:4::/64", "192.0.2.171/32", "192.0.2.172/32", "192.0.2.174/32", 696 | "192.0.2.169/32", "192.0.2.170/32", "192.0.2.173/32", "192.0.2.175/32", 697 | "192.0.2.96/29", "192.0.2.104/29", "192.0.2.112/29", "192.0.2.120/29", 698 | } 699 | 700 | var cidrEntries []CidrEntry 701 | for _, s := range input { 702 | ipnet := netip.MustParsePrefix(s) 703 | cidrEntries = append(cidrEntries, NewBasicCidrEntry(ipnet)) 704 | } 705 | 706 | b.ResetTimer() 707 | for i := 0; i < b.N; i++ { 708 | _ = Aggregate(cidrEntries, mergeDoNothing) 709 | } 710 | } 711 | 712 | func BenchmarkAggregateMergeDoNothing16M(b *testing.B) { 713 | var cidrEntries []CidrEntry 714 | 715 | var x, c, d int 716 | var bStr, cStr, dStr string 717 | 718 | for x = 0; x < 256; x++ { 719 | bStr = strconv.Itoa(x) 720 | for c = 0; c < 256; c++ { 721 | cStr = strconv.Itoa(c) 722 | for d = 0; d < 256; d++ { 723 | dStr = strconv.Itoa(d) 724 | ipnet := netip.MustParsePrefix("1." + bStr + "." + cStr + "." + dStr + "/32") 725 | cidrEntries = append(cidrEntries, NewBasicCidrEntry(ipnet)) 726 | } 727 | } 728 | } 729 | 730 | b.ResetTimer() 731 | for i := 0; i < b.N; i++ { 732 | _ = Aggregate(cidrEntries, mergeDoNothing) 733 | } 734 | } 735 | --------------------------------------------------------------------------------