├── .gitignore ├── LICENSE ├── README.md ├── README_CN.md ├── bencode.go ├── bencode_test.go ├── bitmap.go ├── bitmap_test.go ├── bitmap_xor.go ├── bitmap_xorfast.go ├── blacklist.go ├── blacklist_test.go ├── container.go ├── container_test.go ├── dht.go ├── doc ├── .DS_Store └── screen-shot.png ├── go.mod ├── krpc.go ├── peerwire.go ├── routingtable.go ├── sample ├── getpeers │ └── getpeers.go └── spider │ ├── spider │ └── spider.go ├── util.go └── util_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Dean Karn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://raw.githubusercontent.com/shiyanhui/dht/master/doc/screen-shot.png) 2 | 3 | See the video on the [Youtube](https://www.youtube.com/watch?v=AIpeQtw22kc). 4 | 5 | [中文版README](https://github.com/shiyanhui/dht/blob/master/README_CN.md) 6 | 7 | ## Introduction 8 | 9 | DHT implements the bittorrent DHT protocol in Go. Now it includes: 10 | 11 | - [BEP-3 (part)](http://www.bittorrent.org/beps/bep_0003.html) 12 | - [BEP-5](http://www.bittorrent.org/beps/bep_0005.html) 13 | - [BEP-9](http://www.bittorrent.org/beps/bep_0009.html) 14 | - [BEP-10](http://www.bittorrent.org/beps/bep_0010.html) 15 | 16 | It contains two modes, the standard mode and the crawling mode. The standard 17 | mode follows the BEPs, and you can use it as a standard dht server. The crawling 18 | mode aims to crawl as more metadata info as possiple. It doesn't follow the 19 | standard BEPs protocol. With the crawling mode, you can build another [BTDigg](http://btdigg.org/). 20 | 21 | [bthub.io](http://bthub.io) is a BT search engine based on the crawling mode. 22 | 23 | ## Installation 24 | 25 | go get github.com/shiyanhui/dht 26 | 27 | ## Example 28 | 29 | Below is a simple spider. You can move [here](https://github.com/shiyanhui/dht/blob/master/sample) 30 | to see more samples. 31 | 32 | ```go 33 | import ( 34 | "fmt" 35 | "github.com/shiyanhui/dht" 36 | ) 37 | 38 | func main() { 39 | downloader := dht.NewWire(65535) 40 | go func() { 41 | // once we got the request result 42 | for resp := range downloader.Response() { 43 | fmt.Println(resp.InfoHash, resp.MetadataInfo) 44 | } 45 | }() 46 | go downloader.Run() 47 | 48 | config := dht.NewCrawlConfig() 49 | config.OnAnnouncePeer = func(infoHash, ip string, port int) { 50 | // request to download the metadata info 51 | downloader.Request([]byte(infoHash), ip, port) 52 | } 53 | d := dht.New(config) 54 | 55 | d.Run() 56 | } 57 | ``` 58 | 59 | ## Download 60 | 61 | You can download the demo compiled binary file [here](https://github.com/shiyanhui/dht/files/407021/spider.zip). 62 | 63 | ## Note 64 | 65 | - The default crawl mode configure costs about 300M RAM. Set **MaxNodes** 66 | and **BlackListMaxSize** to fit yourself. 67 | - Now it cant't run in LAN because of NAT. 68 | 69 | ## TODO 70 | 71 | - [ ] NAT Traversal. 72 | - [ ] Implements the full BEP-3. 73 | - [ ] Optimization. 74 | 75 | ## FAQ 76 | 77 | #### Why it is slow compared to other spiders ? 78 | 79 | Well, maybe there are several reasons. 80 | 81 | - DHT aims to implements the standard BitTorrent DHT protocol, not born for crawling the DHT network. 82 | - NAT Traversal issue. You run the crawler in a local network. 83 | - It will block ip which looks like bad and a good ip may be mis-judged. 84 | 85 | ## License 86 | 87 | MIT, read more [here](https://github.com/shiyanhui/dht/blob/master/LICENSE) 88 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | ![](https://raw.githubusercontent.com/shiyanhui/dht/master/doc/screen-shot.png) 2 | 3 | 在这个视频上你可以看到爬取效果[Youtube](https://www.youtube.com/watch?v=AIpeQtw22kc). 4 | 5 | ## Introduction 6 | 7 | DHT实现了BitTorrent DHT协议，主要包括： 8 | 9 | - [BEP-3 (部分)](http://www.bittorrent.org/beps/bep_0003.html) 10 | - [BEP-5](http://www.bittorrent.org/beps/bep_0005.html) 11 | - [BEP-9](http://www.bittorrent.org/beps/bep_0009.html) 12 | - [BEP-10](http://www.bittorrent.org/beps/bep_0010.html) 13 | 14 | 它包含两种模式，标准模式和爬虫模式。标准模式遵循DHT协议，你可以把它当做一个标准 15 | 的DHT组件。爬虫模式是为了嗅探到更多torrent文件信息，它在某些方面不遵循DHT协议。 16 | 基于爬虫模式，你可以打造你自己的[BTDigg](http://btdigg.org/)。 17 | 18 | [bthub.io](http://bthub.io)是一个基于这个爬虫而建的BT搜索引擎，你可以把他当做 19 | BTDigg的替代品。 20 | 21 | ## Installation 22 | 23 | go get github.com/shiyanhui/dht 24 | 25 | ## Example 26 | 27 | 下面是一个简单的爬虫例子，你可以到[这里](https://github.com/shiyanhui/dht/blob/master/sample)看完整的Demo。 28 | 29 | ```go 30 | import ( 31 | "fmt" 32 | "github.com/shiyanhui/dht" 33 | ) 34 | 35 | func main() { 36 | downloader := dht.NewWire(65536) 37 | go func() { 38 | // once we got the request result 39 | for resp := range downloader.Response() { 40 | fmt.Println(resp.InfoHash, resp.MetadataInfo) 41 | } 42 | }() 43 | go downloader.Run() 44 | 45 | config := dht.NewCrawlConfig() 46 | config.OnAnnouncePeer = func(infoHash, ip string, port int) { 47 | // request to download the metadata info 48 | downloader.Request([]byte(infoHash), ip, port) 49 | } 50 | d := dht.New(config) 51 | 52 | d.Run() 53 | } 54 | ``` 55 | 56 | ## Download 57 | 58 | 这个是已经编译好的Demo二进制文件，你可以到这里[下载](https://github.com/shiyanhui/dht/files/407021/spider.zip)。 59 | 60 | ## 注意 61 | 62 | - 默认的爬虫配置需要300M左右内存，你可以根据你的服务器内存大小调整MaxNodes和 63 | BlackListMaxSize 64 | - 目前还不能穿透NAT，因此还不能在局域网运行 65 | 66 | ## TODO 67 | 68 | - [ ] NAT穿透，在局域网内也能够运行 69 | - [ ] 完整地实现BEP-3，这样不但能够下载种子，也能够下载资源 70 | - [ ] 优化 71 | 72 | ## Blog 73 | 74 | 你可以在[这里](https://github.com/shiyanhui/dht/wiki)看到DHT Spider教程。 75 | 76 | ## License 77 | 78 | [MIT](https://github.com/shiyanhui/dht/blob/master/LICENSE) 79 | -------------------------------------------------------------------------------- /bencode.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "strconv" 7 | "strings" 8 | "unicode" 9 | "unicode/utf8" 10 | ) 11 | 12 | // find returns the index of first target in data starting from `start`. 13 | // It returns -1 if target not found. 14 | func find(data []byte, start int, target rune) (index int) { 15 | index = bytes.IndexRune(data[start:], target) 16 | if index != -1 { 17 | return index + start 18 | } 19 | return index 20 | } 21 | 22 | // DecodeString decodes a string in the data. It returns a tuple 23 | // (decoded result, the end position, error). 24 | func DecodeString(data []byte, start int) ( 25 | result interface{}, index int, err error) { 26 | 27 | if start >= len(data) || data[start] < '0' || data[start] > '9' { 28 | err = errors.New("invalid string bencode") 29 | return 30 | } 31 | 32 | i := find(data, start, ':') 33 | if i == -1 { 34 | err = errors.New("':' not found when decode string") 35 | return 36 | } 37 | 38 | length, err := strconv.Atoi(string(data[start:i])) 39 | if err != nil { 40 | return 41 | } 42 | 43 | if length < 0 { 44 | err = errors.New("invalid length of string") 45 | return 46 | } 47 | 48 | index = i + 1 + length 49 | 50 | if index > len(data) || index < i+1 { 51 | err = errors.New("out of range") 52 | return 53 | } 54 | 55 | result = string(data[i+1 : index]) 56 | return 57 | } 58 | 59 | // DecodeInt decodes int value in the data. 60 | func DecodeInt(data []byte, start int) ( 61 | result interface{}, index int, err error) { 62 | 63 | if start >= len(data) || data[start] != 'i' { 64 | err = errors.New("invalid int bencode") 65 | return 66 | } 67 | 68 | index = find(data, start+1, 'e') 69 | 70 | if index == -1 { 71 | err = errors.New("':' not found when decode int") 72 | return 73 | } 74 | 75 | result, err = strconv.Atoi(string(data[start+1 : index])) 76 | if err != nil { 77 | return 78 | } 79 | index++ 80 | 81 | return 82 | } 83 | 84 | // decodeItem decodes an item of dict or list. 85 | func decodeItem(data []byte, i int) ( 86 | result interface{}, index int, err error) { 87 | 88 | var decodeFunc = []func([]byte, int) (interface{}, int, error){ 89 | DecodeString, DecodeInt, DecodeList, DecodeDict, 90 | } 91 | 92 | for _, f := range decodeFunc { 93 | result, index, err = f(data, i) 94 | if err == nil { 95 | return 96 | } 97 | } 98 | 99 | err = errors.New("invalid bencode when decode item") 100 | return 101 | } 102 | 103 | // DecodeList decodes a list value. 104 | func DecodeList(data []byte, start int) ( 105 | result interface{}, index int, err error) { 106 | 107 | if start >= len(data) || data[start] != 'l' { 108 | err = errors.New("invalid list bencode") 109 | return 110 | } 111 | 112 | var item interface{} 113 | r := make([]interface{}, 0, 8) 114 | 115 | index = start + 1 116 | for index < len(data) { 117 | char, _ := utf8.DecodeRune(data[index:]) 118 | if char == 'e' { 119 | break 120 | } 121 | 122 | item, index, err = decodeItem(data, index) 123 | if err != nil { 124 | return 125 | } 126 | r = append(r, item) 127 | } 128 | 129 | if index == len(data) { 130 | err = errors.New("'e' not found when decode list") 131 | return 132 | } 133 | index++ 134 | 135 | result = r 136 | return 137 | } 138 | 139 | // DecodeDict decodes a map value. 140 | func DecodeDict(data []byte, start int) ( 141 | result interface{}, index int, err error) { 142 | 143 | if start >= len(data) || data[start] != 'd' { 144 | err = errors.New("invalid dict bencode") 145 | return 146 | } 147 | 148 | var item, key interface{} 149 | r := make(map[string]interface{}) 150 | 151 | index = start + 1 152 | for index < len(data) { 153 | char, _ := utf8.DecodeRune(data[index:]) 154 | if char == 'e' { 155 | break 156 | } 157 | 158 | if !unicode.IsDigit(char) { 159 | err = errors.New("invalid dict bencode") 160 | return 161 | } 162 | 163 | key, index, err = DecodeString(data, index) 164 | if err != nil { 165 | return 166 | } 167 | 168 | if index >= len(data) { 169 | err = errors.New("out of range") 170 | return 171 | } 172 | 173 | item, index, err = decodeItem(data, index) 174 | if err != nil { 175 | return 176 | } 177 | 178 | r[key.(string)] = item 179 | } 180 | 181 | if index == len(data) { 182 | err = errors.New("'e' not found when decode dict") 183 | return 184 | } 185 | index++ 186 | 187 | result = r 188 | return 189 | } 190 | 191 | // Decode decodes a bencoded string to string, int, list or map. 192 | func Decode(data []byte) (result interface{}, err error) { 193 | result, _, err = decodeItem(data, 0) 194 | return 195 | } 196 | 197 | // EncodeString encodes a string value. 198 | func EncodeString(data string) string { 199 | return strings.Join([]string{strconv.Itoa(len(data)), data}, ":") 200 | } 201 | 202 | // EncodeInt encodes a int value. 203 | func EncodeInt(data int) string { 204 | return strings.Join([]string{"i", strconv.Itoa(data), "e"}, "") 205 | } 206 | 207 | // EncodeItem encodes an item of dict or list. 208 | func encodeItem(data interface{}) (item string) { 209 | switch v := data.(type) { 210 | case string: 211 | item = EncodeString(v) 212 | case int: 213 | item = EncodeInt(v) 214 | case []interface{}: 215 | item = EncodeList(v) 216 | case map[string]interface{}: 217 | item = EncodeDict(v) 218 | default: 219 | panic("invalid type when encode item") 220 | } 221 | return 222 | } 223 | 224 | // EncodeList encodes a list value. 225 | func EncodeList(data []interface{}) string { 226 | result := make([]string, len(data)) 227 | 228 | for i, item := range data { 229 | result[i] = encodeItem(item) 230 | } 231 | 232 | return strings.Join([]string{"l", strings.Join(result, ""), "e"}, "") 233 | } 234 | 235 | // EncodeDict encodes a dict value. 236 | func EncodeDict(data map[string]interface{}) string { 237 | result, i := make([]string, len(data)), 0 238 | 239 | for key, val := range data { 240 | result[i] = strings.Join( 241 | []string{EncodeString(key), encodeItem(val)}, 242 | "") 243 | i++ 244 | } 245 | 246 | return strings.Join([]string{"d", strings.Join(result, ""), "e"}, "") 247 | } 248 | 249 | // Encode encodes a string, int, dict or list value to a bencoded string. 250 | func Encode(data interface{}) string { 251 | switch v := data.(type) { 252 | case string: 253 | return EncodeString(v) 254 | case int: 255 | return EncodeInt(v) 256 | case []interface{}: 257 | return EncodeList(v) 258 | case map[string]interface{}: 259 | return EncodeDict(v) 260 | default: 261 | panic("invalid type when encode") 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /bencode_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDecodeString(t *testing.T) { 8 | cases := []struct { 9 | in string 10 | out string 11 | }{ 12 | {"0:", ""}, 13 | {"1:a", "a"}, 14 | {"5:hello", "hello"}, 15 | } 16 | 17 | for _, c := range cases { 18 | if out, err := Decode([]byte(c.in)); err != nil || out != c.out { 19 | t.Error(err) 20 | } 21 | } 22 | } 23 | 24 | func TestDecodeInt(t *testing.T) { 25 | cases := []struct { 26 | in string 27 | out int 28 | }{ 29 | {"i123e:", 123}, 30 | {"i0e", 0}, 31 | {"i-1e", -1}, 32 | } 33 | 34 | for _, c := range cases { 35 | if out, err := Decode([]byte(c.in)); err != nil || out != c.out { 36 | t.Error(err) 37 | } 38 | } 39 | } 40 | 41 | func TestDecodeList(t *testing.T) { 42 | cases := []struct { 43 | in string 44 | out []interface{} 45 | }{ 46 | {"li123ei-1ee", []interface{}{123, -1}}, 47 | {"l5:helloe", []interface{}{"hello"}}, 48 | {"ld5:hello5:worldee", []interface{}{map[string]interface{}{"hello": "world"}}}, 49 | {"lli1ei2eee", []interface{}{[]interface{}{1, 2}}}, 50 | } 51 | 52 | for i, c := range cases { 53 | v, err := Decode([]byte(c.in)) 54 | if err != nil { 55 | t.Fail() 56 | } 57 | 58 | out := v.([]interface{}) 59 | 60 | switch i { 61 | case 0, 1: 62 | for j, item := range out { 63 | if item != c.out[j] { 64 | t.Fail() 65 | } 66 | } 67 | case 2: 68 | if len(out) != 1 { 69 | t.Fail() 70 | } 71 | 72 | o := out[0].(map[string]interface{}) 73 | cout := c.out[0].(map[string]interface{}) 74 | 75 | for k, v := range o { 76 | if cv, ok := cout[k]; !ok || v != cv { 77 | t.Fail() 78 | } 79 | } 80 | case 3: 81 | if len(out) != 1 { 82 | t.Fail() 83 | } 84 | 85 | o := out[0].([]interface{}) 86 | cout := c.out[0].([]interface{}) 87 | 88 | for j, item := range o { 89 | if item != cout[j] { 90 | t.Fail() 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | func TestDecodeDict(t *testing.T) { 98 | cases := []struct { 99 | in string 100 | out map[string]interface{} 101 | }{ 102 | {"d5:helloi100ee", map[string]interface{}{"hello": 100}}, 103 | {"d3:foo3:bare", map[string]interface{}{"foo": "bar"}}, 104 | {"d1:ad3:foo3:baree", map[string]interface{}{"a": map[string]interface{}{"foo": "bar"}}}, 105 | {"d4:listli1eee", map[string]interface{}{"list": []interface{}{1}}}, 106 | } 107 | 108 | for i, c := range cases { 109 | v, err := Decode([]byte(c.in)) 110 | if err != nil { 111 | t.Fail() 112 | } 113 | 114 | out := v.(map[string]interface{}) 115 | 116 | switch i { 117 | case 0, 1: 118 | for k, v := range out { 119 | if cv, ok := c.out[k]; !ok || v != cv { 120 | t.Fail() 121 | } 122 | } 123 | case 2: 124 | if len(out) != 1 { 125 | t.Fail() 126 | } 127 | 128 | v, ok := out["a"] 129 | if !ok { 130 | t.Fail() 131 | } 132 | 133 | cout := c.out["a"].(map[string]interface{}) 134 | 135 | for k, v := range v.(map[string]interface{}) { 136 | if cv, ok := cout[k]; !ok || v != cv { 137 | t.Fail() 138 | } 139 | } 140 | case 3: 141 | if len(out) != 1 { 142 | t.Fail() 143 | } 144 | 145 | v, ok := out["list"] 146 | if !ok { 147 | t.Fail() 148 | } 149 | 150 | cout := c.out["list"].([]interface{}) 151 | 152 | for j, v := range v.([]interface{}) { 153 | if v != cout[j] { 154 | t.Fail() 155 | } 156 | } 157 | } 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /bitmap.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // bitmap represents a bit array. 10 | type bitmap struct { 11 | Size int 12 | data []byte 13 | } 14 | 15 | // newBitmap returns a size-length bitmap pointer. 16 | func newBitmap(size int) *bitmap { 17 | div, mod := size>>3, size&0x07 18 | if mod > 0 { 19 | div++ 20 | } 21 | return &bitmap{size, make([]byte, div)} 22 | } 23 | 24 | // newBitmapFrom returns a new copyed bitmap pointer which 25 | // newBitmap.data = other.data[:size]. 26 | func newBitmapFrom(other *bitmap, size int) *bitmap { 27 | bitmap := newBitmap(size) 28 | 29 | if size > other.Size { 30 | size = other.Size 31 | } 32 | 33 | div := size >> 3 34 | 35 | for i := 0; i < div; i++ { 36 | bitmap.data[i] = other.data[i] 37 | } 38 | 39 | for i := div << 3; i < size; i++ { 40 | if other.Bit(i) == 1 { 41 | bitmap.Set(i) 42 | } 43 | } 44 | 45 | return bitmap 46 | } 47 | 48 | // newBitmapFromBytes returns a bitmap pointer created from a byte array. 49 | func newBitmapFromBytes(data []byte) *bitmap { 50 | bitmap := newBitmap(len(data) << 3) 51 | copy(bitmap.data, data) 52 | return bitmap 53 | } 54 | 55 | // newBitmapFromString returns a bitmap pointer created from a string. 56 | func newBitmapFromString(data string) *bitmap { 57 | return newBitmapFromBytes([]byte(data)) 58 | } 59 | 60 | // Bit returns the bit at index. 61 | func (bitmap *bitmap) Bit(index int) int { 62 | if index >= bitmap.Size { 63 | panic("index out of range") 64 | } 65 | 66 | div, mod := index>>3, index&0x07 67 | return int((uint(bitmap.data[div]) & (1 << uint(7-mod))) >> uint(7-mod)) 68 | } 69 | 70 | // set sets the bit at index `index`. If bit is true, set 1, otherwise set 0. 71 | func (bitmap *bitmap) set(index int, bit int) { 72 | if index >= bitmap.Size { 73 | panic("index out of range") 74 | } 75 | 76 | div, mod := index>>3, index&0x07 77 | shift := byte(1 << uint(7-mod)) 78 | 79 | bitmap.data[div] &= ^shift 80 | if bit > 0 { 81 | bitmap.data[div] |= shift 82 | } 83 | } 84 | 85 | // Set sets the bit at idnex to 1. 86 | func (bitmap *bitmap) Set(index int) { 87 | bitmap.set(index, 1) 88 | } 89 | 90 | // Unset sets the bit at idnex to 0. 91 | func (bitmap *bitmap) Unset(index int) { 92 | bitmap.set(index, 0) 93 | } 94 | 95 | // Compare compares the prefixLen-prefix of two bitmap. 96 | // - If bitmap.data[:prefixLen] < other.data[:prefixLen], return -1. 97 | // - If bitmap.data[:prefixLen] > other.data[:prefixLen], return 1. 98 | // - Otherwise return 0. 99 | func (bitmap *bitmap) Compare(other *bitmap, prefixLen int) int { 100 | if prefixLen > bitmap.Size || prefixLen > other.Size { 101 | panic("index out of range") 102 | } 103 | 104 | div, mod := prefixLen>>3, prefixLen&0x07 105 | res := bytes.Compare(bitmap.data[:div], other.data[:div]) 106 | if res != 0 { 107 | return res 108 | } 109 | 110 | for i := div << 3; i < (div<<3)+mod; i++ { 111 | bit1, bit2 := bitmap.Bit(i), other.Bit(i) 112 | if bit1 > bit2 { 113 | return 1 114 | } else if bit1 < bit2 { 115 | return -1 116 | } 117 | } 118 | 119 | return 0 120 | } 121 | 122 | // Xor returns the xor value of two bitmap. 123 | func (bitmap *bitmap) Xor(other *bitmap) *bitmap { 124 | if bitmap.Size != other.Size { 125 | panic("size not the same") 126 | } 127 | 128 | distance := newBitmap(bitmap.Size) 129 | xor(distance.data, bitmap.data, other.data) 130 | 131 | return distance 132 | } 133 | 134 | // String returns the bit sequence string of the bitmap. 135 | func (bitmap *bitmap) String() string { 136 | div, mod := bitmap.Size>>3, bitmap.Size&0x07 137 | buff := make([]string, div+mod) 138 | 139 | for i := 0; i < div; i++ { 140 | buff[i] = fmt.Sprintf("%08b", bitmap.data[i]) 141 | } 142 | 143 | for i := div; i < div+mod; i++ { 144 | buff[i] = fmt.Sprintf("%1b", bitmap.Bit(div*8+(i-div))) 145 | } 146 | 147 | return strings.Join(buff, "") 148 | } 149 | 150 | // RawString returns the string value of bitmap.data. 151 | func (bitmap *bitmap) RawString() string { 152 | return string(bitmap.data) 153 | } 154 | -------------------------------------------------------------------------------- /bitmap_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBitmap(t *testing.T) { 8 | a := newBitmap(10) 9 | b := newBitmapFrom(a, 10) 10 | c := newBitmapFromBytes([]byte{48, 49, 50, 51, 52, 53, 54, 55, 56, 57}) 11 | d := newBitmapFromString("0123456789") 12 | e := newBitmap(10) 13 | 14 | // Bit 15 | for i := 0; i < a.Size; i++ { 16 | if a.Bit(i) != 0 { 17 | t.Fail() 18 | } 19 | } 20 | 21 | // Compare 22 | if c.Compare(d, d.Size) != 0 { 23 | t.Fail() 24 | } 25 | 26 | // RawString 27 | if c.RawString() != d.RawString() || c.RawString() != "0123456789" { 28 | t.Fail() 29 | } 30 | 31 | // Set 32 | b.Set(5) 33 | if b.Bit(5) != 1 { 34 | t.Fail() 35 | } 36 | 37 | // Unset 38 | b.Unset(5) 39 | if b.Bit(5) == 1 { 40 | t.Fail() 41 | } 42 | 43 | // String 44 | if e.String() != "0000000000" { 45 | t.Fail() 46 | } 47 | e.Set(9) 48 | if e.String() != "0000000001" { 49 | t.Fail() 50 | } 51 | e.Set(2) 52 | if e.String() != "0010000001" { 53 | t.Fail() 54 | } 55 | 56 | a.Set(0) 57 | a.Set(5) 58 | a.Set(8) 59 | if a.String() != "1000010010" { 60 | t.Fail() 61 | } 62 | 63 | // Xor 64 | b.Set(5) 65 | b.Set(9) 66 | if a.Xor(b).String() != "1000000011" { 67 | t.Fail() 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /bitmap_xor.go: -------------------------------------------------------------------------------- 1 | // +build !amd64,!386 2 | 3 | package dht 4 | 5 | func xor(dst, a, b []byte) { 6 | n := len(a) 7 | for i := 0; i < n; i++ { 8 | dst[i] = a[i] ^ b[i] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /bitmap_xorfast.go: -------------------------------------------------------------------------------- 1 | // +build amd64 386 2 | 3 | package dht 4 | 5 | import "unsafe" 6 | 7 | const wordSize = int(unsafe.Sizeof(uintptr(0))) 8 | 9 | func xor(dst, a, b []byte) { 10 | n := len(a) 11 | 12 | w := n / wordSize 13 | if w > 0 { 14 | dw := *(*[]uintptr)(unsafe.Pointer(&dst)) 15 | aw := *(*[]uintptr)(unsafe.Pointer(&a)) 16 | bw := *(*[]uintptr)(unsafe.Pointer(&b)) 17 | for i := 0; i < w; i++ { 18 | dw[i] = aw[i] ^ bw[i] 19 | } 20 | } 21 | 22 | for i := n - n%wordSize; i < n; i++ { 23 | dst[i] = a[i] ^ b[i] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /blacklist.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // blockedItem represents a blocked node. 8 | type blockedItem struct { 9 | ip string 10 | port int 11 | createTime time.Time 12 | } 13 | 14 | // blackList manages the blocked nodes including which sends bad information 15 | // and can't ping out. 16 | type blackList struct { 17 | list *syncedMap 18 | maxSize int 19 | expiredAfter time.Duration 20 | } 21 | 22 | // newBlackList returns a blackList pointer. 23 | func newBlackList(size int) *blackList { 24 | return &blackList{ 25 | list: newSyncedMap(), 26 | maxSize: size, 27 | expiredAfter: time.Hour * 1, 28 | } 29 | } 30 | 31 | // genKey returns a key. If port is less than 0, the key wil be ip. Ohterwise 32 | // it will be `ip:port` format. 33 | func (bl *blackList) genKey(ip string, port int) string { 34 | key := ip 35 | if port >= 0 { 36 | key = genAddress(ip, port) 37 | } 38 | return key 39 | } 40 | 41 | // insert adds a blocked item to the blacklist. 42 | func (bl *blackList) insert(ip string, port int) { 43 | if bl.list.Len() >= bl.maxSize { 44 | return 45 | } 46 | 47 | bl.list.Set(bl.genKey(ip, port), &blockedItem{ 48 | ip: ip, 49 | port: port, 50 | createTime: time.Now(), 51 | }) 52 | } 53 | 54 | // delete removes blocked item form the blackList. 55 | func (bl *blackList) delete(ip string, port int) { 56 | bl.list.Delete(bl.genKey(ip, port)) 57 | } 58 | 59 | // validate checks whether ip-port pair is in the block nodes list. 60 | func (bl *blackList) in(ip string, port int) bool { 61 | if _, ok := bl.list.Get(ip); ok { 62 | return true 63 | } 64 | 65 | key := bl.genKey(ip, port) 66 | 67 | v, ok := bl.list.Get(key) 68 | if ok { 69 | if time.Now().Sub(v.(*blockedItem).createTime) < bl.expiredAfter { 70 | return true 71 | } 72 | bl.list.Delete(key) 73 | } 74 | return false 75 | } 76 | 77 | // clear cleans the expired items every 10 minutes. 78 | func (bl *blackList) clear() { 79 | for _ = range time.Tick(time.Minute * 10) { 80 | keys := make([]interface{}, 0, 100) 81 | 82 | for item := range bl.list.Iter() { 83 | if time.Now().Sub( 84 | item.val.(*blockedItem).createTime) > bl.expiredAfter { 85 | 86 | keys = append(keys, item.key) 87 | } 88 | } 89 | 90 | bl.list.DeleteMulti(keys) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /blacklist_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | var blacklist = newBlackList(256) 9 | 10 | func TestGenKey(t *testing.T) { 11 | cases := []struct { 12 | in struct { 13 | ip string 14 | port int 15 | } 16 | out string 17 | }{ 18 | {struct { 19 | ip string 20 | port int 21 | }{"0.0.0.0", -1}, "0.0.0.0"}, 22 | {struct { 23 | ip string 24 | port int 25 | }{"1.1.1.1", 8080}, "1.1.1.1:8080"}, 26 | } 27 | 28 | for _, c := range cases { 29 | if blacklist.genKey(c.in.ip, c.in.port) != c.out { 30 | t.Fail() 31 | } 32 | } 33 | } 34 | 35 | func TestBlackList(t *testing.T) { 36 | address := []struct { 37 | ip string 38 | port int 39 | }{ 40 | {"0.0.0.0", -1}, 41 | {"1.1.1.1", 8080}, 42 | {"2.2.2.2", 8081}, 43 | } 44 | 45 | for _, addr := range address { 46 | blacklist.insert(addr.ip, addr.port) 47 | if !blacklist.in(addr.ip, addr.port) { 48 | t.Fail() 49 | } 50 | 51 | blacklist.delete(addr.ip, addr.port) 52 | if blacklist.in(addr.ip, addr.port) { 53 | fmt.Println(addr.ip) 54 | t.Fail() 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /container.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | ) 7 | 8 | type mapItem struct { 9 | key interface{} 10 | val interface{} 11 | } 12 | 13 | // syncedMap represents a goroutine-safe map. 14 | type syncedMap struct { 15 | *sync.RWMutex 16 | data map[interface{}]interface{} 17 | } 18 | 19 | // newSyncedMap returns a syncedMap pointer. 20 | func newSyncedMap() *syncedMap { 21 | return &syncedMap{ 22 | RWMutex: &sync.RWMutex{}, 23 | data: make(map[interface{}]interface{}), 24 | } 25 | } 26 | 27 | // Get returns the value mapped to key. 28 | func (smap *syncedMap) Get(key interface{}) (val interface{}, ok bool) { 29 | smap.RLock() 30 | defer smap.RUnlock() 31 | 32 | val, ok = smap.data[key] 33 | return 34 | } 35 | 36 | // Has returns whether the syncedMap contains the key. 37 | func (smap *syncedMap) Has(key interface{}) bool { 38 | _, ok := smap.Get(key) 39 | return ok 40 | } 41 | 42 | // Set sets pair {key: val}. 43 | func (smap *syncedMap) Set(key interface{}, val interface{}) { 44 | smap.Lock() 45 | defer smap.Unlock() 46 | 47 | smap.data[key] = val 48 | } 49 | 50 | // Delete deletes the key in the map. 51 | func (smap *syncedMap) Delete(key interface{}) { 52 | smap.Lock() 53 | defer smap.Unlock() 54 | 55 | delete(smap.data, key) 56 | } 57 | 58 | // DeleteMulti deletes keys in batch. 59 | func (smap *syncedMap) DeleteMulti(keys []interface{}) { 60 | smap.Lock() 61 | defer smap.Unlock() 62 | 63 | for _, key := range keys { 64 | delete(smap.data, key) 65 | } 66 | } 67 | 68 | // Clear resets the data. 69 | func (smap *syncedMap) Clear() { 70 | smap.Lock() 71 | defer smap.Unlock() 72 | 73 | smap.data = make(map[interface{}]interface{}) 74 | } 75 | 76 | // Iter returns a chan which output all items. 77 | func (smap *syncedMap) Iter() <-chan mapItem { 78 | ch := make(chan mapItem) 79 | go func() { 80 | smap.RLock() 81 | for key, val := range smap.data { 82 | ch <- mapItem{ 83 | key: key, 84 | val: val, 85 | } 86 | } 87 | smap.RUnlock() 88 | close(ch) 89 | }() 90 | return ch 91 | } 92 | 93 | // Len returns the length of syncedMap. 94 | func (smap *syncedMap) Len() int { 95 | smap.RLock() 96 | defer smap.RUnlock() 97 | 98 | return len(smap.data) 99 | } 100 | 101 | // syncedList represents a goroutine-safe list. 102 | type syncedList struct { 103 | *sync.RWMutex 104 | queue *list.List 105 | } 106 | 107 | // newSyncedList returns a syncedList pointer. 108 | func newSyncedList() *syncedList { 109 | return &syncedList{ 110 | RWMutex: &sync.RWMutex{}, 111 | queue: list.New(), 112 | } 113 | } 114 | 115 | // Front returns the first element of slist. 116 | func (slist *syncedList) Front() *list.Element { 117 | slist.RLock() 118 | defer slist.RUnlock() 119 | 120 | return slist.queue.Front() 121 | } 122 | 123 | // Back returns the last element of slist. 124 | func (slist *syncedList) Back() *list.Element { 125 | slist.RLock() 126 | defer slist.RUnlock() 127 | 128 | return slist.queue.Back() 129 | } 130 | 131 | // PushFront pushs an element to the head of slist. 132 | func (slist *syncedList) PushFront(v interface{}) *list.Element { 133 | slist.Lock() 134 | defer slist.Unlock() 135 | 136 | return slist.queue.PushFront(v) 137 | } 138 | 139 | // PushBack pushs an element to the tail of slist. 140 | func (slist *syncedList) PushBack(v interface{}) *list.Element { 141 | slist.Lock() 142 | defer slist.Unlock() 143 | 144 | return slist.queue.PushBack(v) 145 | } 146 | 147 | // InsertBefore inserts v before mark. 148 | func (slist *syncedList) InsertBefore( 149 | v interface{}, mark *list.Element) *list.Element { 150 | 151 | slist.Lock() 152 | defer slist.Unlock() 153 | 154 | return slist.queue.InsertBefore(v, mark) 155 | } 156 | 157 | // InsertAfter inserts v after mark. 158 | func (slist *syncedList) InsertAfter( 159 | v interface{}, mark *list.Element) *list.Element { 160 | 161 | slist.Lock() 162 | defer slist.Unlock() 163 | 164 | return slist.queue.InsertAfter(v, mark) 165 | } 166 | 167 | // Remove removes e from the slist. 168 | func (slist *syncedList) Remove(e *list.Element) interface{} { 169 | slist.Lock() 170 | defer slist.Unlock() 171 | 172 | return slist.queue.Remove(e) 173 | } 174 | 175 | // Clear resets the list queue. 176 | func (slist *syncedList) Clear() { 177 | slist.Lock() 178 | defer slist.Unlock() 179 | 180 | slist.queue.Init() 181 | } 182 | 183 | // Len returns length of the slist. 184 | func (slist *syncedList) Len() int { 185 | slist.RLock() 186 | defer slist.RUnlock() 187 | 188 | return slist.queue.Len() 189 | } 190 | 191 | // Iter returns a chan which output all elements. 192 | func (slist *syncedList) Iter() <-chan *list.Element { 193 | ch := make(chan *list.Element) 194 | go func() { 195 | slist.RLock() 196 | for e := slist.queue.Front(); e != nil; e = e.Next() { 197 | ch <- e 198 | } 199 | slist.RUnlock() 200 | close(ch) 201 | }() 202 | return ch 203 | } 204 | 205 | // KeyedDeque represents a keyed deque. 206 | type keyedDeque struct { 207 | *sync.RWMutex 208 | *syncedList 209 | index map[interface{}]*list.Element 210 | invertedIndex map[*list.Element]interface{} 211 | } 212 | 213 | // newKeyedDeque returns a newKeyedDeque pointer. 214 | func newKeyedDeque() *keyedDeque { 215 | return &keyedDeque{ 216 | RWMutex: &sync.RWMutex{}, 217 | syncedList: newSyncedList(), 218 | index: make(map[interface{}]*list.Element), 219 | invertedIndex: make(map[*list.Element]interface{}), 220 | } 221 | } 222 | 223 | // Push pushs a keyed-value to the end of deque. 224 | func (deque *keyedDeque) Push(key interface{}, val interface{}) { 225 | deque.Lock() 226 | defer deque.Unlock() 227 | 228 | if e, ok := deque.index[key]; ok { 229 | deque.syncedList.Remove(e) 230 | } 231 | deque.index[key] = deque.syncedList.PushBack(val) 232 | deque.invertedIndex[deque.index[key]] = key 233 | } 234 | 235 | // Get returns the keyed value. 236 | func (deque *keyedDeque) Get(key interface{}) (*list.Element, bool) { 237 | deque.RLock() 238 | defer deque.RUnlock() 239 | 240 | v, ok := deque.index[key] 241 | return v, ok 242 | } 243 | 244 | // Has returns whether key already exists. 245 | func (deque *keyedDeque) HasKey(key interface{}) bool { 246 | _, ok := deque.Get(key) 247 | return ok 248 | } 249 | 250 | // Delete deletes a value named key. 251 | func (deque *keyedDeque) Delete(key interface{}) (v interface{}) { 252 | deque.RLock() 253 | e, ok := deque.index[key] 254 | deque.RUnlock() 255 | 256 | deque.Lock() 257 | defer deque.Unlock() 258 | 259 | if ok { 260 | v = deque.syncedList.Remove(e) 261 | delete(deque.index, key) 262 | delete(deque.invertedIndex, e) 263 | } 264 | 265 | return 266 | } 267 | 268 | // Removes overwrites list.List.Remove. 269 | func (deque *keyedDeque) Remove(e *list.Element) (v interface{}) { 270 | deque.RLock() 271 | key, ok := deque.invertedIndex[e] 272 | deque.RUnlock() 273 | 274 | if ok { 275 | v = deque.Delete(key) 276 | } 277 | 278 | return 279 | } 280 | 281 | // Clear resets the deque. 282 | func (deque *keyedDeque) Clear() { 283 | deque.Lock() 284 | defer deque.Unlock() 285 | 286 | deque.syncedList.Clear() 287 | deque.index = make(map[interface{}]*list.Element) 288 | deque.invertedIndex = make(map[*list.Element]interface{}) 289 | } 290 | -------------------------------------------------------------------------------- /container_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | ) 7 | 8 | func TestSyncedMap(t *testing.T) { 9 | cases := []mapItem{ 10 | {"a", 0}, 11 | {"b", 1}, 12 | {"c", 2}, 13 | } 14 | 15 | sm := newSyncedMap() 16 | 17 | set := func() { 18 | group := sync.WaitGroup{} 19 | for _, item := range cases { 20 | group.Add(1) 21 | go func(item mapItem) { 22 | sm.Set(item.key, item.val) 23 | group.Done() 24 | }(item) 25 | } 26 | group.Wait() 27 | } 28 | 29 | isEmpty := func() { 30 | if sm.Len() != 0 { 31 | t.Fail() 32 | } 33 | } 34 | 35 | // Set 36 | set() 37 | if sm.Len() != len(cases) { 38 | t.Fail() 39 | } 40 | 41 | Loop: 42 | // Iter 43 | for item := range sm.Iter() { 44 | for _, c := range cases { 45 | if item.key == c.key && item.val == c.val { 46 | continue Loop 47 | } 48 | } 49 | t.Fail() 50 | } 51 | 52 | // Get, Delete, Has 53 | for _, item := range cases { 54 | val, ok := sm.Get(item.key) 55 | if !ok || val != item.val { 56 | t.Fail() 57 | } 58 | 59 | sm.Delete(item.key) 60 | if sm.Has(item.key) { 61 | t.Fail() 62 | } 63 | } 64 | isEmpty() 65 | 66 | // DeleteMulti 67 | set() 68 | sm.DeleteMulti([]interface{}{"a", "b", "c"}) 69 | isEmpty() 70 | 71 | // Clear 72 | set() 73 | sm.Clear() 74 | isEmpty() 75 | } 76 | 77 | func TestSyncedList(t *testing.T) { 78 | sl := newSyncedList() 79 | 80 | insert := func() { 81 | for i := 0; i < 10; i++ { 82 | sl.PushBack(i) 83 | } 84 | } 85 | 86 | isEmpty := func() { 87 | if sl.Len() != 0 { 88 | t.Fail() 89 | } 90 | } 91 | 92 | // PushBack 93 | insert() 94 | 95 | // Len 96 | if sl.Len() != 10 { 97 | t.Fail() 98 | } 99 | 100 | // Iter 101 | i := 0 102 | for item := range sl.Iter() { 103 | if item.Value.(int) != i { 104 | t.Fail() 105 | } 106 | i++ 107 | } 108 | 109 | // Front 110 | if sl.Front().Value.(int) != 0 { 111 | t.Fail() 112 | } 113 | 114 | // Back 115 | if sl.Back().Value.(int) != 9 { 116 | t.Fail() 117 | } 118 | 119 | // Remove 120 | for i := 0; i < 10; i++ { 121 | if sl.Remove(sl.Front()).(int) != i { 122 | t.Fail() 123 | } 124 | } 125 | isEmpty() 126 | 127 | // Clear 128 | insert() 129 | sl.Clear() 130 | isEmpty() 131 | } 132 | 133 | func TestKeyedDeque(t *testing.T) { 134 | cases := []mapItem{ 135 | {"a", 0}, 136 | {"b", 1}, 137 | {"c", 2}, 138 | } 139 | 140 | deque := newKeyedDeque() 141 | 142 | insert := func() { 143 | for _, item := range cases { 144 | deque.Push(item.key, item.val) 145 | } 146 | } 147 | 148 | isEmpty := func() { 149 | if deque.Len() != 0 { 150 | t.Fail() 151 | } 152 | } 153 | 154 | // Push 155 | insert() 156 | 157 | // Len 158 | if deque.Len() != 3 { 159 | t.Fail() 160 | } 161 | 162 | // Iter 163 | i := 0 164 | for e := range deque.Iter() { 165 | if e.Value.(int) != i { 166 | t.Fail() 167 | } 168 | i++ 169 | } 170 | 171 | // HasKey, Get, Delete 172 | for _, item := range cases { 173 | if !deque.HasKey(item.key) { 174 | t.Fail() 175 | } 176 | 177 | e, ok := deque.Get(item.key) 178 | if !ok || e.Value.(int) != item.val { 179 | t.Fail() 180 | } 181 | 182 | if deque.Delete(item.key) != item.val { 183 | t.Fail() 184 | } 185 | 186 | if deque.HasKey(item.key) { 187 | t.Fail() 188 | } 189 | } 190 | isEmpty() 191 | 192 | // Clear 193 | insert() 194 | deque.Clear() 195 | isEmpty() 196 | } 197 | -------------------------------------------------------------------------------- /dht.go: -------------------------------------------------------------------------------- 1 | // Package dht implements the bittorrent dht protocol. For more information 2 | // see http://www.bittorrent.org/beps/bep_0005.html. 3 | package dht 4 | 5 | import ( 6 | "encoding/hex" 7 | "errors" 8 | "math" 9 | "net" 10 | "time" 11 | ) 12 | 13 | const ( 14 | // StandardMode follows the standard protocol 15 | StandardMode = iota 16 | // CrawlMode for crawling the dht network. 17 | CrawlMode 18 | ) 19 | 20 | var ( 21 | // ErrNotReady is the error when DHT is not initialized. 22 | ErrNotReady = errors.New("dht is not ready") 23 | // ErrOnGetPeersResponseNotSet is the error that config 24 | // OnGetPeersResponseNotSet is not set when call dht.GetPeers. 25 | ErrOnGetPeersResponseNotSet = errors.New("OnGetPeersResponse is not set") 26 | ) 27 | 28 | // Config represents the configure of dht. 29 | type Config struct { 30 | // in mainline dht, k = 8 31 | K int 32 | // for crawling mode, we put all nodes in one bucket, so KBucketSize may 33 | // not be K 34 | KBucketSize int 35 | // candidates are udp, udp4, udp6 36 | Network string 37 | // format is `ip:port` 38 | Address string 39 | // the prime nodes through which we can join in dht network 40 | PrimeNodes []string 41 | // the kbucket expired duration 42 | KBucketExpiredAfter time.Duration 43 | // the node expired duration 44 | NodeExpriedAfter time.Duration 45 | // how long it checks whether the bucket is expired 46 | CheckKBucketPeriod time.Duration 47 | // peer token expired duration 48 | TokenExpiredAfter time.Duration 49 | // the max transaction id 50 | MaxTransactionCursor uint64 51 | // how many nodes routing table can hold 52 | MaxNodes int 53 | // callback when got get_peers request 54 | OnGetPeers func(string, string, int) 55 | // callback when receive get_peers response 56 | OnGetPeersResponse func(string, *Peer) 57 | // callback when got announce_peer request 58 | OnAnnouncePeer func(string, string, int) 59 | // blcoked ips 60 | BlockedIPs []string 61 | // blacklist size 62 | BlackListMaxSize int 63 | // StandardMode or CrawlMode 64 | Mode int 65 | // the times it tries when send fails 66 | Try int 67 | // the size of packet need to be dealt with 68 | PacketJobLimit int 69 | // the size of packet handler 70 | PacketWorkerLimit int 71 | // the nodes num to be fresh in a kbucket 72 | RefreshNodeNum int 73 | } 74 | 75 | // NewStandardConfig returns a Config pointer with default values. 76 | func NewStandardConfig() *Config { 77 | return &Config{ 78 | K: 8, 79 | KBucketSize: 8, 80 | Network: "udp4", 81 | Address: ":6881", 82 | PrimeNodes: []string{ 83 | "router.bittorrent.com:6881", 84 | "router.utorrent.com:6881", 85 | "dht.transmissionbt.com:6881", 86 | }, 87 | NodeExpriedAfter: time.Duration(time.Minute * 15), 88 | KBucketExpiredAfter: time.Duration(time.Minute * 15), 89 | CheckKBucketPeriod: time.Duration(time.Second * 30), 90 | TokenExpiredAfter: time.Duration(time.Minute * 10), 91 | MaxTransactionCursor: math.MaxUint32, 92 | MaxNodes: 5000, 93 | BlockedIPs: make([]string, 0), 94 | BlackListMaxSize: 65536, 95 | Try: 2, 96 | Mode: StandardMode, 97 | PacketJobLimit: 1024, 98 | PacketWorkerLimit: 256, 99 | RefreshNodeNum: 8, 100 | } 101 | } 102 | 103 | // NewCrawlConfig returns a config in crawling mode. 104 | func NewCrawlConfig() *Config { 105 | config := NewStandardConfig() 106 | config.NodeExpriedAfter = 0 107 | config.KBucketExpiredAfter = 0 108 | config.CheckKBucketPeriod = time.Second * 5 109 | config.KBucketSize = math.MaxInt32 110 | config.Mode = CrawlMode 111 | config.RefreshNodeNum = 256 112 | 113 | return config 114 | } 115 | 116 | // DHT represents a DHT node. 117 | type DHT struct { 118 | *Config 119 | node *node 120 | conn *net.UDPConn 121 | routingTable *routingTable 122 | transactionManager *transactionManager 123 | peersManager *peersManager 124 | tokenManager *tokenManager 125 | blackList *blackList 126 | Ready bool 127 | packets chan packet 128 | workerTokens chan struct{} 129 | } 130 | 131 | // New returns a DHT pointer. If config is nil, then config will be set to 132 | // the default config. 133 | func New(config *Config) *DHT { 134 | if config == nil { 135 | config = NewStandardConfig() 136 | } 137 | 138 | node, err := newNode(randomString(20), config.Network, config.Address) 139 | if err != nil { 140 | panic(err) 141 | } 142 | 143 | d := &DHT{ 144 | Config: config, 145 | node: node, 146 | blackList: newBlackList(config.BlackListMaxSize), 147 | packets: make(chan packet, config.PacketJobLimit), 148 | workerTokens: make(chan struct{}, config.PacketWorkerLimit), 149 | } 150 | 151 | for _, ip := range config.BlockedIPs { 152 | d.blackList.insert(ip, -1) 153 | } 154 | 155 | go func() { 156 | for _, ip := range getLocalIPs() { 157 | d.blackList.insert(ip, -1) 158 | } 159 | 160 | ip, err := getRemoteIP() 161 | if err != nil { 162 | d.blackList.insert(ip, -1) 163 | } 164 | }() 165 | 166 | return d 167 | } 168 | 169 | // IsStandardMode returns whether mode is StandardMode. 170 | func (dht *DHT) IsStandardMode() bool { 171 | return dht.Mode == StandardMode 172 | } 173 | 174 | // IsCrawlMode returns whether mode is CrawlMode. 175 | func (dht *DHT) IsCrawlMode() bool { 176 | return dht.Mode == CrawlMode 177 | } 178 | 179 | // init initializes global varables. 180 | func (dht *DHT) init() { 181 | listener, err := net.ListenPacket(dht.Network, dht.Address) 182 | if err != nil { 183 | panic(err) 184 | } 185 | 186 | dht.conn = listener.(*net.UDPConn) 187 | dht.routingTable = newRoutingTable(dht.KBucketSize, dht) 188 | dht.peersManager = newPeersManager(dht) 189 | dht.tokenManager = newTokenManager(dht.TokenExpiredAfter, dht) 190 | dht.transactionManager = newTransactionManager( 191 | dht.MaxTransactionCursor, dht) 192 | 193 | go dht.transactionManager.run() 194 | go dht.tokenManager.clear() 195 | go dht.blackList.clear() 196 | } 197 | 198 | // join makes current node join the dht network. 199 | func (dht *DHT) join() { 200 | for _, addr := range dht.PrimeNodes { 201 | raddr, err := net.ResolveUDPAddr(dht.Network, addr) 202 | if err != nil { 203 | continue 204 | } 205 | 206 | // NOTE: Temporary node has NOT node id. 207 | dht.transactionManager.findNode( 208 | &node{addr: raddr}, 209 | dht.node.id.RawString(), 210 | ) 211 | } 212 | } 213 | 214 | // listen receives message from udp. 215 | func (dht *DHT) listen() { 216 | go func() { 217 | buff := make([]byte, 8192) 218 | for { 219 | n, raddr, err := dht.conn.ReadFromUDP(buff) 220 | if err != nil { 221 | continue 222 | } 223 | 224 | dht.packets <- packet{buff[:n], raddr} 225 | } 226 | }() 227 | } 228 | 229 | // id returns a id near to target if target is not null, otherwise it returns 230 | // the dht's node id. 231 | func (dht *DHT) id(target string) string { 232 | if dht.IsStandardMode() || target == "" { 233 | return dht.node.id.RawString() 234 | } 235 | return target[:15] + dht.node.id.RawString()[15:] 236 | } 237 | 238 | // GetPeers returns peers who have announced having infoHash. 239 | func (dht *DHT) GetPeers(infoHash string) error { 240 | if !dht.Ready { 241 | return ErrNotReady 242 | } 243 | 244 | if dht.OnGetPeersResponse == nil { 245 | return ErrOnGetPeersResponseNotSet 246 | } 247 | 248 | if len(infoHash) == 40 { 249 | data, err := hex.DecodeString(infoHash) 250 | if err != nil { 251 | return err 252 | } 253 | infoHash = string(data) 254 | } 255 | 256 | neighbors := dht.routingTable.GetNeighbors( 257 | newBitmapFromString(infoHash), dht.routingTable.Len()) 258 | 259 | for _, no := range neighbors { 260 | dht.transactionManager.getPeers(no, infoHash) 261 | } 262 | 263 | return nil 264 | } 265 | 266 | // Run starts the dht. 267 | func (dht *DHT) Run() { 268 | dht.init() 269 | dht.listen() 270 | dht.join() 271 | 272 | dht.Ready = true 273 | 274 | var pkt packet 275 | tick := time.Tick(dht.CheckKBucketPeriod) 276 | 277 | for { 278 | select { 279 | case pkt = <-dht.packets: 280 | handle(dht, pkt) 281 | case <-tick: 282 | if dht.routingTable.Len() == 0 { 283 | dht.join() 284 | } else if dht.transactionManager.len() == 0 { 285 | go dht.routingTable.Fresh() 286 | } 287 | } 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /doc/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shiyanhui/dht/5a20f319926383187ead6e7eb2509b08599a9ee9/doc/.DS_Store -------------------------------------------------------------------------------- /doc/screen-shot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shiyanhui/dht/5a20f319926383187ead6e7eb2509b08599a9ee9/doc/screen-shot.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/shiyanhui/dht 2 | 3 | go 1.15 4 | -------------------------------------------------------------------------------- /krpc.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | "strings" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | const ( 12 | pingType = "ping" 13 | findNodeType = "find_node" 14 | getPeersType = "get_peers" 15 | announcePeerType = "announce_peer" 16 | ) 17 | 18 | const ( 19 | generalError = 201 + iota 20 | serverError 21 | protocolError 22 | unknownError 23 | ) 24 | 25 | // packet represents the information receive from udp. 26 | type packet struct { 27 | data []byte 28 | raddr *net.UDPAddr 29 | } 30 | 31 | // token represents the token when response getPeers request. 32 | type token struct { 33 | data string 34 | createTime time.Time 35 | } 36 | 37 | // tokenManager managers the tokens. 38 | type tokenManager struct { 39 | *syncedMap 40 | expiredAfter time.Duration 41 | dht *DHT 42 | } 43 | 44 | // newTokenManager returns a new tokenManager. 45 | func newTokenManager(expiredAfter time.Duration, dht *DHT) *tokenManager { 46 | return &tokenManager{ 47 | syncedMap: newSyncedMap(), 48 | expiredAfter: expiredAfter, 49 | dht: dht, 50 | } 51 | } 52 | 53 | // token returns a token. If it doesn't exist or is expired, it will add a 54 | // new token. 55 | func (tm *tokenManager) token(addr *net.UDPAddr) string { 56 | v, ok := tm.Get(addr.IP.String()) 57 | tk, _ := v.(token) 58 | 59 | if !ok || time.Now().Sub(tk.createTime) > tm.expiredAfter { 60 | tk = token{ 61 | data: randomString(5), 62 | createTime: time.Now(), 63 | } 64 | 65 | tm.Set(addr.IP.String(), tk) 66 | } 67 | 68 | return tk.data 69 | } 70 | 71 | // clear removes expired tokens. 72 | func (tm *tokenManager) clear() { 73 | for _ = range time.Tick(time.Minute * 3) { 74 | keys := make([]interface{}, 0, 100) 75 | 76 | for item := range tm.Iter() { 77 | if time.Now().Sub(item.val.(token).createTime) > tm.expiredAfter { 78 | keys = append(keys, item.key) 79 | } 80 | } 81 | 82 | tm.DeleteMulti(keys) 83 | } 84 | } 85 | 86 | // check returns whether the token is valid. 87 | func (tm *tokenManager) check(addr *net.UDPAddr, tokenString string) bool { 88 | key := addr.IP.String() 89 | v, ok := tm.Get(key) 90 | tk, _ := v.(token) 91 | 92 | if ok { 93 | tm.Delete(key) 94 | } 95 | 96 | return ok && tokenString == tk.data 97 | } 98 | 99 | // makeQuery returns a query-formed data. 100 | func makeQuery(t, q string, a map[string]interface{}) map[string]interface{} { 101 | return map[string]interface{}{ 102 | "t": t, 103 | "y": "q", 104 | "q": q, 105 | "a": a, 106 | } 107 | } 108 | 109 | // makeResponse returns a response-formed data. 110 | func makeResponse(t string, r map[string]interface{}) map[string]interface{} { 111 | return map[string]interface{}{ 112 | "t": t, 113 | "y": "r", 114 | "r": r, 115 | } 116 | } 117 | 118 | // makeError returns a err-formed data. 119 | func makeError(t string, errCode int, errMsg string) map[string]interface{} { 120 | return map[string]interface{}{ 121 | "t": t, 122 | "y": "e", 123 | "e": []interface{}{errCode, errMsg}, 124 | } 125 | } 126 | 127 | // send sends data to the udp. 128 | func send(dht *DHT, addr *net.UDPAddr, data map[string]interface{}) error { 129 | dht.conn.SetWriteDeadline(time.Now().Add(time.Second * 15)) 130 | 131 | _, err := dht.conn.WriteToUDP([]byte(Encode(data)), addr) 132 | if err != nil { 133 | dht.blackList.insert(addr.IP.String(), -1) 134 | } 135 | return err 136 | } 137 | 138 | // query represents the query data included queried node and query-formed data. 139 | type query struct { 140 | node *node 141 | data map[string]interface{} 142 | } 143 | 144 | // transaction implements transaction. 145 | type transaction struct { 146 | *query 147 | id string 148 | response chan struct{} 149 | } 150 | 151 | // transactionManager represents the manager of transactions. 152 | type transactionManager struct { 153 | *sync.RWMutex 154 | transactions *syncedMap 155 | index *syncedMap 156 | cursor uint64 157 | maxCursor uint64 158 | queryChan chan *query 159 | dht *DHT 160 | } 161 | 162 | // newTransactionManager returns new transactionManager pointer. 163 | func newTransactionManager(maxCursor uint64, dht *DHT) *transactionManager { 164 | return &transactionManager{ 165 | RWMutex: &sync.RWMutex{}, 166 | transactions: newSyncedMap(), 167 | index: newSyncedMap(), 168 | maxCursor: maxCursor, 169 | queryChan: make(chan *query, 1024), 170 | dht: dht, 171 | } 172 | } 173 | 174 | // genTransID generates a transaction id and returns it. 175 | func (tm *transactionManager) genTransID() string { 176 | tm.Lock() 177 | defer tm.Unlock() 178 | 179 | tm.cursor = (tm.cursor + 1) % tm.maxCursor 180 | return string(int2bytes(tm.cursor)) 181 | } 182 | 183 | // newTransaction creates a new transaction. 184 | func (tm *transactionManager) newTransaction(id string, q *query) *transaction { 185 | return &transaction{ 186 | id: id, 187 | query: q, 188 | response: make(chan struct{}, tm.dht.Try+1), 189 | } 190 | } 191 | 192 | // genIndexKey generates an indexed key which consists of queryType and 193 | // address. 194 | func (tm *transactionManager) genIndexKey(queryType, address string) string { 195 | return strings.Join([]string{queryType, address}, ":") 196 | } 197 | 198 | // genIndexKeyByTrans generates an indexed key by a transaction. 199 | func (tm *transactionManager) genIndexKeyByTrans(trans *transaction) string { 200 | return tm.genIndexKey(trans.data["q"].(string), trans.node.addr.String()) 201 | } 202 | 203 | // insert adds a transaction to transactionManager. 204 | func (tm *transactionManager) insert(trans *transaction) { 205 | tm.Lock() 206 | defer tm.Unlock() 207 | 208 | tm.transactions.Set(trans.id, trans) 209 | tm.index.Set(tm.genIndexKeyByTrans(trans), trans) 210 | } 211 | 212 | // delete removes a transaction from transactionManager. 213 | func (tm *transactionManager) delete(transID string) { 214 | v, ok := tm.transactions.Get(transID) 215 | if !ok { 216 | return 217 | } 218 | 219 | tm.Lock() 220 | defer tm.Unlock() 221 | 222 | trans := v.(*transaction) 223 | tm.transactions.Delete(trans.id) 224 | tm.index.Delete(tm.genIndexKeyByTrans(trans)) 225 | } 226 | 227 | // len returns how many transactions are requesting now. 228 | func (tm *transactionManager) len() int { 229 | return tm.transactions.Len() 230 | } 231 | 232 | // transaction returns a transaction. keyType should be one of 0, 1 which 233 | // represents transId and index each. 234 | func (tm *transactionManager) transaction( 235 | key string, keyType int) *transaction { 236 | 237 | sm := tm.transactions 238 | if keyType == 1 { 239 | sm = tm.index 240 | } 241 | 242 | v, ok := sm.Get(key) 243 | if !ok { 244 | return nil 245 | } 246 | 247 | return v.(*transaction) 248 | } 249 | 250 | // getByTransID returns a transaction by transID. 251 | func (tm *transactionManager) getByTransID(transID string) *transaction { 252 | return tm.transaction(transID, 0) 253 | } 254 | 255 | // getByIndex returns a transaction by indexed key. 256 | func (tm *transactionManager) getByIndex(index string) *transaction { 257 | return tm.transaction(index, 1) 258 | } 259 | 260 | // transaction gets the proper transaction with whose id is transId and 261 | // address is addr. 262 | func (tm *transactionManager) filterOne( 263 | transID string, addr *net.UDPAddr) *transaction { 264 | 265 | trans := tm.getByTransID(transID) 266 | if trans == nil || trans.node.addr.String() != addr.String() { 267 | return nil 268 | } 269 | 270 | return trans 271 | } 272 | 273 | // query sends the query-formed data to udp and wait for the response. 274 | // When timeout, it will retry `try - 1` times, which means it will query 275 | // `try` times totally. 276 | func (tm *transactionManager) query(q *query, try int) { 277 | transID := q.data["t"].(string) 278 | trans := tm.newTransaction(transID, q) 279 | 280 | tm.insert(trans) 281 | defer tm.delete(trans.id) 282 | 283 | success := false 284 | for i := 0; i < try; i++ { 285 | if err := send(tm.dht, q.node.addr, q.data); err != nil { 286 | break 287 | } 288 | 289 | select { 290 | case <-trans.response: 291 | success = true 292 | break 293 | case <-time.After(time.Second * 15): 294 | } 295 | } 296 | 297 | if !success && q.node.id != nil { 298 | tm.dht.blackList.insert(q.node.addr.IP.String(), q.node.addr.Port) 299 | tm.dht.routingTable.RemoveByAddr(q.node.addr.String()) 300 | } 301 | } 302 | 303 | // run starts to listen and consume the query chan. 304 | func (tm *transactionManager) run() { 305 | var q *query 306 | 307 | for { 308 | select { 309 | case q = <-tm.queryChan: 310 | go tm.query(q, tm.dht.Try) 311 | } 312 | } 313 | } 314 | 315 | // sendQuery send query-formed data to the chan. 316 | func (tm *transactionManager) sendQuery( 317 | no *node, queryType string, a map[string]interface{}) { 318 | 319 | // If the target is self, then stop. 320 | if no.id != nil && no.id.RawString() == tm.dht.node.id.RawString() || 321 | tm.getByIndex(tm.genIndexKey(queryType, no.addr.String())) != nil || 322 | tm.dht.blackList.in(no.addr.IP.String(), no.addr.Port) { 323 | return 324 | } 325 | 326 | data := makeQuery(tm.genTransID(), queryType, a) 327 | tm.queryChan <- &query{ 328 | node: no, 329 | data: data, 330 | } 331 | } 332 | 333 | // ping sends ping query to the chan. 334 | func (tm *transactionManager) ping(no *node) { 335 | tm.sendQuery(no, pingType, map[string]interface{}{ 336 | "id": tm.dht.id(no.id.RawString()), 337 | }) 338 | } 339 | 340 | // findNode sends find_node query to the chan. 341 | func (tm *transactionManager) findNode(no *node, target string) { 342 | tm.sendQuery(no, findNodeType, map[string]interface{}{ 343 | "id": tm.dht.id(target), 344 | "target": target, 345 | }) 346 | } 347 | 348 | // getPeers sends get_peers query to the chan. 349 | func (tm *transactionManager) getPeers(no *node, infoHash string) { 350 | tm.sendQuery(no, getPeersType, map[string]interface{}{ 351 | "id": tm.dht.id(infoHash), 352 | "info_hash": infoHash, 353 | }) 354 | } 355 | 356 | // announcePeer sends announce_peer query to the chan. 357 | func (tm *transactionManager) announcePeer( 358 | no *node, infoHash string, impliedPort, port int, token string) { 359 | 360 | tm.sendQuery(no, announcePeerType, map[string]interface{}{ 361 | "id": tm.dht.id(no.id.RawString()), 362 | "info_hash": infoHash, 363 | "implied_port": impliedPort, 364 | "port": port, 365 | "token": token, 366 | }) 367 | } 368 | 369 | // ParseKey parses the key in dict data. `t` is type of the keyed value. 370 | // It's one of "int", "string", "map", "list". 371 | func ParseKey(data map[string]interface{}, key string, t string) error { 372 | val, ok := data[key] 373 | if !ok { 374 | return errors.New("lack of key") 375 | } 376 | 377 | switch t { 378 | case "string": 379 | _, ok = val.(string) 380 | case "int": 381 | _, ok = val.(int) 382 | case "map": 383 | _, ok = val.(map[string]interface{}) 384 | case "list": 385 | _, ok = val.([]interface{}) 386 | default: 387 | panic("invalid type") 388 | } 389 | 390 | if !ok { 391 | return errors.New("invalid key type") 392 | } 393 | 394 | return nil 395 | } 396 | 397 | // ParseKeys parses keys. It just wraps ParseKey. 398 | func ParseKeys(data map[string]interface{}, pairs [][]string) error { 399 | for _, args := range pairs { 400 | key, t := args[0], args[1] 401 | if err := ParseKey(data, key, t); err != nil { 402 | return err 403 | } 404 | } 405 | return nil 406 | } 407 | 408 | // parseMessage parses the basic data received from udp. 409 | // It returns a map value. 410 | func parseMessage(data interface{}) (map[string]interface{}, error) { 411 | response, ok := data.(map[string]interface{}) 412 | if !ok { 413 | return nil, errors.New("response is not dict") 414 | } 415 | 416 | if err := ParseKeys( 417 | response, [][]string{{"t", "string"}, {"y", "string"}}); err != nil { 418 | return nil, err 419 | } 420 | 421 | return response, nil 422 | } 423 | 424 | // handleRequest handles the requests received from udp. 425 | func handleRequest(dht *DHT, addr *net.UDPAddr, 426 | response map[string]interface{}) (success bool) { 427 | 428 | t := response["t"].(string) 429 | 430 | if err := ParseKeys( 431 | response, [][]string{{"q", "string"}, {"a", "map"}}); err != nil { 432 | 433 | send(dht, addr, makeError(t, protocolError, err.Error())) 434 | return 435 | } 436 | 437 | q := response["q"].(string) 438 | a := response["a"].(map[string]interface{}) 439 | 440 | if err := ParseKey(a, "id", "string"); err != nil { 441 | send(dht, addr, makeError(t, protocolError, err.Error())) 442 | return 443 | } 444 | 445 | id := a["id"].(string) 446 | 447 | if id == dht.node.id.RawString() { 448 | return 449 | } 450 | 451 | if len(id) != 20 { 452 | send(dht, addr, makeError(t, protocolError, "invalid id")) 453 | return 454 | } 455 | 456 | if no, ok := dht.routingTable.GetNodeByAddress(addr.String()); ok && 457 | no.id.RawString() != id { 458 | 459 | dht.blackList.insert(addr.IP.String(), addr.Port) 460 | dht.routingTable.RemoveByAddr(addr.String()) 461 | 462 | send(dht, addr, makeError(t, protocolError, "invalid id")) 463 | return 464 | } 465 | 466 | switch q { 467 | case pingType: 468 | send(dht, addr, makeResponse(t, map[string]interface{}{ 469 | "id": dht.id(id), 470 | })) 471 | case findNodeType: 472 | if dht.IsStandardMode() { 473 | if err := ParseKey(a, "target", "string"); err != nil { 474 | send(dht, addr, makeError(t, protocolError, err.Error())) 475 | return 476 | } 477 | 478 | target := a["target"].(string) 479 | if len(target) != 20 { 480 | send(dht, addr, makeError(t, protocolError, "invalid target")) 481 | return 482 | } 483 | 484 | var nodes string 485 | targetID := newBitmapFromString(target) 486 | 487 | no, _ := dht.routingTable.GetNodeKBucktByID(targetID) 488 | if no != nil { 489 | nodes = no.CompactNodeInfo() 490 | } else { 491 | nodes = strings.Join( 492 | dht.routingTable.GetNeighborCompactInfos(targetID, dht.K), 493 | "", 494 | ) 495 | } 496 | 497 | send(dht, addr, makeResponse(t, map[string]interface{}{ 498 | "id": dht.id(target), 499 | "nodes": nodes, 500 | })) 501 | } 502 | case getPeersType: 503 | if err := ParseKey(a, "info_hash", "string"); err != nil { 504 | send(dht, addr, makeError(t, protocolError, err.Error())) 505 | return 506 | } 507 | 508 | infoHash := a["info_hash"].(string) 509 | 510 | if len(infoHash) != 20 { 511 | send(dht, addr, makeError(t, protocolError, "invalid info_hash")) 512 | return 513 | } 514 | 515 | if dht.IsCrawlMode() { 516 | send(dht, addr, makeResponse(t, map[string]interface{}{ 517 | "id": dht.id(infoHash), 518 | "token": dht.tokenManager.token(addr), 519 | "nodes": "", 520 | })) 521 | } else if peers := dht.peersManager.GetPeers( 522 | infoHash, dht.K); len(peers) > 0 { 523 | 524 | values := make([]interface{}, len(peers)) 525 | for i, p := range peers { 526 | values[i] = p.CompactIPPortInfo() 527 | } 528 | 529 | send(dht, addr, makeResponse(t, map[string]interface{}{ 530 | "id": dht.id(infoHash), 531 | "values": values, 532 | "token": dht.tokenManager.token(addr), 533 | })) 534 | } else { 535 | send(dht, addr, makeResponse(t, map[string]interface{}{ 536 | "id": dht.id(infoHash), 537 | "token": dht.tokenManager.token(addr), 538 | "nodes": strings.Join(dht.routingTable.GetNeighborCompactInfos( 539 | newBitmapFromString(infoHash), dht.K), ""), 540 | })) 541 | } 542 | 543 | if dht.OnGetPeers != nil { 544 | dht.OnGetPeers(infoHash, addr.IP.String(), addr.Port) 545 | } 546 | case announcePeerType: 547 | if err := ParseKeys(a, [][]string{ 548 | {"info_hash", "string"}, 549 | {"port", "int"}, 550 | {"token", "string"}}); err != nil { 551 | 552 | send(dht, addr, makeError(t, protocolError, err.Error())) 553 | return 554 | } 555 | 556 | infoHash := a["info_hash"].(string) 557 | port := a["port"].(int) 558 | token := a["token"].(string) 559 | 560 | if !dht.tokenManager.check(addr, token) { 561 | // send(dht, addr, makeError(t, protocolError, "invalid token")) 562 | return 563 | } 564 | 565 | if impliedPort, ok := a["implied_port"]; ok && 566 | impliedPort.(int) != 0 { 567 | 568 | port = addr.Port 569 | } 570 | 571 | if dht.IsStandardMode() { 572 | dht.peersManager.Insert(infoHash, newPeer(addr.IP, port, token)) 573 | 574 | send(dht, addr, makeResponse(t, map[string]interface{}{ 575 | "id": dht.id(id), 576 | })) 577 | } 578 | 579 | if dht.OnAnnouncePeer != nil { 580 | dht.OnAnnouncePeer(infoHash, addr.IP.String(), port) 581 | } 582 | default: 583 | // send(dht, addr, makeError(t, protocolError, "invalid q")) 584 | return 585 | } 586 | 587 | no, _ := newNode(id, addr.Network(), addr.String()) 588 | dht.routingTable.Insert(no) 589 | return true 590 | } 591 | 592 | // findOn puts nodes in the response to the routingTable, then if target is in 593 | // the nodes or all nodes are in the routingTable, it stops. Otherwise it 594 | // continues to findNode or getPeers. 595 | func findOn(dht *DHT, r map[string]interface{}, target *bitmap, 596 | queryType string) error { 597 | 598 | if err := ParseKey(r, "nodes", "string"); err != nil { 599 | return err 600 | } 601 | 602 | nodes := r["nodes"].(string) 603 | if len(nodes)%26 != 0 { 604 | return errors.New("the length of nodes should can be divided by 26") 605 | } 606 | 607 | hasNew, found := false, false 608 | for i := 0; i < len(nodes)/26; i++ { 609 | no, _ := newNodeFromCompactInfo( 610 | string(nodes[i*26:(i+1)*26]), dht.Network) 611 | 612 | if no.id.RawString() == target.RawString() { 613 | found = true 614 | } 615 | 616 | if dht.routingTable.Insert(no) { 617 | hasNew = true 618 | } 619 | } 620 | 621 | if found || !hasNew { 622 | return nil 623 | } 624 | 625 | targetID := target.RawString() 626 | for _, no := range dht.routingTable.GetNeighbors(target, dht.K) { 627 | switch queryType { 628 | case findNodeType: 629 | dht.transactionManager.findNode(no, targetID) 630 | case getPeersType: 631 | dht.transactionManager.getPeers(no, targetID) 632 | default: 633 | panic("invalid find type") 634 | } 635 | } 636 | return nil 637 | } 638 | 639 | // handleResponse handles responses received from udp. 640 | func handleResponse(dht *DHT, addr *net.UDPAddr, 641 | response map[string]interface{}) (success bool) { 642 | 643 | t := response["t"].(string) 644 | 645 | trans := dht.transactionManager.filterOne(t, addr) 646 | if trans == nil { 647 | return 648 | } 649 | 650 | // inform transManager to delete the transaction. 651 | if err := ParseKey(response, "r", "map"); err != nil { 652 | return 653 | } 654 | 655 | q := trans.data["q"].(string) 656 | a := trans.data["a"].(map[string]interface{}) 657 | r := response["r"].(map[string]interface{}) 658 | 659 | if err := ParseKey(r, "id", "string"); err != nil { 660 | return 661 | } 662 | 663 | id := r["id"].(string) 664 | 665 | // If response's node id is not the same with the node id in the 666 | // transaction, raise error. 667 | if trans.node.id != nil && trans.node.id.RawString() != r["id"].(string) { 668 | dht.blackList.insert(addr.IP.String(), addr.Port) 669 | dht.routingTable.RemoveByAddr(addr.String()) 670 | return 671 | } 672 | 673 | node, err := newNode(id, addr.Network(), addr.String()) 674 | if err != nil { 675 | return 676 | } 677 | 678 | switch q { 679 | case pingType: 680 | case findNodeType: 681 | if trans.data["q"].(string) != findNodeType { 682 | return 683 | } 684 | 685 | target := trans.data["a"].(map[string]interface{})["target"].(string) 686 | if findOn(dht, r, newBitmapFromString(target), findNodeType) != nil { 687 | return 688 | } 689 | case getPeersType: 690 | if err := ParseKey(r, "token", "string"); err != nil { 691 | return 692 | } 693 | 694 | token := r["token"].(string) 695 | infoHash := a["info_hash"].(string) 696 | 697 | if err := ParseKey(r, "values", "list"); err == nil { 698 | values := r["values"].([]interface{}) 699 | for _, v := range values { 700 | p, err := newPeerFromCompactIPPortInfo(v.(string), token) 701 | if err != nil { 702 | continue 703 | } 704 | dht.peersManager.Insert(infoHash, p) 705 | if dht.OnGetPeersResponse != nil { 706 | dht.OnGetPeersResponse(infoHash, p) 707 | } 708 | } 709 | } else if findOn( 710 | dht, r, newBitmapFromString(infoHash), getPeersType) != nil { 711 | return 712 | } 713 | case announcePeerType: 714 | default: 715 | return 716 | } 717 | 718 | // inform transManager to delete transaction. 719 | trans.response <- struct{}{} 720 | 721 | dht.blackList.delete(addr.IP.String(), addr.Port) 722 | dht.routingTable.Insert(node) 723 | 724 | return true 725 | } 726 | 727 | // handleError handles errors received from udp. 728 | func handleError(dht *DHT, addr *net.UDPAddr, 729 | response map[string]interface{}) (success bool) { 730 | 731 | if err := ParseKey(response, "e", "list"); err != nil { 732 | return 733 | } 734 | 735 | if e := response["e"].([]interface{}); len(e) != 2 { 736 | return 737 | } 738 | 739 | if trans := dht.transactionManager.filterOne( 740 | response["t"].(string), addr); trans != nil { 741 | 742 | trans.response <- struct{}{} 743 | } 744 | 745 | return true 746 | } 747 | 748 | var handlers = map[string]func(*DHT, *net.UDPAddr, map[string]interface{}) bool{ 749 | "q": handleRequest, 750 | "r": handleResponse, 751 | "e": handleError, 752 | } 753 | 754 | // handle handles packets received from udp. 755 | func handle(dht *DHT, pkt packet) { 756 | if len(dht.workerTokens) == dht.PacketWorkerLimit { 757 | return 758 | } 759 | 760 | dht.workerTokens <- struct{}{} 761 | 762 | go func() { 763 | defer func() { 764 | <-dht.workerTokens 765 | }() 766 | 767 | if dht.blackList.in(pkt.raddr.IP.String(), pkt.raddr.Port) { 768 | return 769 | } 770 | 771 | data, err := Decode(pkt.data) 772 | if err != nil { 773 | return 774 | } 775 | 776 | response, err := parseMessage(data) 777 | if err != nil { 778 | return 779 | } 780 | 781 | if f, ok := handlers[response["y"].(string)]; ok { 782 | f(dht, pkt.raddr, response) 783 | } 784 | }() 785 | } 786 | -------------------------------------------------------------------------------- /peerwire.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "encoding/binary" 7 | "errors" 8 | "io" 9 | "io/ioutil" 10 | "net" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | const ( 16 | // REQUEST represents request message type 17 | REQUEST = iota 18 | // DATA represents data message type 19 | DATA 20 | // REJECT represents reject message type 21 | REJECT 22 | ) 23 | 24 | const ( 25 | // BLOCK is 2 ^ 14 26 | BLOCK = 16384 27 | // MaxMetadataSize represents the max medata it can accept 28 | MaxMetadataSize = BLOCK * 1000 29 | // EXTENDED represents it is a extended message 30 | EXTENDED = 20 31 | // HANDSHAKE represents handshake bit 32 | HANDSHAKE = 0 33 | ) 34 | 35 | var handshakePrefix = []byte{ 36 | 19, 66, 105, 116, 84, 111, 114, 114, 101, 110, 116, 32, 112, 114, 37 | 111, 116, 111, 99, 111, 108, 0, 0, 0, 0, 0, 16, 0, 1, 38 | } 39 | 40 | // read reads size-length bytes from conn to data. 41 | func read(conn *net.TCPConn, size int, data *bytes.Buffer) error { 42 | conn.SetReadDeadline(time.Now().Add(time.Second * 15)) 43 | 44 | n, err := io.CopyN(data, conn, int64(size)) 45 | if err != nil || n != int64(size) { 46 | return errors.New("read error") 47 | } 48 | return nil 49 | } 50 | 51 | // readMessage gets a message from the tcp connection. 52 | func readMessage(conn *net.TCPConn, data *bytes.Buffer) ( 53 | length int, err error) { 54 | 55 | if err = read(conn, 4, data); err != nil { 56 | return 57 | } 58 | 59 | length = int(bytes2int(data.Next(4))) 60 | if length == 0 { 61 | return 62 | } 63 | 64 | if err = read(conn, length, data); err != nil { 65 | return 66 | } 67 | return 68 | } 69 | 70 | // sendMessage sends data to the connection. 71 | func sendMessage(conn *net.TCPConn, data []byte) error { 72 | length := int32(len(data)) 73 | 74 | buffer := bytes.NewBuffer(nil) 75 | binary.Write(buffer, binary.BigEndian, length) 76 | 77 | conn.SetWriteDeadline(time.Now().Add(time.Second * 10)) 78 | _, err := conn.Write(append(buffer.Bytes(), data...)) 79 | return err 80 | } 81 | 82 | // sendHandshake sends handshake message to conn. 83 | func sendHandshake(conn *net.TCPConn, infoHash, peerID []byte) error { 84 | data := make([]byte, 68) 85 | copy(data[:28], handshakePrefix) 86 | copy(data[28:48], infoHash) 87 | copy(data[48:], peerID) 88 | 89 | conn.SetWriteDeadline(time.Now().Add(time.Second * 10)) 90 | _, err := conn.Write(data) 91 | return err 92 | } 93 | 94 | // onHandshake handles the handshake response. 95 | func onHandshake(data []byte) (err error) { 96 | if !(bytes.Equal(handshakePrefix[:20], data[:20]) && data[25]&0x10 != 0) { 97 | err = errors.New("invalid handshake response") 98 | } 99 | return 100 | } 101 | 102 | // sendExtHandshake requests for the ut_metadata and metadata_size. 103 | func sendExtHandshake(conn *net.TCPConn) error { 104 | data := append( 105 | []byte{EXTENDED, HANDSHAKE}, 106 | Encode(map[string]interface{}{ 107 | "m": map[string]interface{}{"ut_metadata": 1}, 108 | })..., 109 | ) 110 | 111 | return sendMessage(conn, data) 112 | } 113 | 114 | // getUTMetaSize returns the ut_metadata and metadata_size. 115 | func getUTMetaSize(data []byte) ( 116 | utMetadata int, metadataSize int, err error) { 117 | 118 | v, err := Decode(data) 119 | if err != nil { 120 | return 121 | } 122 | 123 | dict, ok := v.(map[string]interface{}) 124 | if !ok { 125 | err = errors.New("invalid dict") 126 | return 127 | } 128 | 129 | if err = ParseKeys( 130 | dict, [][]string{{"metadata_size", "int"}, {"m", "map"}}); err != nil { 131 | return 132 | } 133 | 134 | m := dict["m"].(map[string]interface{}) 135 | if err = ParseKey(m, "ut_metadata", "int"); err != nil { 136 | return 137 | } 138 | 139 | utMetadata = m["ut_metadata"].(int) 140 | metadataSize = dict["metadata_size"].(int) 141 | 142 | if metadataSize > MaxMetadataSize { 143 | err = errors.New("metadata_size too long") 144 | } 145 | return 146 | } 147 | 148 | // Request represents the request context. 149 | type Request struct { 150 | InfoHash []byte 151 | IP string 152 | Port int 153 | } 154 | 155 | // Response contains the request context and the metadata info. 156 | type Response struct { 157 | Request 158 | MetadataInfo []byte 159 | } 160 | 161 | // Wire represents the wire protocol. 162 | type Wire struct { 163 | blackList *blackList 164 | queue *syncedMap 165 | requests chan Request 166 | responses chan Response 167 | workerTokens chan struct{} 168 | } 169 | 170 | // NewWire returns a Wire pointer. 171 | // - blackListSize: the blacklist size 172 | // - requestQueueSize: the max requests it can buffers 173 | // - workerQueueSize: the max goroutine downloading workers 174 | func NewWire(blackListSize, requestQueueSize, workerQueueSize int) *Wire { 175 | return &Wire{ 176 | blackList: newBlackList(blackListSize), 177 | queue: newSyncedMap(), 178 | requests: make(chan Request, requestQueueSize), 179 | responses: make(chan Response, 1024), 180 | workerTokens: make(chan struct{}, workerQueueSize), 181 | } 182 | } 183 | 184 | // Request pushes the request to the queue. 185 | func (wire *Wire) Request(infoHash []byte, ip string, port int) { 186 | wire.requests <- Request{InfoHash: infoHash, IP: ip, Port: port} 187 | } 188 | 189 | // Response returns a chan of Response. 190 | func (wire *Wire) Response() <-chan Response { 191 | return wire.responses 192 | } 193 | 194 | // isDone returns whether the wire get all pieces of the metadata info. 195 | func (wire *Wire) isDone(pieces [][]byte) bool { 196 | for _, piece := range pieces { 197 | if len(piece) == 0 { 198 | return false 199 | } 200 | } 201 | return true 202 | } 203 | 204 | func (wire *Wire) requestPieces( 205 | conn *net.TCPConn, utMetadata int, metadataSize int, piecesNum int) { 206 | 207 | buffer := make([]byte, 1024) 208 | for i := 0; i < piecesNum; i++ { 209 | buffer[0] = EXTENDED 210 | buffer[1] = byte(utMetadata) 211 | 212 | msg := Encode(map[string]interface{}{ 213 | "msg_type": REQUEST, 214 | "piece": i, 215 | }) 216 | 217 | length := len(msg) + 2 218 | copy(buffer[2:length], msg) 219 | 220 | sendMessage(conn, buffer[:length]) 221 | } 222 | buffer = nil 223 | } 224 | 225 | // fetchMetadata fetchs medata info accroding to infohash from dht. 226 | func (wire *Wire) fetchMetadata(r Request) { 227 | var ( 228 | length int 229 | msgType byte 230 | piecesNum int 231 | pieces [][]byte 232 | utMetadata int 233 | metadataSize int 234 | ) 235 | 236 | defer func() { 237 | pieces = nil 238 | recover() 239 | }() 240 | 241 | infoHash := r.InfoHash 242 | address := genAddress(r.IP, r.Port) 243 | 244 | dial, err := net.DialTimeout("tcp", address, time.Second*15) 245 | if err != nil { 246 | wire.blackList.insert(r.IP, r.Port) 247 | return 248 | } 249 | conn := dial.(*net.TCPConn) 250 | conn.SetLinger(0) 251 | defer conn.Close() 252 | 253 | data := bytes.NewBuffer(nil) 254 | data.Grow(BLOCK) 255 | 256 | if sendHandshake(conn, infoHash, []byte(randomString(20))) != nil || 257 | read(conn, 68, data) != nil || 258 | onHandshake(data.Next(68)) != nil || 259 | sendExtHandshake(conn) != nil { 260 | return 261 | } 262 | 263 | for { 264 | length, err = readMessage(conn, data) 265 | if err != nil { 266 | return 267 | } 268 | 269 | if length == 0 { 270 | continue 271 | } 272 | 273 | msgType, err = data.ReadByte() 274 | if err != nil { 275 | return 276 | } 277 | 278 | switch msgType { 279 | case EXTENDED: 280 | extendedID, err := data.ReadByte() 281 | if err != nil { 282 | return 283 | } 284 | 285 | payload, err := ioutil.ReadAll(data) 286 | if err != nil { 287 | return 288 | } 289 | 290 | if extendedID == 0 { 291 | if pieces != nil { 292 | return 293 | } 294 | 295 | utMetadata, metadataSize, err = getUTMetaSize(payload) 296 | if err != nil { 297 | return 298 | } 299 | 300 | piecesNum = metadataSize / BLOCK 301 | if metadataSize%BLOCK != 0 { 302 | piecesNum++ 303 | } 304 | 305 | pieces = make([][]byte, piecesNum) 306 | go wire.requestPieces(conn, utMetadata, metadataSize, piecesNum) 307 | 308 | continue 309 | } 310 | 311 | if pieces == nil { 312 | return 313 | } 314 | 315 | d, index, err := DecodeDict(payload, 0) 316 | if err != nil { 317 | return 318 | } 319 | dict := d.(map[string]interface{}) 320 | 321 | if err = ParseKeys(dict, [][]string{ 322 | {"msg_type", "int"}, 323 | {"piece", "int"}}); err != nil { 324 | return 325 | } 326 | 327 | if dict["msg_type"].(int) != DATA { 328 | continue 329 | } 330 | 331 | piece := dict["piece"].(int) 332 | pieceLen := length - 2 - index 333 | 334 | if (piece != piecesNum-1 && pieceLen != BLOCK) || 335 | (piece == piecesNum-1 && pieceLen != metadataSize%BLOCK) { 336 | return 337 | } 338 | 339 | pieces[piece] = payload[index:] 340 | 341 | if wire.isDone(pieces) { 342 | metadataInfo := bytes.Join(pieces, nil) 343 | 344 | info := sha1.Sum(metadataInfo) 345 | if !bytes.Equal(infoHash, info[:]) { 346 | return 347 | } 348 | 349 | wire.responses <- Response{ 350 | Request: r, 351 | MetadataInfo: metadataInfo, 352 | } 353 | return 354 | } 355 | default: 356 | data.Reset() 357 | } 358 | } 359 | } 360 | 361 | // Run starts the peer wire protocol. 362 | func (wire *Wire) Run() { 363 | go wire.blackList.clear() 364 | 365 | for r := range wire.requests { 366 | wire.workerTokens <- struct{}{} 367 | 368 | go func(r Request) { 369 | defer func() { 370 | <-wire.workerTokens 371 | }() 372 | 373 | key := strings.Join([]string{ 374 | string(r.InfoHash), genAddress(r.IP, r.Port), 375 | }, ":") 376 | 377 | if len(r.InfoHash) != 20 || wire.blackList.in(r.IP, r.Port) || 378 | wire.queue.Has(key) { 379 | return 380 | } 381 | 382 | wire.fetchMetadata(r) 383 | }(r) 384 | } 385 | } 386 | -------------------------------------------------------------------------------- /routingtable.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "container/heap" 5 | "errors" 6 | "net" 7 | "strings" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // maxPrefixLength is the length of DHT node. 13 | const maxPrefixLength = 160 14 | 15 | // node represents a DHT node. 16 | type node struct { 17 | id *bitmap 18 | addr *net.UDPAddr 19 | lastActiveTime time.Time 20 | } 21 | 22 | // newNode returns a node pointer. 23 | func newNode(id, network, address string) (*node, error) { 24 | if len(id) != 20 { 25 | return nil, errors.New("node id should be a 20-length string") 26 | } 27 | 28 | addr, err := net.ResolveUDPAddr(network, address) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return &node{newBitmapFromString(id), addr, time.Now()}, nil 34 | } 35 | 36 | // newNodeFromCompactInfo parses compactNodeInfo and returns a node pointer. 37 | func newNodeFromCompactInfo( 38 | compactNodeInfo string, network string) (*node, error) { 39 | 40 | if len(compactNodeInfo) != 26 { 41 | return nil, errors.New("compactNodeInfo should be a 26-length string") 42 | } 43 | 44 | id := compactNodeInfo[:20] 45 | ip, port, _ := decodeCompactIPPortInfo(compactNodeInfo[20:]) 46 | 47 | return newNode(id, network, genAddress(ip.String(), port)) 48 | } 49 | 50 | // CompactIPPortInfo returns "Compact IP-address/port info". 51 | // See http://www.bittorrent.org/beps/bep_0005.html. 52 | func (node *node) CompactIPPortInfo() string { 53 | info, _ := encodeCompactIPPortInfo(node.addr.IP, node.addr.Port) 54 | return info 55 | } 56 | 57 | // CompactNodeInfo returns "Compact node info". 58 | // See http://www.bittorrent.org/beps/bep_0005.html. 59 | func (node *node) CompactNodeInfo() string { 60 | return strings.Join([]string{ 61 | node.id.RawString(), node.CompactIPPortInfo(), 62 | }, "") 63 | } 64 | 65 | // Peer represents a peer contact. 66 | type Peer struct { 67 | IP net.IP 68 | Port int 69 | token string 70 | } 71 | 72 | // newPeer returns a new peer pointer. 73 | func newPeer(ip net.IP, port int, token string) *Peer { 74 | return &Peer{ 75 | IP: ip, 76 | Port: port, 77 | token: token, 78 | } 79 | } 80 | 81 | // newPeerFromCompactIPPortInfo create a peer pointer by compact ip/port info. 82 | func newPeerFromCompactIPPortInfo(compactInfo, token string) (*Peer, error) { 83 | ip, port, err := decodeCompactIPPortInfo(compactInfo) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return newPeer(ip, port, token), nil 89 | } 90 | 91 | // CompactIPPortInfo returns "Compact node info". 92 | // See http://www.bittorrent.org/beps/bep_0005.html. 93 | func (p *Peer) CompactIPPortInfo() string { 94 | info, _ := encodeCompactIPPortInfo(p.IP, p.Port) 95 | return info 96 | } 97 | 98 | // peersManager represents a proxy that manipulates peers. 99 | type peersManager struct { 100 | sync.RWMutex 101 | table *syncedMap 102 | dht *DHT 103 | } 104 | 105 | // newPeersManager returns a new peersManager. 106 | func newPeersManager(dht *DHT) *peersManager { 107 | return &peersManager{ 108 | table: newSyncedMap(), 109 | dht: dht, 110 | } 111 | } 112 | 113 | // Insert adds a peer into peersManager. 114 | func (pm *peersManager) Insert(infoHash string, peer *Peer) { 115 | pm.Lock() 116 | if _, ok := pm.table.Get(infoHash); !ok { 117 | pm.table.Set(infoHash, newKeyedDeque()) 118 | } 119 | pm.Unlock() 120 | 121 | v, _ := pm.table.Get(infoHash) 122 | queue := v.(*keyedDeque) 123 | 124 | queue.Push(peer.CompactIPPortInfo(), peer) 125 | if queue.Len() > pm.dht.K { 126 | queue.Remove(queue.Front()) 127 | } 128 | } 129 | 130 | // GetPeers returns size-length peers who announces having infoHash. 131 | func (pm *peersManager) GetPeers(infoHash string, size int) []*Peer { 132 | peers := make([]*Peer, 0, size) 133 | 134 | v, ok := pm.table.Get(infoHash) 135 | if !ok { 136 | return peers 137 | } 138 | 139 | for e := range v.(*keyedDeque).Iter() { 140 | peers = append(peers, e.Value.(*Peer)) 141 | } 142 | 143 | if len(peers) > size { 144 | peers = peers[len(peers)-size:] 145 | } 146 | return peers 147 | } 148 | 149 | // kbucket represents a k-size bucket. 150 | type kbucket struct { 151 | sync.RWMutex 152 | nodes, candidates *keyedDeque 153 | lastChanged time.Time 154 | prefix *bitmap 155 | } 156 | 157 | // newKBucket returns a new kbucket pointer. 158 | func newKBucket(prefix *bitmap) *kbucket { 159 | bucket := &kbucket{ 160 | nodes: newKeyedDeque(), 161 | candidates: newKeyedDeque(), 162 | lastChanged: time.Now(), 163 | prefix: prefix, 164 | } 165 | return bucket 166 | } 167 | 168 | // LastChanged return the last time when it changes. 169 | func (bucket *kbucket) LastChanged() time.Time { 170 | bucket.RLock() 171 | defer bucket.RUnlock() 172 | 173 | return bucket.lastChanged 174 | } 175 | 176 | // RandomChildID returns a random id that has the same prefix with bucket. 177 | func (bucket *kbucket) RandomChildID() string { 178 | prefixLen := bucket.prefix.Size / 8 179 | 180 | return strings.Join([]string{ 181 | bucket.prefix.RawString()[:prefixLen], 182 | randomString(20 - prefixLen), 183 | }, "") 184 | } 185 | 186 | // UpdateTimestamp update bucket's last changed time.. 187 | func (bucket *kbucket) UpdateTimestamp() { 188 | bucket.Lock() 189 | defer bucket.Unlock() 190 | 191 | bucket.lastChanged = time.Now() 192 | } 193 | 194 | // Insert inserts node to the bucket. It returns whether the node is new in 195 | // the bucket. 196 | func (bucket *kbucket) Insert(no *node) bool { 197 | isNew := !bucket.nodes.HasKey(no.id.RawString()) 198 | 199 | bucket.nodes.Push(no.id.RawString(), no) 200 | bucket.UpdateTimestamp() 201 | 202 | return isNew 203 | } 204 | 205 | // Replace removes node, then put bucket.candidates.Back() to the right 206 | // place of bucket.nodes. 207 | func (bucket *kbucket) Replace(no *node) { 208 | bucket.nodes.Delete(no.id.RawString()) 209 | bucket.UpdateTimestamp() 210 | 211 | if bucket.candidates.Len() == 0 { 212 | return 213 | } 214 | 215 | no = bucket.candidates.Remove(bucket.candidates.Back()).(*node) 216 | 217 | inserted := false 218 | for e := range bucket.nodes.Iter() { 219 | if e.Value.(*node).lastActiveTime.After( 220 | no.lastActiveTime) && !inserted { 221 | 222 | bucket.nodes.InsertBefore(no, e) 223 | inserted = true 224 | } 225 | } 226 | 227 | if !inserted { 228 | bucket.nodes.PushBack(no) 229 | } 230 | } 231 | 232 | // Fresh pings the expired nodes in the bucket. 233 | func (bucket *kbucket) Fresh(dht *DHT) { 234 | for e := range bucket.nodes.Iter() { 235 | no := e.Value.(*node) 236 | if time.Since(no.lastActiveTime) > dht.NodeExpriedAfter { 237 | dht.transactionManager.ping(no) 238 | } 239 | } 240 | } 241 | 242 | // routingTableNode represents routing table tree node. 243 | type routingTableNode struct { 244 | sync.RWMutex 245 | children []*routingTableNode 246 | bucket *kbucket 247 | } 248 | 249 | // newRoutingTableNode returns a new routingTableNode pointer. 250 | func newRoutingTableNode(prefix *bitmap) *routingTableNode { 251 | return &routingTableNode{ 252 | children: make([]*routingTableNode, 2), 253 | bucket: newKBucket(prefix), 254 | } 255 | } 256 | 257 | // Child returns routingTableNode's left or right child. 258 | func (tableNode *routingTableNode) Child(index int) *routingTableNode { 259 | if index >= 2 { 260 | return nil 261 | } 262 | 263 | tableNode.RLock() 264 | defer tableNode.RUnlock() 265 | 266 | return tableNode.children[index] 267 | } 268 | 269 | // SetChild sets routingTableNode's left or right child. When index is 0, it's 270 | // the left child, if 1, it's the right child. 271 | func (tableNode *routingTableNode) SetChild(index int, c *routingTableNode) { 272 | tableNode.Lock() 273 | defer tableNode.Unlock() 274 | 275 | tableNode.children[index] = c 276 | } 277 | 278 | // KBucket returns the bucket routingTableNode holds. 279 | func (tableNode *routingTableNode) KBucket() *kbucket { 280 | tableNode.RLock() 281 | defer tableNode.RUnlock() 282 | 283 | return tableNode.bucket 284 | } 285 | 286 | // SetKBucket sets the bucket. 287 | func (tableNode *routingTableNode) SetKBucket(bucket *kbucket) { 288 | tableNode.Lock() 289 | defer tableNode.Unlock() 290 | 291 | tableNode.bucket = bucket 292 | } 293 | 294 | // Split splits current routingTableNode and sets it's two children. 295 | func (tableNode *routingTableNode) Split() { 296 | prefixLen := tableNode.KBucket().prefix.Size 297 | 298 | if prefixLen == maxPrefixLength { 299 | return 300 | } 301 | 302 | for i := 0; i < 2; i++ { 303 | tableNode.SetChild(i, newRoutingTableNode(newBitmapFrom( 304 | tableNode.KBucket().prefix, prefixLen+1))) 305 | } 306 | 307 | tableNode.Lock() 308 | tableNode.children[1].bucket.prefix.Set(prefixLen) 309 | tableNode.Unlock() 310 | 311 | for e := range tableNode.KBucket().nodes.Iter() { 312 | nd := e.Value.(*node) 313 | tableNode.Child(nd.id.Bit(prefixLen)).KBucket().nodes.PushBack(nd) 314 | } 315 | 316 | for e := range tableNode.KBucket().candidates.Iter() { 317 | nd := e.Value.(*node) 318 | tableNode.Child(nd.id.Bit(prefixLen)).KBucket().candidates.PushBack(nd) 319 | } 320 | 321 | for i := 0; i < 2; i++ { 322 | tableNode.Child(i).KBucket().UpdateTimestamp() 323 | } 324 | } 325 | 326 | // routingTable implements the routing table in DHT protocol. 327 | type routingTable struct { 328 | *sync.RWMutex 329 | k int 330 | root *routingTableNode 331 | cachedNodes *syncedMap 332 | cachedKBuckets *keyedDeque 333 | dht *DHT 334 | clearQueue *syncedList 335 | } 336 | 337 | // newRoutingTable returns a new routingTable pointer. 338 | func newRoutingTable(k int, dht *DHT) *routingTable { 339 | root := newRoutingTableNode(newBitmap(0)) 340 | 341 | rt := &routingTable{ 342 | RWMutex: &sync.RWMutex{}, 343 | k: k, 344 | root: root, 345 | cachedNodes: newSyncedMap(), 346 | cachedKBuckets: newKeyedDeque(), 347 | dht: dht, 348 | clearQueue: newSyncedList(), 349 | } 350 | 351 | rt.cachedKBuckets.Push(root.bucket.prefix.String(), root.bucket) 352 | return rt 353 | } 354 | 355 | // Insert adds a node to routing table. It returns whether the node is new 356 | // in the routingtable. 357 | func (rt *routingTable) Insert(nd *node) bool { 358 | rt.Lock() 359 | defer rt.Unlock() 360 | 361 | if rt.dht.blackList.in(nd.addr.IP.String(), nd.addr.Port) || 362 | rt.cachedNodes.Len() >= rt.dht.MaxNodes { 363 | return false 364 | } 365 | 366 | var ( 367 | next *routingTableNode 368 | bucket *kbucket 369 | ) 370 | root := rt.root 371 | 372 | for prefixLen := 1; prefixLen <= maxPrefixLength; prefixLen++ { 373 | next = root.Child(nd.id.Bit(prefixLen - 1)) 374 | 375 | if next != nil { 376 | // If next is not the leaf. 377 | root = next 378 | } else if root.KBucket().nodes.Len() < rt.k || 379 | root.KBucket().nodes.HasKey(nd.id.RawString()) { 380 | 381 | bucket = root.KBucket() 382 | isNew := bucket.Insert(nd) 383 | 384 | rt.cachedNodes.Set(nd.addr.String(), nd) 385 | rt.cachedKBuckets.Push(bucket.prefix.String(), bucket) 386 | 387 | return isNew 388 | } else if root.KBucket().prefix.Compare(nd.id, prefixLen-1) == 0 { 389 | // If node has the same prefix with bucket, split it. 390 | 391 | root.Split() 392 | 393 | rt.cachedKBuckets.Delete(root.KBucket().prefix.String()) 394 | root.SetKBucket(nil) 395 | 396 | for i := 0; i < 2; i++ { 397 | bucket = root.Child(i).KBucket() 398 | rt.cachedKBuckets.Push(bucket.prefix.String(), bucket) 399 | } 400 | 401 | root = root.Child(nd.id.Bit(prefixLen - 1)) 402 | } else { 403 | // Finally, store node as a candidate and fresh the bucket. 404 | root.KBucket().candidates.PushBack(nd) 405 | if root.KBucket().candidates.Len() > rt.k { 406 | root.KBucket().candidates.Remove( 407 | root.KBucket().candidates.Front()) 408 | } 409 | 410 | go root.KBucket().Fresh(rt.dht) 411 | return false 412 | } 413 | } 414 | return false 415 | } 416 | 417 | // GetNeighbors returns the size-length nodes closest to id. 418 | func (rt *routingTable) GetNeighbors(id *bitmap, size int) []*node { 419 | rt.RLock() 420 | nodes := make([]interface{}, 0, rt.cachedNodes.Len()) 421 | for item := range rt.cachedNodes.Iter() { 422 | nodes = append(nodes, item.val.(*node)) 423 | } 424 | rt.RUnlock() 425 | 426 | neighbors := getTopK(nodes, id, size) 427 | result := make([]*node, len(neighbors)) 428 | 429 | for i, nd := range neighbors { 430 | result[i] = nd.(*node) 431 | } 432 | return result 433 | } 434 | 435 | // GetNeighborIds return the size-length compact node info closest to id. 436 | func (rt *routingTable) GetNeighborCompactInfos(id *bitmap, size int) []string { 437 | neighbors := rt.GetNeighbors(id, size) 438 | infos := make([]string, len(neighbors)) 439 | 440 | for i, no := range neighbors { 441 | infos[i] = no.CompactNodeInfo() 442 | } 443 | 444 | return infos 445 | } 446 | 447 | // GetNodeKBucktById returns node whose id is `id` and the bucket it 448 | // belongs to. 449 | func (rt *routingTable) GetNodeKBucktByID(id *bitmap) ( 450 | nd *node, bucket *kbucket) { 451 | 452 | rt.RLock() 453 | defer rt.RUnlock() 454 | 455 | var next *routingTableNode 456 | root := rt.root 457 | 458 | for prefixLen := 1; prefixLen <= maxPrefixLength; prefixLen++ { 459 | next = root.Child(id.Bit(prefixLen - 1)) 460 | if next == nil { 461 | v, ok := root.KBucket().nodes.Get(id.RawString()) 462 | if !ok { 463 | return 464 | } 465 | nd, bucket = v.Value.(*node), root.KBucket() 466 | return 467 | } 468 | root = next 469 | } 470 | return 471 | } 472 | 473 | // GetNodeByAddress finds node by address. 474 | func (rt *routingTable) GetNodeByAddress(address string) (no *node, ok bool) { 475 | rt.RLock() 476 | defer rt.RUnlock() 477 | 478 | v, ok := rt.cachedNodes.Get(address) 479 | if ok { 480 | no = v.(*node) 481 | } 482 | return 483 | } 484 | 485 | // Remove deletes the node whose id is `id`. 486 | func (rt *routingTable) Remove(id *bitmap) { 487 | if nd, bucket := rt.GetNodeKBucktByID(id); nd != nil { 488 | bucket.Replace(nd) 489 | rt.cachedNodes.Delete(nd.addr.String()) 490 | rt.cachedKBuckets.Push(bucket.prefix.String(), bucket) 491 | } 492 | } 493 | 494 | // Remove deletes the node whose address is `ip:port`. 495 | func (rt *routingTable) RemoveByAddr(address string) { 496 | v, ok := rt.cachedNodes.Get(address) 497 | if ok { 498 | rt.Remove(v.(*node).id) 499 | } 500 | } 501 | 502 | // Fresh sends findNode to all nodes in the expired nodes. 503 | func (rt *routingTable) Fresh() { 504 | now := time.Now() 505 | 506 | for e := range rt.cachedKBuckets.Iter() { 507 | bucket := e.Value.(*kbucket) 508 | if now.Sub(bucket.LastChanged()) < rt.dht.KBucketExpiredAfter || 509 | bucket.nodes.Len() == 0 { 510 | continue 511 | } 512 | 513 | i := 0 514 | for e := range bucket.nodes.Iter() { 515 | if i < rt.dht.RefreshNodeNum { 516 | no := e.Value.(*node) 517 | rt.dht.transactionManager.findNode(no, bucket.RandomChildID()) 518 | rt.clearQueue.PushBack(no) 519 | } 520 | i++ 521 | } 522 | } 523 | 524 | if rt.dht.IsCrawlMode() { 525 | for e := range rt.clearQueue.Iter() { 526 | rt.Remove(e.Value.(*node).id) 527 | } 528 | } 529 | 530 | rt.clearQueue.Clear() 531 | } 532 | 533 | // Len returns the number of nodes in table. 534 | func (rt *routingTable) Len() int { 535 | rt.RLock() 536 | defer rt.RUnlock() 537 | 538 | return rt.cachedNodes.Len() 539 | } 540 | 541 | // Implementation of heap with heap.Interface. 542 | type heapItem struct { 543 | distance *bitmap 544 | value interface{} 545 | } 546 | 547 | type topKHeap []*heapItem 548 | 549 | func (kHeap topKHeap) Len() int { 550 | return len(kHeap) 551 | } 552 | 553 | func (kHeap topKHeap) Less(i, j int) bool { 554 | return kHeap[i].distance.Compare(kHeap[j].distance, maxPrefixLength) == 1 555 | } 556 | 557 | func (kHeap topKHeap) Swap(i, j int) { 558 | kHeap[i], kHeap[j] = kHeap[j], kHeap[i] 559 | } 560 | 561 | func (kHeap *topKHeap) Push(x interface{}) { 562 | *kHeap = append(*kHeap, x.(*heapItem)) 563 | } 564 | 565 | func (kHeap *topKHeap) Pop() interface{} { 566 | n := len(*kHeap) 567 | x := (*kHeap)[n-1] 568 | *kHeap = (*kHeap)[:n-1] 569 | return x 570 | } 571 | 572 | // getTopK solves the top-k problem with heap. It's time complexity is 573 | // O(n*log(k)). When n is large, time complexity will be too high, need to be 574 | // optimized. 575 | func getTopK(queue []interface{}, id *bitmap, k int) []interface{} { 576 | topkHeap := make(topKHeap, 0, k+1) 577 | 578 | for _, value := range queue { 579 | node := value.(*node) 580 | distance := id.Xor(node.id) 581 | if topkHeap.Len() == k { 582 | var last = topkHeap[topkHeap.Len() - 1] 583 | if last.distance.Compare(distance, maxPrefixLength) == 1 { 584 | item := &heapItem{ 585 | distance, 586 | value, 587 | } 588 | heap.Push(&topkHeap, item) 589 | heap.Pop(&topkHeap) 590 | } 591 | } else { 592 | item := &heapItem{ 593 | distance, 594 | value, 595 | } 596 | heap.Push(&topkHeap, item) 597 | } 598 | } 599 | 600 | tops := make([]interface{}, topkHeap.Len()) 601 | for i := len(tops) - 1; i >= 0; i-- { 602 | tops[i] = heap.Pop(&topkHeap).(*heapItem).value 603 | } 604 | 605 | return tops 606 | } 607 | -------------------------------------------------------------------------------- /sample/getpeers/getpeers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "time" 7 | 8 | "github.com/shiyanhui/dht" 9 | ) 10 | 11 | func main() { 12 | d := dht.New(nil) 13 | d.OnGetPeersResponse = func(infoHash string, peer *dht.Peer) { 14 | fmt.Printf("GOT PEER: <%s:%d>\n", peer.IP, peer.Port) 15 | } 16 | 17 | go func() { 18 | for { 19 | // ubuntu-14.04.2-desktop-amd64.iso 20 | err := d.GetPeers("546cf15f724d19c4319cc17b179d7e035f89c1f4") 21 | if err != nil && err != dht.ErrNotReady { 22 | log.Fatal(err) 23 | } 24 | 25 | if err == dht.ErrNotReady { 26 | time.Sleep(time.Second * 1) 27 | continue 28 | } 29 | 30 | break 31 | } 32 | }() 33 | 34 | d.Run() 35 | } 36 | -------------------------------------------------------------------------------- /sample/spider/spider: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shiyanhui/dht/5a20f319926383187ead6e7eb2509b08599a9ee9/sample/spider/spider -------------------------------------------------------------------------------- /sample/spider/spider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/hex" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/shiyanhui/dht" 8 | "net/http" 9 | _ "net/http/pprof" 10 | ) 11 | 12 | type file struct { 13 | Path []interface{} `json:"path"` 14 | Length int `json:"length"` 15 | } 16 | 17 | type bitTorrent struct { 18 | InfoHash string `json:"infohash"` 19 | Name string `json:"name"` 20 | Files []file `json:"files,omitempty"` 21 | Length int `json:"length,omitempty"` 22 | } 23 | 24 | func main() { 25 | go func() { 26 | http.ListenAndServe(":6060", nil) 27 | }() 28 | 29 | w := dht.NewWire(65536, 1024, 256) 30 | go func() { 31 | for resp := range w.Response() { 32 | metadata, err := dht.Decode(resp.MetadataInfo) 33 | if err != nil { 34 | continue 35 | } 36 | info := metadata.(map[string]interface{}) 37 | 38 | if _, ok := info["name"]; !ok { 39 | continue 40 | } 41 | 42 | bt := bitTorrent{ 43 | InfoHash: hex.EncodeToString(resp.InfoHash), 44 | Name: info["name"].(string), 45 | } 46 | 47 | if v, ok := info["files"]; ok { 48 | files := v.([]interface{}) 49 | bt.Files = make([]file, len(files)) 50 | 51 | for i, item := range files { 52 | f := item.(map[string]interface{}) 53 | bt.Files[i] = file{ 54 | Path: f["path"].([]interface{}), 55 | Length: f["length"].(int), 56 | } 57 | } 58 | } else if _, ok := info["length"]; ok { 59 | bt.Length = info["length"].(int) 60 | } 61 | 62 | data, err := json.Marshal(bt) 63 | if err == nil { 64 | fmt.Printf("%s\n\n", data) 65 | } 66 | } 67 | }() 68 | go w.Run() 69 | 70 | config := dht.NewCrawlConfig() 71 | config.OnAnnouncePeer = func(infoHash, ip string, port int) { 72 | w.Request([]byte(infoHash), ip, port) 73 | } 74 | d := dht.New(config) 75 | 76 | d.Run() 77 | } 78 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "crypto/rand" 5 | "errors" 6 | "io/ioutil" 7 | "net" 8 | "net/http" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // randomString generates a size-length string randomly. 15 | func randomString(size int) string { 16 | buff := make([]byte, size) 17 | rand.Read(buff) 18 | return string(buff) 19 | } 20 | 21 | // bytes2int returns the int value it represents. 22 | func bytes2int(data []byte) uint64 { 23 | n, val := len(data), uint64(0) 24 | if n > 8 { 25 | panic("data too long") 26 | } 27 | 28 | for i, b := range data { 29 | val += uint64(b) << uint64((n-i-1)*8) 30 | } 31 | return val 32 | } 33 | 34 | // int2bytes returns the byte array it represents. 35 | func int2bytes(val uint64) []byte { 36 | data, j := make([]byte, 8), -1 37 | for i := 0; i < 8; i++ { 38 | shift := uint64((7 - i) * 8) 39 | data[i] = byte((val & (0xff << shift)) >> shift) 40 | 41 | if j == -1 && data[i] != 0 { 42 | j = i 43 | } 44 | } 45 | 46 | if j != -1 { 47 | return data[j:] 48 | } 49 | return data[:1] 50 | } 51 | 52 | // decodeCompactIPPortInfo decodes compactIP-address/port info in BitTorrent 53 | // DHT Protocol. It returns the ip and port number. 54 | func decodeCompactIPPortInfo(info string) (ip net.IP, port int, err error) { 55 | if len(info) != 6 { 56 | err = errors.New("compact info should be 6-length long") 57 | return 58 | } 59 | 60 | ip = net.IPv4(info[0], info[1], info[2], info[3]) 61 | port = int((uint16(info[4]) << 8) | uint16(info[5])) 62 | return 63 | } 64 | 65 | // encodeCompactIPPortInfo encodes an ip and a port number to 66 | // compactIP-address/port info. 67 | func encodeCompactIPPortInfo(ip net.IP, port int) (info string, err error) { 68 | if port > 65535 || port < 0 { 69 | err = errors.New( 70 | "port should be no greater than 65535 and no less than 0") 71 | return 72 | } 73 | 74 | p := int2bytes(uint64(port)) 75 | if len(p) < 2 { 76 | p = append(p, p[0]) 77 | p[0] = 0 78 | } 79 | 80 | info = string(append(ip, p...)) 81 | return 82 | } 83 | 84 | // getLocalIPs returns local ips. 85 | func getLocalIPs() (ips []string) { 86 | ips = make([]string, 0, 6) 87 | 88 | addrs, err := net.InterfaceAddrs() 89 | if err != nil { 90 | return 91 | } 92 | 93 | for _, addr := range addrs { 94 | ip, _, err := net.ParseCIDR(addr.String()) 95 | if err != nil { 96 | continue 97 | } 98 | ips = append(ips, ip.String()) 99 | } 100 | return 101 | } 102 | 103 | // getRemoteIP returns the wlan ip. 104 | func getRemoteIP() (ip string, err error) { 105 | client := &http.Client{ 106 | Timeout: time.Second * 30, 107 | } 108 | 109 | req, err := http.NewRequest("GET", "http://ifconfig.me", nil) 110 | if err != nil { 111 | return 112 | } 113 | 114 | req.Header.Set("User-Agent", "curl") 115 | res, err := client.Do(req) 116 | if err != nil { 117 | return 118 | } 119 | 120 | defer res.Body.Close() 121 | 122 | data, err := ioutil.ReadAll(res.Body) 123 | if err != nil { 124 | return 125 | } 126 | ip = string(data) 127 | 128 | return 129 | } 130 | 131 | // genAddress returns a ip:port address. 132 | func genAddress(ip string, port int) string { 133 | return strings.Join([]string{ip, strconv.Itoa(port)}, ":") 134 | } 135 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestInt2Bytes(t *testing.T) { 8 | cases := []struct { 9 | in uint64 10 | out []byte 11 | }{ 12 | {0, []byte{0}}, 13 | {1, []byte{1}}, 14 | {256, []byte{1, 0}}, 15 | {22129, []byte{86, 113}}, 16 | } 17 | 18 | for _, c := range cases { 19 | r := int2bytes(c.in) 20 | if len(r) != len(c.out) { 21 | t.Fail() 22 | } 23 | 24 | for i, v := range r { 25 | if v != c.out[i] { 26 | t.Fail() 27 | } 28 | } 29 | } 30 | } 31 | 32 | func TestBytes2Int(t *testing.T) { 33 | cases := []struct { 34 | in []byte 35 | out uint64 36 | }{ 37 | {[]byte{0}, 0}, 38 | {[]byte{1}, 1}, 39 | {[]byte{1, 0}, 256}, 40 | {[]byte{86, 113}, 22129}, 41 | } 42 | 43 | for _, c := range cases { 44 | if bytes2int(c.in) != c.out { 45 | t.Fail() 46 | } 47 | } 48 | } 49 | 50 | func TestDecodeCompactIPPortInfo(t *testing.T) { 51 | cases := []struct { 52 | in string 53 | out struct { 54 | ip string 55 | port int 56 | } 57 | }{ 58 | {"123456", struct { 59 | ip string 60 | port int 61 | }{"49.50.51.52", 13622}}, 62 | {"abcdef", struct { 63 | ip string 64 | port int 65 | }{"97.98.99.100", 25958}}, 66 | } 67 | 68 | for _, item := range cases { 69 | ip, port, err := decodeCompactIPPortInfo(item.in) 70 | if err != nil || ip.String() != item.out.ip || port != item.out.port { 71 | t.Fail() 72 | } 73 | } 74 | } 75 | 76 | func TestEncodeCompactIPPortInfo(t *testing.T) { 77 | cases := []struct { 78 | in struct { 79 | ip []byte 80 | port int 81 | } 82 | out string 83 | }{ 84 | {struct { 85 | ip []byte 86 | port int 87 | }{[]byte{49, 50, 51, 52}, 13622}, "123456"}, 88 | {struct { 89 | ip []byte 90 | port int 91 | }{[]byte{97, 98, 99, 100}, 25958}, "abcdef"}, 92 | } 93 | 94 | for _, item := range cases { 95 | info, err := encodeCompactIPPortInfo(item.in.ip, item.in.port) 96 | if err != nil || info != item.out { 97 | t.Fail() 98 | } 99 | } 100 | } 101 | --------------------------------------------------------------------------------