├── web ├── README.md ├── .env ├── public │ ├── favicon.ico │ ├── logo192.png │ ├── logo512.png │ ├── robots.txt │ ├── manifest.json │ └── index.html ├── nginx │ └── nginx.conf ├── .gitignore ├── Dockerfile ├── src │ ├── components │ │ ├── header.js │ │ ├── listitem.js │ │ ├── search.js │ │ ├── listbar.js │ │ ├── searchbar.js │ │ └── list.js │ ├── index.js │ └── style.css └── package.json ├── .gitignore ├── doc └── architecture.jpg ├── config └── elasticsearch.yml ├── nginx ├── .gitignore └── nginx │ ├── .gitignore │ └── site-confs │ └── default ├── crawler ├── dht │ ├── bitmap_xor.go │ ├── bitmap_xorfast.go │ ├── blacklist_test.go │ ├── LICENSE │ ├── bitmap_test.go │ ├── util_test.go │ ├── blacklist.go │ ├── util.go │ ├── container_test.go │ ├── bencode_test.go │ ├── bitmap.go │ ├── bencode.go │ ├── container.go │ ├── dht.go │ ├── peerwire.go │ ├── routingtable.go │ └── krpc.go ├── go.mod ├── Dockerfile ├── main.go └── go.sum ├── server ├── go.mod ├── Dockerfile ├── main.go └── go.sum ├── service ├── bittorrent.proto ├── bittorrent_grpc.pb.go └── bittorrent.pb.go ├── LICENSE ├── docker-compose.yml └── README.md /web/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | -------------------------------------------------------------------------------- /web/.env: -------------------------------------------------------------------------------- 1 | REACT_APP_USERNAME=web 2 | REACT_APP_PASSWORD=password -------------------------------------------------------------------------------- /doc/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Olament/gDHT/HEAD/doc/architecture.jpg -------------------------------------------------------------------------------- /web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Olament/gDHT/HEAD/web/public/favicon.ico -------------------------------------------------------------------------------- /web/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Olament/gDHT/HEAD/web/public/logo192.png -------------------------------------------------------------------------------- /web/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Olament/gDHT/HEAD/web/public/logo512.png -------------------------------------------------------------------------------- /web/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | xpack.security.enabled: true 2 | cluster.name: "docker-cluster" 3 | network.host: 0.0.0.0 4 | -------------------------------------------------------------------------------- /nginx/.gitignore: -------------------------------------------------------------------------------- 1 | crontabs 2 | donoteditthisfile.conf 3 | fail2ban 4 | keys 5 | php 6 | dns-conf 7 | etc 8 | geoip2db 9 | log 10 | www 11 | 12 | -------------------------------------------------------------------------------- /nginx/nginx/.gitignore: -------------------------------------------------------------------------------- 1 | authelia-location.conf 2 | dhparams.pem 3 | proxy-confs 4 | ssl.conf 5 | authelia-server.conf 6 | ldap.conf 7 | proxy.conf 8 | nginx.conf 9 | -------------------------------------------------------------------------------- /crawler/dht/bitmap_xor.go: -------------------------------------------------------------------------------- 1 | // +build !amd64,!386 2 | 3 | package dht 4 | 5 | func xor(dst, a, b []byte) { 6 | n := len(a) 7 | for i := 0; i < n; i++ { 8 | dst[i] = a[i] ^ b[i] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /crawler/go.mod: -------------------------------------------------------------------------------- 1 | module crawler 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/Olament/gDHT v0.0.0-20200812202725-3771e54dc061 7 | github.com/golang/protobuf v1.4.2 8 | google.golang.org/grpc v1.31.0 9 | google.golang.org/protobuf v1.23.0 10 | ) 11 | -------------------------------------------------------------------------------- /server/go.mod: -------------------------------------------------------------------------------- 1 | module server 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/Olament/gDHT v0.0.0-20200812202725-3771e54dc061 7 | github.com/go-redis/redis v6.15.9+incompatible 8 | github.com/olivere/elastic/v7 v7.0.19 9 | google.golang.org/grpc v1.31.0 10 | ) 11 | -------------------------------------------------------------------------------- /web/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 3000; 3 | 4 | location / { 5 | root /usr/share/nginx/html; 6 | index index.html index.htm; 7 | try_files $uri $uri/ /index.html; 8 | } 9 | 10 | error_page 500 502 503 504 /50x.html; 11 | 12 | location = /50x.html { 13 | root /usr/share/nginx/html; 14 | } 15 | } -------------------------------------------------------------------------------- /crawler/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang as builder 2 | 3 | ENV GO111MODULE=on 4 | 5 | WORKDIR /app 6 | 7 | COPY go.mod . 8 | COPY go.sum . 9 | 10 | RUN go mod download 11 | 12 | COPY . . 13 | 14 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build 15 | 16 | # final stage 17 | FROM scratch 18 | COPY --from=builder /app/crawler /app/ 19 | ENTRYPOINT ["/app/crawler"] -------------------------------------------------------------------------------- /server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang as builder 2 | 3 | ENV GO111MODULE=on 4 | 5 | WORKDIR /app 6 | 7 | COPY go.mod . 8 | COPY go.sum . 9 | 10 | RUN go mod download 11 | 12 | COPY . . 13 | 14 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build 15 | 16 | # final stage 17 | FROM scratch 18 | COPY --from=builder /app/server /app/ 19 | EXPOSE 50051 20 | ENTRYPOINT ["/app/server"] -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /service/bittorrent.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package service; 4 | 5 | option go_package = "github.com/Olament/gDHT"; 6 | 7 | service bitTorrent { 8 | rpc send (BitTorrent) returns (Empty) {} 9 | } 10 | 11 | message Empty { 12 | 13 | } 14 | 15 | message BitTorrent { 16 | string infohash = 1; 17 | string name = 2; 18 | repeated File files = 3; 19 | int32 length = 4; 20 | } 21 | 22 | message File { 23 | string path = 1; 24 | int32 length = 2; 25 | } -------------------------------------------------------------------------------- /web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:13.12.0-alpine as build 2 | 3 | WORKDIR /app 4 | ENV PATH /app/node_modules/.bin:$PATH 5 | 6 | COPY package.json ./ 7 | COPY package-lock.json ./ 8 | RUN npm ci --silent 9 | RUN npm install react-scripts@3.4.1 -g --silent 10 | COPY . ./ 11 | RUN npm run build 12 | 13 | # production environment 14 | FROM nginx:stable-alpine 15 | 16 | COPY --from=build /app/build /usr/share/nginx/html 17 | COPY nginx/nginx.conf /etc/nginx/conf.d/default.conf 18 | 19 | EXPOSE 3000 20 | CMD ["nginx", "-g", "daemon off;"] 21 | -------------------------------------------------------------------------------- /crawler/dht/bitmap_xorfast.go: -------------------------------------------------------------------------------- 1 | // +build amd64 386 2 | 3 | package dht 4 | 5 | import "unsafe" 6 | 7 | const wordSize = int(unsafe.Sizeof(uintptr(0))) 8 | 9 | func xor(dst, a, b []byte) { 10 | n := len(a) 11 | 12 | w := n / wordSize 13 | if w > 0 { 14 | dw := *(*[]uintptr)(unsafe.Pointer(&dst)) 15 | aw := *(*[]uintptr)(unsafe.Pointer(&a)) 16 | bw := *(*[]uintptr)(unsafe.Pointer(&b)) 17 | for i := 0; i < w; i++ { 18 | dw[i] = aw[i] ^ bw[i] 19 | } 20 | } 21 | 22 | for i := n - n%wordSize; i < n; i++ { 23 | dst[i] = a[i] ^ b[i] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /web/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "gDHT Demo", 3 | "name": "gDHT Demostration", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /web/src/components/header.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | const header = () => ( 4 |
5 | 20 |
21 | ) 22 | 23 | export default header; -------------------------------------------------------------------------------- /web/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | 4 | import './style.css'; 5 | 6 | import Search from "./components/search"; 7 | import Header from "./components/header"; 8 | 9 | ReactDOM.render( 10 | 11 |
12 |
13 |
14 |

🔍gDHT

15 |

16 | A distributed torrent search engine 17 |

18 |
19 |
20 |
21 | 22 |
23 | , 24 | document.getElementById('root') 25 | ); 26 | 27 | -------------------------------------------------------------------------------- /web/src/components/listitem.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | export default class ListItem extends React.Component { 4 | render() { 5 | return ( 6 |
7 |
8 | {this.props.name} 9 |
10 |
11 | {this.props.size} 12 |
13 |
14 | Link 15 |
16 |
17 | ) 18 | } 19 | } -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gDHT_web", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@testing-library/jest-dom": "^4.2.4", 7 | "@testing-library/react": "^9.3.2", 8 | "@testing-library/user-event": "^7.1.2", 9 | "react": "^16.13.1", 10 | "react-autocomplete": "^1.8.1", 11 | "react-autosuggest": "^10.0.2", 12 | "react-dom": "^16.13.1", 13 | "react-helmet": "^6.1.0", 14 | "react-scripts": "3.4.3" 15 | }, 16 | "scripts": { 17 | "start": "react-scripts start", 18 | "build": "react-scripts build", 19 | "test": "react-scripts test", 20 | "eject": "react-scripts eject" 21 | }, 22 | "eslintConfig": { 23 | "extends": "react-app" 24 | }, 25 | "browserslist": { 26 | "production": [ 27 | ">0.2%", 28 | "not dead", 29 | "not op_mini all" 30 | ], 31 | "development": [ 32 | "last 1 chrome version", 33 | "last 1 firefox version", 34 | "last 1 safari version" 35 | ] 36 | }, 37 | "proxy": "https://guo.sh" 38 | } 39 | -------------------------------------------------------------------------------- /crawler/dht/blacklist_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | var blacklist = newBlackList(256) 9 | 10 | func TestGenKey(t *testing.T) { 11 | cases := []struct { 12 | in struct { 13 | ip string 14 | port int 15 | } 16 | out string 17 | }{ 18 | {struct { 19 | ip string 20 | port int 21 | }{"0.0.0.0", -1}, "0.0.0.0"}, 22 | {struct { 23 | ip string 24 | port int 25 | }{"1.1.1.1", 8080}, "1.1.1.1:8080"}, 26 | } 27 | 28 | for _, c := range cases { 29 | if blacklist.genKey(c.in.ip, c.in.port) != c.out { 30 | t.Fail() 31 | } 32 | } 33 | } 34 | 35 | func TestBlackList(t *testing.T) { 36 | address := []struct { 37 | ip string 38 | port int 39 | }{ 40 | {"0.0.0.0", -1}, 41 | {"1.1.1.1", 8080}, 42 | {"2.2.2.2", 8081}, 43 | } 44 | 45 | for _, addr := range address { 46 | blacklist.insert(addr.ip, addr.port) 47 | if !blacklist.in(addr.ip, addr.port) { 48 | t.Fail() 49 | } 50 | 51 | blacklist.delete(addr.ip, addr.port) 52 | if blacklist.in(addr.ip, addr.port) { 53 | fmt.Println(addr.ip) 54 | t.Fail() 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Zixuan Guo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /crawler/dht/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Dean Karn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /crawler/dht/bitmap_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBitmap(t *testing.T) { 8 | a := newBitmap(10) 9 | b := newBitmapFrom(a, 10) 10 | c := newBitmapFromBytes([]byte{48, 49, 50, 51, 52, 53, 54, 55, 56, 57}) 11 | d := newBitmapFromString("0123456789") 12 | e := newBitmap(10) 13 | 14 | // Bit 15 | for i := 0; i < a.Size; i++ { 16 | if a.Bit(i) != 0 { 17 | t.Fail() 18 | } 19 | } 20 | 21 | // Compare 22 | if c.Compare(d, d.Size) != 0 { 23 | t.Fail() 24 | } 25 | 26 | // RawString 27 | if c.RawString() != d.RawString() || c.RawString() != "0123456789" { 28 | t.Fail() 29 | } 30 | 31 | // Set 32 | b.Set(5) 33 | if b.Bit(5) != 1 { 34 | t.Fail() 35 | } 36 | 37 | // Unset 38 | b.Unset(5) 39 | if b.Bit(5) == 1 { 40 | t.Fail() 41 | } 42 | 43 | // String 44 | if e.String() != "0000000000" { 45 | t.Fail() 46 | } 47 | e.Set(9) 48 | if e.String() != "0000000001" { 49 | t.Fail() 50 | } 51 | e.Set(2) 52 | if e.String() != "0010000001" { 53 | t.Fail() 54 | } 55 | 56 | a.Set(0) 57 | a.Set(5) 58 | a.Set(8) 59 | if a.String() != "1000010010" { 60 | t.Fail() 61 | } 62 | 63 | // Xor 64 | b.Set(5) 65 | b.Set(9) 66 | if a.Xor(b).String() != "1000000011" { 67 | t.Fail() 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | nginx: 5 | image: linuxserver/letsencrypt 6 | environment: 7 | - PUID=1000 8 | - PGID=1000 9 | - TZ=America/New_York 10 | - URL=YOURDOMAIN.COM 11 | - SUBDOMAINS=www, 12 | - VALIDATION=http 13 | - DNSPLUGIN=cloudflare #optional 14 | - EMAIL=i@zxguo.me #optional 15 | volumes: 16 | - ./nginx:/config 17 | ports: 18 | - 443:443 19 | - 80:80 #optional 20 | depends_on: 21 | - web 22 | web: 23 | build: web/. 24 | depends_on: 25 | - server 26 | crawler: 27 | build: crawler/. 28 | network_mode: "host" 29 | environment: 30 | - address="localhost:50051" 31 | server: 32 | build: server/. 33 | depends_on: 34 | - elastic 35 | - redis 36 | ports: 37 | - "50051:50051" 38 | environment: 39 | - username=crawler 40 | - password=PASSWORD 41 | redis: 42 | image: "redis:alpine" 43 | elastic: 44 | image: "docker.elastic.co/elasticsearch/elasticsearch:7.8.1" 45 | environment: 46 | - discovery.type=single-node 47 | - bootstrap.memory_lock=true 48 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 49 | ulimits: 50 | memlock: 51 | soft: -1 52 | hard: -1 53 | volumes: 54 | - esdata:/usr/share/elasticsearch/data 55 | - ./config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml 56 | expose: 57 | - "9200" 58 | 59 | volumes: 60 | esdata: 61 | driver: local 62 | -------------------------------------------------------------------------------- /web/src/components/search.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import List from "./list"; 4 | import SearchBar from "./searchbar"; 5 | 6 | export default class Search extends React.Component { 7 | constructor(props) { 8 | super(props); 9 | this.state = { 10 | queryText: "" 11 | } 12 | this.listElement = React.createRef() 13 | } 14 | 15 | updateQueryText = (newText) => { 16 | this.setState({ 17 | queryText: newText 18 | }) 19 | } 20 | 21 | handleClick = () => { 22 | this.listElement.current.searchByKeyword(this.state.queryText) 23 | } 24 | 25 | handleKeyDown = (e) => { 26 | if (e.key === 'Enter') { 27 | this.listElement.current.searchByKeyword(this.state.queryText) 28 | } 29 | } 30 | 31 | render() { 32 | return( 33 |
34 |
35 | 39 | 42 |
43 |
44 | 47 |
48 |
49 | ) 50 | } 51 | } -------------------------------------------------------------------------------- /web/src/components/listbar.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | export default class ListItem extends React.Component { 4 | render() { 5 | return ( 6 |
7 |
8 | { 9 | (this.props.currentPage > 1) && ( 10 | 17 | ) 18 | } 19 |
20 |
21 | {this.props.currentPage}/{this.props.totalPage} 22 |
23 |
24 | { 25 | (this.props.currentPage < this.props.totalPage) && ( 26 | 34 | ) 35 | } 36 |
37 |
38 | ); 39 | } 40 | } -------------------------------------------------------------------------------- /web/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | gDHT Demo 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /crawler/dht/util_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestInt2Bytes(t *testing.T) { 8 | cases := []struct { 9 | in uint64 10 | out []byte 11 | }{ 12 | {0, []byte{0}}, 13 | {1, []byte{1}}, 14 | {256, []byte{1, 0}}, 15 | {22129, []byte{86, 113}}, 16 | } 17 | 18 | for _, c := range cases { 19 | r := int2bytes(c.in) 20 | if len(r) != len(c.out) { 21 | t.Fail() 22 | } 23 | 24 | for i, v := range r { 25 | if v != c.out[i] { 26 | t.Fail() 27 | } 28 | } 29 | } 30 | } 31 | 32 | func TestBytes2Int(t *testing.T) { 33 | cases := []struct { 34 | in []byte 35 | out uint64 36 | }{ 37 | {[]byte{0}, 0}, 38 | {[]byte{1}, 1}, 39 | {[]byte{1, 0}, 256}, 40 | {[]byte{86, 113}, 22129}, 41 | } 42 | 43 | for _, c := range cases { 44 | if bytes2int(c.in) != c.out { 45 | t.Fail() 46 | } 47 | } 48 | } 49 | 50 | func TestDecodeCompactIPPortInfo(t *testing.T) { 51 | cases := []struct { 52 | in string 53 | out struct { 54 | ip string 55 | port int 56 | } 57 | }{ 58 | {"123456", struct { 59 | ip string 60 | port int 61 | }{"49.50.51.52", 13622}}, 62 | {"abcdef", struct { 63 | ip string 64 | port int 65 | }{"97.98.99.100", 25958}}, 66 | } 67 | 68 | for _, item := range cases { 69 | ip, port, err := decodeCompactIPPortInfo(item.in) 70 | if err != nil || ip.String() != item.out.ip || port != item.out.port { 71 | t.Fail() 72 | } 73 | } 74 | } 75 | 76 | func TestEncodeCompactIPPortInfo(t *testing.T) { 77 | cases := []struct { 78 | in struct { 79 | ip []byte 80 | port int 81 | } 82 | out string 83 | }{ 84 | {struct { 85 | ip []byte 86 | port int 87 | }{[]byte{49, 50, 51, 52}, 13622}, "123456"}, 88 | {struct { 89 | ip []byte 90 | port int 91 | }{[]byte{97, 98, 99, 100}, 25958}, "abcdef"}, 92 | } 93 | 94 | for _, item := range cases { 95 | info, err := encodeCompactIPPortInfo(item.in.ip, item.in.port) 96 | if err != nil || info != item.out { 97 | t.Fail() 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /crawler/dht/blacklist.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // blockedItem represents a blocked node. 8 | type blockedItem struct { 9 | ip string 10 | port int 11 | createTime time.Time 12 | } 13 | 14 | // blackList manages the blocked nodes including which sends bad information 15 | // and can't ping out. 16 | type blackList struct { 17 | list *syncedMap 18 | maxSize int 19 | expiredAfter time.Duration 20 | } 21 | 22 | // newBlackList returns a blackList pointer. 23 | func newBlackList(size int) *blackList { 24 | return &blackList{ 25 | list: newSyncedMap(), 26 | maxSize: size, 27 | expiredAfter: time.Hour * 1, 28 | } 29 | } 30 | 31 | // genKey returns a key. If port is less than 0, the key wil be ip. Ohterwise 32 | // it will be `ip:port` format. 33 | func (bl *blackList) genKey(ip string, port int) string { 34 | key := ip 35 | if port >= 0 { 36 | key = genAddress(ip, port) 37 | } 38 | return key 39 | } 40 | 41 | // insert adds a blocked item to the blacklist. 42 | func (bl *blackList) insert(ip string, port int) { 43 | if bl.list.Len() >= bl.maxSize { 44 | return 45 | } 46 | 47 | bl.list.Set(bl.genKey(ip, port), &blockedItem{ 48 | ip: ip, 49 | port: port, 50 | createTime: time.Now(), 51 | }) 52 | } 53 | 54 | // delete removes blocked item form the blackList. 55 | func (bl *blackList) delete(ip string, port int) { 56 | bl.list.Delete(bl.genKey(ip, port)) 57 | } 58 | 59 | // validate checks whether ip-port pair is in the block nodes list. 60 | func (bl *blackList) in(ip string, port int) bool { 61 | if _, ok := bl.list.Get(ip); ok { 62 | return true 63 | } 64 | 65 | key := bl.genKey(ip, port) 66 | 67 | v, ok := bl.list.Get(key) 68 | if ok { 69 | if time.Now().Sub(v.(*blockedItem).createTime) < bl.expiredAfter { 70 | return true 71 | } 72 | bl.list.Delete(key) 73 | } 74 | return false 75 | } 76 | 77 | // clear cleans the expired items every 10 minutes. 78 | func (bl *blackList) clear() { 79 | for _ = range time.Tick(time.Minute * 10) { 80 | keys := make([]interface{}, 0, 100) 81 | 82 | for item := range bl.list.Iter() { 83 | if time.Now().Sub( 84 | item.val.(*blockedItem).createTime) > bl.expiredAfter { 85 | 86 | keys = append(keys, item.key) 87 | } 88 | } 89 | 90 | bl.list.DeleteMulti(keys) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gDHT 2 | A distributed self-host DHT torrent search suite 3 | 4 | demo site: [guo.sh](https://www.guo.sh) 5 | 6 | ## Introduction 7 | gDHT is a search engine suite that allows user to host their own torrent search engine. There are four major components of the suite: `crawler`, `server`, `ElasticSearch`, and `web`. The distributed `crawler` will monitor the traffic on DHT network to collect meta information of the torrent and then sent collected information to `server` via gRPC. Upon on receiving the information, the `server` will push them into Redis message queue and asynchronously process (Ex. filter unwanted torrent) and index them into `ElasticSearch`. Finally, you can search the torrent information at the React web interface. 8 | 9 | ![](https://github.com/Olament/gDHT/blob/master/doc/architecture.jpg) 10 | 11 | ## Getting Started 12 | 13 | ### Nginx 14 | 15 | The nginx server is bound to `YOURDOMAIN.COM` by default. If you want to host your own torrent search engine, your can change Nginx's environment variable `URL` in `docker-compose.yml`. 16 | 17 | ### Golang Crawler 18 | 19 | You can leave the setting in `docker-compose.yml` unchanged if you run the suite with only one crawler. However, to add additional crawler to the system, change environment variable `address` under `crawler` to `master-server-ip-address:50051`. 20 | 21 | ### ElasticSearch Security 22 | 23 | The security features of the `ElasticSearch` is enabled by default. To ensure that `crawler` and `web` function normally, you need to create two user: *web* and *crawler*. Notice that *crawler* must have permission to write and read index and *web* must have permission to read index. Once you created those two users, you can pass the username and password to Golang `crawler` via the environment variables in `docker-compose.yml`. To pass username and password of *web* to `web`, change `REACT_APP_USERNAME` and `REACT_APP_PASSWORD` under `web/src/.env`. For more information on how to set up ElasticSearch, check those two articles [Configuring security in Elasticsearchedit](https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-security.html) and [Getting started with Elasticsearch security](https://www.elastic.co/blog/getting-started-with-elasticsearch-security). 24 | 25 | Then, start the server by 26 | 27 | ``` bash 28 | docker-compose build 29 | docker-compose up 30 | ``` 31 | 32 | ## Acknowledge 33 | 34 | DHT crawler from [shiyanhui](https://github.com/shiyanhui/dht) 35 | 36 | Web CSS theme from [Tania Rascia](https://github.com/taniarascia/taniarascia.com/) 37 | 38 | -------------------------------------------------------------------------------- /web/src/components/searchbar.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import Autosuggest from 'react-autosuggest'; 3 | 4 | const getSuggestionValue = suggestion => suggestion; 5 | 6 | const renderSuggestion = suggestion => ( 7 |
8 | {suggestion} 9 |
10 | ); 11 | 12 | export default class SearchBar extends React.Component { 13 | constructor(props) { 14 | super(props); 15 | this.state = { 16 | value: '', 17 | suggestions: [] 18 | }; 19 | } 20 | 21 | onChange = (event, { newValue }) => { 22 | this.props.updateQueryText(newValue); 23 | this.setState({ 24 | value: newValue 25 | }); 26 | }; 27 | 28 | onSuggestionsFetchRequested = ({ value }) => { 29 | const requestOptions = { 30 | method: 'POST', 31 | headers: { 32 | 'Content-Type': 'application/json', 33 | 'Authorization': 'Basic ' + Buffer.from(process.env.REACT_APP_USERNAME + ':' + process.env.REACT_APP_PASSWORD).toString('base64') 34 | }, 35 | body: JSON.stringify({ 36 | "suggest": { 37 | "search-suggest" : { 38 | "prefix" : value, 39 | "completion" : { 40 | "field" : "name_suggest", 41 | "size": 10 42 | } 43 | } 44 | } 45 | }) 46 | }; 47 | 48 | fetch('/api/_search', requestOptions) 49 | .then(response => response.json()) 50 | .then(data => this.setState({ 51 | suggestions: data.suggest["search-suggest"][0].options.map(item => item.text) 52 | })); 53 | }; 54 | 55 | onSuggestionsClearRequested = () => { 56 | this.setState({ 57 | suggestions: [] 58 | }); 59 | }; 60 | 61 | render() { 62 | const { value, suggestions } = this.state; 63 | 64 | const inputProps = { 65 | placeholder: 'Search for anything...', 66 | id: "search", 67 | type: "search", 68 | value, 69 | onChange: this.onChange, 70 | onKeyDown: this.props.handleKeyDown, 71 | }; 72 | 73 | return ( 74 | 82 | ); 83 | } 84 | } -------------------------------------------------------------------------------- /crawler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Olament/gDHT/crawler/dht" 5 | pb "github.com/Olament/gDHT/service" 6 | "context" 7 | "encoding/hex" 8 | "google.golang.org/grpc" 9 | "log" 10 | "net/http" 11 | _ "net/http/pprof" 12 | "os" 13 | "strings" 14 | ) 15 | 16 | type file struct { 17 | Path string `json:"path"` 18 | Length int `json:"length"` 19 | } 20 | 21 | type bitTorrent struct { 22 | InfoHash string `json:"infohash"` 23 | Name string `json:"name"` 24 | Files []file `json:"files,omitempty"` 25 | Length int `json:"length,omitempty"` 26 | } 27 | 28 | func main() { 29 | go func() { 30 | http.ListenAndServe(":6060", nil) 31 | }() 32 | 33 | conn, err := grpc.Dial(os.Getenv("address"), grpc.WithInsecure(), grpc.WithBlock()) 34 | if err != nil { 35 | log.Fatalf("did not connect: %v", err) 36 | } 37 | //defer conn.Close() 38 | c := pb.NewBitTorrentClient(conn) 39 | 40 | ctx := context.Background() 41 | 42 | w := dht.NewWire(65536, 1024, 256) 43 | go func() { 44 | for resp := range w.Response() { 45 | metadata, err := dht.Decode(resp.MetadataInfo) 46 | if err != nil { 47 | continue 48 | } 49 | info := metadata.(map[string]interface{}) 50 | 51 | if _, ok := info["name"]; !ok { 52 | continue 53 | } 54 | 55 | bt := bitTorrent{ 56 | InfoHash: hex.EncodeToString(resp.InfoHash), 57 | Name: info["name"].(string), 58 | } 59 | 60 | if v, ok := info["files"]; ok { 61 | files := v.([]interface{}) 62 | bt.Files = make([]file, len(files)) 63 | 64 | for i, item := range files { 65 | f := item.(map[string]interface{}) 66 | 67 | pathInterface := f["path"].([]interface{}) 68 | pathString := make([]string, len(pathInterface)) 69 | 70 | for index, value := range pathInterface { 71 | pathString[index] = value.(string) 72 | } 73 | 74 | bt.Files[i] = file{ 75 | Path: strings.Join(pathString, ""), 76 | Length: f["length"].(int), 77 | } 78 | } 79 | } else if _, ok := info["length"]; ok { 80 | bt.Length = info["length"].(int) 81 | } 82 | 83 | files := make([]*pb.File, len(bt.Files)) 84 | for _, value := range bt.Files { 85 | f := &pb.File{ 86 | Path: value.Path, 87 | Length: int32(value.Length), 88 | } 89 | files = append(files, f) 90 | } 91 | log.Println(bt.InfoHash) 92 | _, err = c.Send(ctx, &pb.BitTorrent{ 93 | Infohash: bt.InfoHash, 94 | Name: bt.Name, 95 | Files: files, 96 | Length: int32(bt.Length), 97 | }) 98 | } 99 | }() 100 | go w.Run() 101 | 102 | config := dht.NewCrawlConfig() 103 | config.OnAnnouncePeer = func(infoHash, ip string, port int) { 104 | w.Request([]byte(infoHash), ip, port) 105 | } 106 | d := dht.New(config) 107 | 108 | d.Run() 109 | } 110 | -------------------------------------------------------------------------------- /service/bittorrent_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | 3 | package service 4 | 5 | import ( 6 | context "context" 7 | grpc "google.golang.org/grpc" 8 | codes "google.golang.org/grpc/codes" 9 | status "google.golang.org/grpc/status" 10 | ) 11 | 12 | // This is a compile-time assertion to ensure that this generated file 13 | // is compatible with the grpc package it is being compiled against. 14 | const _ = grpc.SupportPackageIsVersion6 15 | 16 | // BitTorrentClient is the client API for BitTorrent service. 17 | // 18 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 19 | type BitTorrentClient interface { 20 | Send(ctx context.Context, in *BitTorrent, opts ...grpc.CallOption) (*Empty, error) 21 | } 22 | 23 | type bitTorrentClient struct { 24 | cc grpc.ClientConnInterface 25 | } 26 | 27 | func NewBitTorrentClient(cc grpc.ClientConnInterface) BitTorrentClient { 28 | return &bitTorrentClient{cc} 29 | } 30 | 31 | func (c *bitTorrentClient) Send(ctx context.Context, in *BitTorrent, opts ...grpc.CallOption) (*Empty, error) { 32 | out := new(Empty) 33 | err := c.cc.Invoke(ctx, "/service.bitTorrent/send", in, out, opts...) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return out, nil 38 | } 39 | 40 | // BitTorrentServer is the server API for BitTorrent service. 41 | // All implementations must embed UnimplementedBitTorrentServer 42 | // for forward compatibility 43 | type BitTorrentServer interface { 44 | Send(context.Context, *BitTorrent) (*Empty, error) 45 | mustEmbedUnimplementedBitTorrentServer() 46 | } 47 | 48 | // UnimplementedBitTorrentServer must be embedded to have forward compatible implementations. 49 | type UnimplementedBitTorrentServer struct { 50 | } 51 | 52 | func (*UnimplementedBitTorrentServer) Send(context.Context, *BitTorrent) (*Empty, error) { 53 | return nil, status.Errorf(codes.Unimplemented, "method Send not implemented") 54 | } 55 | func (*UnimplementedBitTorrentServer) mustEmbedUnimplementedBitTorrentServer() {} 56 | 57 | func RegisterBitTorrentServer(s *grpc.Server, srv BitTorrentServer) { 58 | s.RegisterService(&_BitTorrent_serviceDesc, srv) 59 | } 60 | 61 | func _BitTorrent_Send_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 62 | in := new(BitTorrent) 63 | if err := dec(in); err != nil { 64 | return nil, err 65 | } 66 | if interceptor == nil { 67 | return srv.(BitTorrentServer).Send(ctx, in) 68 | } 69 | info := &grpc.UnaryServerInfo{ 70 | Server: srv, 71 | FullMethod: "/service.bitTorrent/Send", 72 | } 73 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 74 | return srv.(BitTorrentServer).Send(ctx, req.(*BitTorrent)) 75 | } 76 | return interceptor(ctx, in, info, handler) 77 | } 78 | 79 | var _BitTorrent_serviceDesc = grpc.ServiceDesc{ 80 | ServiceName: "service.bitTorrent", 81 | HandlerType: (*BitTorrentServer)(nil), 82 | Methods: []grpc.MethodDesc{ 83 | { 84 | MethodName: "send", 85 | Handler: _BitTorrent_Send_Handler, 86 | }, 87 | }, 88 | Streams: []grpc.StreamDesc{}, 89 | Metadata: "bittorrent.proto", 90 | } 91 | -------------------------------------------------------------------------------- /crawler/dht/util.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "crypto/rand" 5 | "errors" 6 | "io/ioutil" 7 | "net" 8 | "net/http" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // randomString generates a size-length string randomly. 15 | func randomString(size int) string { 16 | buff := make([]byte, size) 17 | rand.Read(buff) 18 | return string(buff) 19 | } 20 | 21 | // bytes2int returns the int value it represents. 22 | func bytes2int(data []byte) uint64 { 23 | n, val := len(data), uint64(0) 24 | if n > 8 { 25 | panic("data too long") 26 | } 27 | 28 | for i, b := range data { 29 | val += uint64(b) << uint64((n-i-1)*8) 30 | } 31 | return val 32 | } 33 | 34 | // int2bytes returns the byte array it represents. 35 | func int2bytes(val uint64) []byte { 36 | data, j := make([]byte, 8), -1 37 | for i := 0; i < 8; i++ { 38 | shift := uint64((7 - i) * 8) 39 | data[i] = byte((val & (0xff << shift)) >> shift) 40 | 41 | if j == -1 && data[i] != 0 { 42 | j = i 43 | } 44 | } 45 | 46 | if j != -1 { 47 | return data[j:] 48 | } 49 | return data[:1] 50 | } 51 | 52 | // decodeCompactIPPortInfo decodes compactIP-address/port info in BitTorrent 53 | // DHT Protocol. It returns the ip and port number. 54 | func decodeCompactIPPortInfo(info string) (ip net.IP, port int, err error) { 55 | if len(info) != 6 { 56 | err = errors.New("compact info should be 6-length long") 57 | return 58 | } 59 | 60 | ip = net.IPv4(info[0], info[1], info[2], info[3]) 61 | port = int((uint16(info[4]) << 8) | uint16(info[5])) 62 | return 63 | } 64 | 65 | // encodeCompactIPPortInfo encodes an ip and a port number to 66 | // compactIP-address/port info. 67 | func encodeCompactIPPortInfo(ip net.IP, port int) (info string, err error) { 68 | if port > 65535 || port < 0 { 69 | err = errors.New( 70 | "port should be no greater than 65535 and no less than 0") 71 | return 72 | } 73 | 74 | p := int2bytes(uint64(port)) 75 | if len(p) < 2 { 76 | p = append(p, p[0]) 77 | p[0] = 0 78 | } 79 | 80 | info = string(append(ip, p...)) 81 | return 82 | } 83 | 84 | // getLocalIPs returns local ips. 85 | func getLocalIPs() (ips []string) { 86 | ips = make([]string, 0, 6) 87 | 88 | addrs, err := net.InterfaceAddrs() 89 | if err != nil { 90 | return 91 | } 92 | 93 | for _, addr := range addrs { 94 | ip, _, err := net.ParseCIDR(addr.String()) 95 | if err != nil { 96 | continue 97 | } 98 | ips = append(ips, ip.String()) 99 | } 100 | return 101 | } 102 | 103 | // getRemoteIP returns the wlan ip. 104 | func getRemoteIP() (ip string, err error) { 105 | client := &http.Client{ 106 | Timeout: time.Second * 30, 107 | } 108 | 109 | req, err := http.NewRequest("GET", "http://ifconfig.me", nil) 110 | if err != nil { 111 | return 112 | } 113 | 114 | req.Header.Set("User-Agent", "curl") 115 | res, err := client.Do(req) 116 | if err != nil { 117 | return 118 | } 119 | 120 | defer res.Body.Close() 121 | 122 | data, err := ioutil.ReadAll(res.Body) 123 | if err != nil { 124 | return 125 | } 126 | ip = string(data) 127 | 128 | return 129 | } 130 | 131 | // genAddress returns a ip:port address. 132 | func genAddress(ip string, port int) string { 133 | return strings.Join([]string{ip, strconv.Itoa(port)}, ":") 134 | } 135 | -------------------------------------------------------------------------------- /crawler/dht/container_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | ) 7 | 8 | func TestSyncedMap(t *testing.T) { 9 | cases := []mapItem{ 10 | {"a", 0}, 11 | {"b", 1}, 12 | {"c", 2}, 13 | } 14 | 15 | sm := newSyncedMap() 16 | 17 | set := func() { 18 | group := sync.WaitGroup{} 19 | for _, item := range cases { 20 | group.Add(1) 21 | go func(item mapItem) { 22 | sm.Set(item.key, item.val) 23 | group.Done() 24 | }(item) 25 | } 26 | group.Wait() 27 | } 28 | 29 | isEmpty := func() { 30 | if sm.Len() != 0 { 31 | t.Fail() 32 | } 33 | } 34 | 35 | // Set 36 | set() 37 | if sm.Len() != len(cases) { 38 | t.Fail() 39 | } 40 | 41 | Loop: 42 | // Iter 43 | for item := range sm.Iter() { 44 | for _, c := range cases { 45 | if item.key == c.key && item.val == c.val { 46 | continue Loop 47 | } 48 | } 49 | t.Fail() 50 | } 51 | 52 | // Get, Delete, Has 53 | for _, item := range cases { 54 | val, ok := sm.Get(item.key) 55 | if !ok || val != item.val { 56 | t.Fail() 57 | } 58 | 59 | sm.Delete(item.key) 60 | if sm.Has(item.key) { 61 | t.Fail() 62 | } 63 | } 64 | isEmpty() 65 | 66 | // DeleteMulti 67 | set() 68 | sm.DeleteMulti([]interface{}{"a", "b", "c"}) 69 | isEmpty() 70 | 71 | // Clear 72 | set() 73 | sm.Clear() 74 | isEmpty() 75 | } 76 | 77 | func TestSyncedList(t *testing.T) { 78 | sl := newSyncedList() 79 | 80 | insert := func() { 81 | for i := 0; i < 10; i++ { 82 | sl.PushBack(i) 83 | } 84 | } 85 | 86 | isEmpty := func() { 87 | if sl.Len() != 0 { 88 | t.Fail() 89 | } 90 | } 91 | 92 | // PushBack 93 | insert() 94 | 95 | // Len 96 | if sl.Len() != 10 { 97 | t.Fail() 98 | } 99 | 100 | // Iter 101 | i := 0 102 | for item := range sl.Iter() { 103 | if item.Value.(int) != i { 104 | t.Fail() 105 | } 106 | i++ 107 | } 108 | 109 | // Front 110 | if sl.Front().Value.(int) != 0 { 111 | t.Fail() 112 | } 113 | 114 | // Back 115 | if sl.Back().Value.(int) != 9 { 116 | t.Fail() 117 | } 118 | 119 | // Remove 120 | for i := 0; i < 10; i++ { 121 | if sl.Remove(sl.Front()).(int) != i { 122 | t.Fail() 123 | } 124 | } 125 | isEmpty() 126 | 127 | // Clear 128 | insert() 129 | sl.Clear() 130 | isEmpty() 131 | } 132 | 133 | func TestKeyedDeque(t *testing.T) { 134 | cases := []mapItem{ 135 | {"a", 0}, 136 | {"b", 1}, 137 | {"c", 2}, 138 | } 139 | 140 | deque := newKeyedDeque() 141 | 142 | insert := func() { 143 | for _, item := range cases { 144 | deque.Push(item.key, item.val) 145 | } 146 | } 147 | 148 | isEmpty := func() { 149 | if deque.Len() != 0 { 150 | t.Fail() 151 | } 152 | } 153 | 154 | // Push 155 | insert() 156 | 157 | // Len 158 | if deque.Len() != 3 { 159 | t.Fail() 160 | } 161 | 162 | // Iter 163 | i := 0 164 | for e := range deque.Iter() { 165 | if e.Value.(int) != i { 166 | t.Fail() 167 | } 168 | i++ 169 | } 170 | 171 | // HasKey, Get, Delete 172 | for _, item := range cases { 173 | if !deque.HasKey(item.key) { 174 | t.Fail() 175 | } 176 | 177 | e, ok := deque.Get(item.key) 178 | if !ok || e.Value.(int) != item.val { 179 | t.Fail() 180 | } 181 | 182 | if deque.Delete(item.key) != item.val { 183 | t.Fail() 184 | } 185 | 186 | if deque.HasKey(item.key) { 187 | t.Fail() 188 | } 189 | } 190 | isEmpty() 191 | 192 | // Clear 193 | insert() 194 | deque.Clear() 195 | isEmpty() 196 | } 197 | -------------------------------------------------------------------------------- /crawler/dht/bencode_test.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDecodeString(t *testing.T) { 8 | cases := []struct { 9 | in string 10 | out string 11 | }{ 12 | {"0:", ""}, 13 | {"1:a", "a"}, 14 | {"5:hello", "hello"}, 15 | } 16 | 17 | for _, c := range cases { 18 | if out, err := Decode([]byte(c.in)); err != nil || out != c.out { 19 | t.Error(err) 20 | } 21 | } 22 | } 23 | 24 | func TestDecodeInt(t *testing.T) { 25 | cases := []struct { 26 | in string 27 | out int 28 | }{ 29 | {"i123e:", 123}, 30 | {"i0e", 0}, 31 | {"i-1e", -1}, 32 | } 33 | 34 | for _, c := range cases { 35 | if out, err := Decode([]byte(c.in)); err != nil || out != c.out { 36 | t.Error(err) 37 | } 38 | } 39 | } 40 | 41 | func TestDecodeList(t *testing.T) { 42 | cases := []struct { 43 | in string 44 | out []interface{} 45 | }{ 46 | {"li123ei-1ee", []interface{}{123, -1}}, 47 | {"l5:helloe", []interface{}{"hello"}}, 48 | {"ld5:hello5:worldee", []interface{}{map[string]interface{}{"hello": "world"}}}, 49 | {"lli1ei2eee", []interface{}{[]interface{}{1, 2}}}, 50 | } 51 | 52 | for i, c := range cases { 53 | v, err := Decode([]byte(c.in)) 54 | if err != nil { 55 | t.Fail() 56 | } 57 | 58 | out := v.([]interface{}) 59 | 60 | switch i { 61 | case 0, 1: 62 | for j, item := range out { 63 | if item != c.out[j] { 64 | t.Fail() 65 | } 66 | } 67 | case 2: 68 | if len(out) != 1 { 69 | t.Fail() 70 | } 71 | 72 | o := out[0].(map[string]interface{}) 73 | cout := c.out[0].(map[string]interface{}) 74 | 75 | for k, v := range o { 76 | if cv, ok := cout[k]; !ok || v != cv { 77 | t.Fail() 78 | } 79 | } 80 | case 3: 81 | if len(out) != 1 { 82 | t.Fail() 83 | } 84 | 85 | o := out[0].([]interface{}) 86 | cout := c.out[0].([]interface{}) 87 | 88 | for j, item := range o { 89 | if item != cout[j] { 90 | t.Fail() 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | func TestDecodeDict(t *testing.T) { 98 | cases := []struct { 99 | in string 100 | out map[string]interface{} 101 | }{ 102 | {"d5:helloi100ee", map[string]interface{}{"hello": 100}}, 103 | {"d3:foo3:bare", map[string]interface{}{"foo": "bar"}}, 104 | {"d1:ad3:foo3:baree", map[string]interface{}{"a": map[string]interface{}{"foo": "bar"}}}, 105 | {"d4:listli1eee", map[string]interface{}{"list": []interface{}{1}}}, 106 | } 107 | 108 | for i, c := range cases { 109 | v, err := Decode([]byte(c.in)) 110 | if err != nil { 111 | t.Fail() 112 | } 113 | 114 | out := v.(map[string]interface{}) 115 | 116 | switch i { 117 | case 0, 1: 118 | for k, v := range out { 119 | if cv, ok := c.out[k]; !ok || v != cv { 120 | t.Fail() 121 | } 122 | } 123 | case 2: 124 | if len(out) != 1 { 125 | t.Fail() 126 | } 127 | 128 | v, ok := out["a"] 129 | if !ok { 130 | t.Fail() 131 | } 132 | 133 | cout := c.out["a"].(map[string]interface{}) 134 | 135 | for k, v := range v.(map[string]interface{}) { 136 | if cv, ok := cout[k]; !ok || v != cv { 137 | t.Fail() 138 | } 139 | } 140 | case 3: 141 | if len(out) != 1 { 142 | t.Fail() 143 | } 144 | 145 | v, ok := out["list"] 146 | if !ok { 147 | t.Fail() 148 | } 149 | 150 | cout := c.out["list"].([]interface{}) 151 | 152 | for j, v := range v.([]interface{}) { 153 | if v != cout[j] { 154 | t.Fail() 155 | } 156 | } 157 | } 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /crawler/dht/bitmap.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // bitmap represents a bit array. 10 | type bitmap struct { 11 | Size int 12 | data []byte 13 | } 14 | 15 | // newBitmap returns a size-length bitmap pointer. 16 | func newBitmap(size int) *bitmap { 17 | div, mod := size>>3, size&0x07 18 | if mod > 0 { 19 | div++ 20 | } 21 | return &bitmap{size, make([]byte, div)} 22 | } 23 | 24 | // newBitmapFrom returns a new copyed bitmap pointer which 25 | // newBitmap.data = other.data[:size]. 26 | func newBitmapFrom(other *bitmap, size int) *bitmap { 27 | bitmap := newBitmap(size) 28 | 29 | if size > other.Size { 30 | size = other.Size 31 | } 32 | 33 | div := size >> 3 34 | 35 | for i := 0; i < div; i++ { 36 | bitmap.data[i] = other.data[i] 37 | } 38 | 39 | for i := div << 3; i < size; i++ { 40 | if other.Bit(i) == 1 { 41 | bitmap.Set(i) 42 | } 43 | } 44 | 45 | return bitmap 46 | } 47 | 48 | // newBitmapFromBytes returns a bitmap pointer created from a byte array. 49 | func newBitmapFromBytes(data []byte) *bitmap { 50 | bitmap := newBitmap(len(data) << 3) 51 | copy(bitmap.data, data) 52 | return bitmap 53 | } 54 | 55 | // newBitmapFromString returns a bitmap pointer created from a string. 56 | func newBitmapFromString(data string) *bitmap { 57 | return newBitmapFromBytes([]byte(data)) 58 | } 59 | 60 | // Bit returns the bit at index. 61 | func (bitmap *bitmap) Bit(index int) int { 62 | if index >= bitmap.Size { 63 | panic("index out of range") 64 | } 65 | 66 | div, mod := index>>3, index&0x07 67 | return int((uint(bitmap.data[div]) & (1 << uint(7-mod))) >> uint(7-mod)) 68 | } 69 | 70 | // set sets the bit at index `index`. If bit is true, set 1, otherwise set 0. 71 | func (bitmap *bitmap) set(index int, bit int) { 72 | if index >= bitmap.Size { 73 | panic("index out of range") 74 | } 75 | 76 | div, mod := index>>3, index&0x07 77 | shift := byte(1 << uint(7-mod)) 78 | 79 | bitmap.data[div] &= ^shift 80 | if bit > 0 { 81 | bitmap.data[div] |= shift 82 | } 83 | } 84 | 85 | // Set sets the bit at idnex to 1. 86 | func (bitmap *bitmap) Set(index int) { 87 | bitmap.set(index, 1) 88 | } 89 | 90 | // Unset sets the bit at idnex to 0. 91 | func (bitmap *bitmap) Unset(index int) { 92 | bitmap.set(index, 0) 93 | } 94 | 95 | // Compare compares the prefixLen-prefix of two bitmap. 96 | // - If bitmap.data[:prefixLen] < other.data[:prefixLen], return -1. 97 | // - If bitmap.data[:prefixLen] > other.data[:prefixLen], return 1. 98 | // - Otherwise return 0. 99 | func (bitmap *bitmap) Compare(other *bitmap, prefixLen int) int { 100 | if prefixLen > bitmap.Size || prefixLen > other.Size { 101 | panic("index out of range") 102 | } 103 | 104 | div, mod := prefixLen>>3, prefixLen&0x07 105 | res := bytes.Compare(bitmap.data[:div], other.data[:div]) 106 | if res != 0 { 107 | return res 108 | } 109 | 110 | for i := div << 3; i < (div<<3)+mod; i++ { 111 | bit1, bit2 := bitmap.Bit(i), other.Bit(i) 112 | if bit1 > bit2 { 113 | return 1 114 | } else if bit1 < bit2 { 115 | return -1 116 | } 117 | } 118 | 119 | return 0 120 | } 121 | 122 | // Xor returns the xor value of two bitmap. 123 | func (bitmap *bitmap) Xor(other *bitmap) *bitmap { 124 | if bitmap.Size != other.Size { 125 | panic("size not the same") 126 | } 127 | 128 | distance := newBitmap(bitmap.Size) 129 | xor(distance.data, bitmap.data, other.data) 130 | 131 | return distance 132 | } 133 | 134 | // String returns the bit sequence string of the bitmap. 135 | func (bitmap *bitmap) String() string { 136 | div, mod := bitmap.Size>>3, bitmap.Size&0x07 137 | buff := make([]string, div+mod) 138 | 139 | for i := 0; i < div; i++ { 140 | buff[i] = fmt.Sprintf("%08b", bitmap.data[i]) 141 | } 142 | 143 | for i := div; i < div+mod; i++ { 144 | buff[i] = fmt.Sprintf("%1b", bitmap.Bit(div*8+(i-div))) 145 | } 146 | 147 | return strings.Join(buff, "") 148 | } 149 | 150 | // RawString returns the string value of bitmap.data. 151 | func (bitmap *bitmap) RawString() string { 152 | return string(bitmap.data) 153 | } 154 | -------------------------------------------------------------------------------- /nginx/nginx/site-confs/default: -------------------------------------------------------------------------------- 1 | ## Version 2020/05/23 - Changelog: https://github.com/linuxserver/docker-letsencrypt/commits/master/root/defaults/default 2 | 3 | # redirect all traffic to https 4 | server { 5 | listen 80 default_server; 6 | listen [::]:80 default_server; 7 | server_name _; 8 | return 301 https://$host$request_uri; 9 | } 10 | 11 | # main server block 12 | server { 13 | listen 443 ssl http2 default_server; 14 | listen [::]:443 ssl http2 default_server; 15 | 16 | server_name _; 17 | 18 | # all ssl related config moved to ssl.conf 19 | include /config/nginx/ssl.conf; 20 | 21 | client_max_body_size 0; 22 | 23 | location /api/ { 24 | proxy_pass http://elastic:9200/; 25 | } 26 | location / { 27 | proxy_pass http://web:3000/; 28 | } 29 | 30 | # sample reverse proxy config for password protected couchpotato running at IP 192.168.1.50 port 5050 with base url "cp" 31 | # notice this is within the same server block as the base 32 | # don't forget to generate the .htpasswd file as described on docker hub 33 | # location ^~ /cp { 34 | # auth_basic "Restricted"; 35 | # auth_basic_user_file /config/nginx/.htpasswd; 36 | # include /config/nginx/proxy.conf; 37 | # proxy_pass http://192.168.1.50:5050/cp; 38 | # } 39 | 40 | } 41 | 42 | # sample reverse proxy config without url base, but as a subdomain "cp", ip and port same as above 43 | # notice this is a new server block, you need a new server block for each subdomain 44 | #server { 45 | # listen 443 ssl http2; 46 | # listen [::]:443 ssl http2; 47 | # 48 | # root /config/www; 49 | # index index.html index.htm index.php; 50 | # 51 | # server_name cp.*; 52 | # 53 | # include /config/nginx/ssl.conf; 54 | # 55 | # client_max_body_size 0; 56 | # 57 | # location / { 58 | # auth_basic "Restricted"; 59 | # auth_basic_user_file /config/nginx/.htpasswd; 60 | # include /config/nginx/proxy.conf; 61 | # proxy_pass http://192.168.1.50:5050; 62 | # } 63 | #} 64 | 65 | # sample reverse proxy config for "heimdall" via subdomain, with ldap authentication 66 | # ldap-auth container has to be running and the /config/nginx/ldap.conf file should be filled with ldap info 67 | # notice this is a new server block, you need a new server block for each subdomain 68 | #server { 69 | # listen 443 ssl http2; 70 | # listen [::]:443 ssl http2; 71 | # 72 | # root /config/www; 73 | # index index.html index.htm index.php; 74 | # 75 | # server_name heimdall.*; 76 | # 77 | # include /config/nginx/ssl.conf; 78 | # 79 | # include /config/nginx/ldap.conf; 80 | # 81 | # client_max_body_size 0; 82 | # 83 | # location / { 84 | # # the next two lines will enable ldap auth along with the included ldap.conf in the server block 85 | # auth_request /auth; 86 | # error_page 401 =200 /ldaplogin; 87 | # 88 | # include /config/nginx/proxy.conf; 89 | # resolver 127.0.0.11 valid=30s; 90 | # set $upstream_app heimdall; 91 | # set $upstream_port 443; 92 | # set $upstream_proto https; 93 | # proxy_pass $upstream_proto://$upstream_app:$upstream_port; 94 | # } 95 | #} 96 | 97 | # sample reverse proxy config for "heimdall" via subdomain, with Authelia 98 | # Authelia container has to be running in the same user defined bridge network, with container name "authelia", and with 'path: "authelia"' set in its configuration.yml 99 | # notice this is a new server block, you need a new server block for each subdomain 100 | #server { 101 | # listen 443 ssl http2; 102 | # listen [::]:443 ssl http2; 103 | # 104 | # root /config/www; 105 | # index index.html index.htm index.php; 106 | # 107 | # server_name heimdall.*; 108 | # 109 | # include /config/nginx/ssl.conf; 110 | # 111 | # include /config/nginx/authelia-server.conf; 112 | # 113 | # client_max_body_size 0; 114 | # 115 | # location / { 116 | # # the next line will enable Authelia along with the included authelia-server.conf in the server block 117 | # include /config/nginx/authelia-location.conf; 118 | # 119 | # include /config/nginx/proxy.conf; 120 | # resolver 127.0.0.11 valid=30s; 121 | # set $upstream_app heimdall; 122 | # set $upstream_port 443; 123 | # set $upstream_proto https; 124 | # proxy_pass $upstream_proto://$upstream_app:$upstream_port; 125 | # } 126 | #} 127 | 128 | # enable subdomain method reverse proxy confs 129 | include /config/nginx/proxy-confs/*.subdomain.conf; 130 | # enable proxy cache for auth 131 | proxy_cache_path cache/ keys_zone=auth_cache:10m; 132 | -------------------------------------------------------------------------------- /server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | pb "github.com/Olament/gDHT/service" 8 | "github.com/go-redis/redis" 9 | "google.golang.org/grpc" 10 | "log" 11 | "net" 12 | "github.com/olivere/elastic/v7" 13 | "os" 14 | "time" 15 | ) 16 | 17 | type server struct { 18 | pb.UnimplementedBitTorrentServer 19 | } 20 | 21 | type file struct { 22 | Path string `json:"path"` 23 | Length int `json:"length"` 24 | } 25 | 26 | type bitdata struct { 27 | InfoHash string `json:"infohash"` 28 | Name string `json:"name"` 29 | Name_Suggest string `json:"name_suggest"` 30 | Files []file `json:"files,omitempty"` 31 | Length int `json:"length,omitempty"` 32 | } 33 | 34 | var ctx = context.Background() 35 | var rdb = redis.NewClient(&redis.Options{ 36 | Addr: "redis:6379", 37 | Password: "", 38 | DB: 0, 39 | }) 40 | 41 | 42 | func (s *server) Send(ctx context.Context, in *pb.BitTorrent) (*pb.Empty, error) { 43 | /* map gRPC struct to bittorrent data */ 44 | files := make([]file, len(in.Files)) 45 | for i, v := range in.Files { 46 | files[i] = file{ 47 | Path: v.Path, 48 | Length: int(v.Length), 49 | } 50 | } 51 | data := bitdata{ 52 | InfoHash: in.Infohash, 53 | Name: in.Name, 54 | Files: files, 55 | Length: int(in.Length), 56 | } 57 | 58 | value, err := json.Marshal(data) 59 | if err != nil { 60 | return &pb.Empty{}, err 61 | } 62 | fmt.Printf("Receive %s\n", data.Name) 63 | err = rdb.LPush("queue", value).Err() 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | return &pb.Empty{}, nil 69 | } 70 | 71 | // process data from redis message queue by indexing them to elastic search 72 | func Process(client *elastic.Client, bulk *elastic.BulkService, value string) { 73 | 74 | // TODO: filter the value 75 | 76 | var torrent bitdata 77 | err := json.Unmarshal([]byte(value), &torrent) 78 | if err != nil { 79 | log.Printf("Fail to unmarshal %s\n", value) 80 | return 81 | } 82 | torrent.Name_Suggest = torrent.Name 83 | 84 | isExist, err := client.Exists().Index("torrent").Id(torrent.InfoHash).Do(ctx) 85 | if err != nil { 86 | log.Printf("Fail to check if %s exist in ES\n", torrent.InfoHash) 87 | return 88 | } 89 | 90 | if !isExist { 91 | newRequest := elastic.NewBulkIndexRequest().Index("torrent").Id(torrent.InfoHash).Doc(torrent) 92 | bulk = bulk.Add(newRequest) 93 | 94 | log.Printf("[%d] %s\n", bulk.NumberOfActions(), torrent.InfoHash) 95 | 96 | if bulk.NumberOfActions() > 20 { //TODO: change this later 97 | _, err = bulk.Do(ctx) 98 | if err != nil { 99 | log.Printf("Failed to bulk index: %s\n", err) 100 | } 101 | } 102 | } 103 | } 104 | 105 | func main() { 106 | /* setup gRPC service */ 107 | lis, err := net.Listen("tcp", ":50051") 108 | if err != nil { 109 | log.Fatalf("failed to listen: %v\n", err) 110 | } 111 | go func() { 112 | s := grpc.NewServer() 113 | pb.RegisterBitTorrentServer(s, &server{}) 114 | if err := s.Serve(lis); err != nil { 115 | log.Fatalf("failed to serve: %v\n", err) 116 | } 117 | }() 118 | 119 | time.Sleep(time.Minute) // wait elasticsearch to boot, temp solution 120 | log.Println("Connect to elastic search node") 121 | 122 | /* setup elastic search */ 123 | es, err := elastic.NewSimpleClient( 124 | elastic.SetURL("http://elastic:9200"), 125 | elastic.SetBasicAuth(os.Getenv("username"), os.Getenv("password")), 126 | ) 127 | if err != nil { 128 | log.Fatalf("Error creating the client: %s\n", err) 129 | } 130 | 131 | /* create index */ 132 | torrentMapping := `{ 133 | "mappings": { 134 | "properties": { 135 | "files": { 136 | "properties": { 137 | "length": { 138 | "type": "long" 139 | }, 140 | "path": { 141 | "type": "text", 142 | "analyzer": "standard", 143 | "search_analyzer": "standard" 144 | } 145 | } 146 | }, 147 | "infohash": { 148 | "type": "keyword" 149 | }, 150 | "length": { 151 | "type": "long" 152 | }, 153 | "name": { 154 | "type": "text", 155 | "analyzer": "standard", 156 | "search_analyzer": "standard" 157 | }, 158 | "name_suggest": { 159 | "type": "completion" 160 | } 161 | } 162 | } 163 | }` 164 | 165 | isIndexExist, err := es.IndexExists("torrent").Do(ctx) 166 | if err != nil { 167 | log.Fatalf("Cannot check if index exist: %s\n", err) 168 | } 169 | 170 | if !isIndexExist { 171 | _, err = es.CreateIndex("torrent").BodyString(torrentMapping).Do(ctx) 172 | if err != nil { 173 | log.Fatalf("Fail to create index: %s\n", err) 174 | } 175 | } 176 | 177 | bulkBody := es.Bulk(); 178 | 179 | for { 180 | value, err := rdb.BLPop(0*time.Second, "queue").Result() 181 | if err != nil { 182 | log.Fatalf("Failed to pull data from redis: %s\n", err) 183 | } 184 | Process(es, bulkBody, value[1]) 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /web/src/components/list.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import ListItem from "./listitem"; 4 | import ListBar from "./listbar" 5 | 6 | 7 | function humanFileSize(bytes, dp=1) { 8 | bytes = Math.abs(bytes) 9 | const thresh = 1000; 10 | 11 | if (Math.abs(bytes) < thresh) { 12 | return bytes + ' B'; 13 | } 14 | 15 | const units = ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; 16 | let u = -1; 17 | const r = 10**dp; 18 | 19 | do { 20 | bytes /= thresh; 21 | ++u; 22 | } while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1); 23 | 24 | return bytes.toFixed(dp) + ' ' + units[u]; 25 | } 26 | 27 | const RESULT_PER_PAGE = 20; 28 | 29 | export default class List extends React.Component { 30 | constructor(props) { 31 | super(props); 32 | this.state = { 33 | data: [], 34 | keyword: "", 35 | currentPage: 0, 36 | totalPage: 0, 37 | } 38 | }; 39 | 40 | searchByKeyword = (keyword, page=1) => { 41 | // edge case: no empty search 42 | if (keyword === "") { 43 | return 44 | } 45 | 46 | const requestOptions = { 47 | method: 'POST', 48 | headers: { 49 | 'Content-Type': 'application/json', 50 | 'Authorization': 'Basic ' + Buffer.from(process.env.REACT_APP_USERNAME + ':' + process.env.REACT_APP_PASSWORD).toString('base64'), 51 | }, 52 | body: JSON.stringify({ 53 | "query": { 54 | "multi_match": { 55 | "query" : keyword, 56 | "fields" : ["name^2", "files.path"] 57 | } 58 | }, 59 | "size": RESULT_PER_PAGE, 60 | "from": (page - 1) * RESULT_PER_PAGE, 61 | }) 62 | }; 63 | 64 | const handleFiles = (torrent) => { 65 | if (torrent.files) { 66 | let filteredFiles = torrent.files.filter(item => item.path); 67 | let files = filteredFiles.map(file => ({ 68 | fileName: file.path, 69 | size: humanFileSize(parseInt(file.length, 10)) 70 | })); 71 | let totalSize = humanFileSize(filteredFiles.map(item => 72 | parseInt(item['length'], 10)).reduce((acc, curr) => acc + Math.abs(curr))); 73 | 74 | 75 | if (files.length > 50) { 76 | let original_size = files.length; 77 | files = files.slice(0, 50); 78 | files.push({ 79 | fileName: (original_size - 50) + " more files", 80 | size: "..." 81 | }) 82 | } 83 | 84 | return [totalSize, files]; 85 | } 86 | 87 | return [humanFileSize(Math.abs(parseInt(torrent.length, 10))), [{ 88 | fileName: torrent.name, 89 | size: humanFileSize(Math.abs(parseInt(torrent.length, 10))) 90 | }]]; 91 | }; 92 | 93 | fetch('/api/torrent/_search', requestOptions) 94 | .then(response => response.json()) 95 | .then(data => this.setState({ 96 | data: data.hits.hits.map(item => ({ 97 | name: item._source.name, 98 | link: item._source.infohash, 99 | //files: handleFiles(item._source)[1], 100 | size: handleFiles(item._source)[0] 101 | })), 102 | keyword: keyword, 103 | totalPage: Math.ceil(data.hits.total.value / RESULT_PER_PAGE), 104 | currentPage: page, 105 | })); 106 | }; 107 | 108 | render() { 109 | // no result page 110 | if (this.state.keyword === "") { 111 | return ( 112 |
113 | ); 114 | } 115 | 116 | if (this.state.data.length === 0) { 117 | return ( 118 |

No result for "{this.state.keyword}"

119 | ) 120 | } 121 | 122 | return ( 123 |
124 |
125 | {this.state.data.map(item => 126 | 132 | )} 133 |
134 | 140 |
141 | ) 142 | }; 143 | } -------------------------------------------------------------------------------- /crawler/dht/bencode.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "strconv" 7 | "strings" 8 | "unicode" 9 | "unicode/utf8" 10 | ) 11 | 12 | // find returns the index of first target in data starting from `start`. 13 | // It returns -1 if target not found. 14 | func find(data []byte, start int, target rune) (index int) { 15 | index = bytes.IndexRune(data[start:], target) 16 | if index != -1 { 17 | return index + start 18 | } 19 | return index 20 | } 21 | 22 | // DecodeString decodes a string in the data. It returns a tuple 23 | // (decoded result, the end position, error). 24 | func DecodeString(data []byte, start int) ( 25 | result interface{}, index int, err error) { 26 | 27 | if start >= len(data) || data[start] < '0' || data[start] > '9' { 28 | err = errors.New("invalid string bencode") 29 | return 30 | } 31 | 32 | i := find(data, start, ':') 33 | if i == -1 { 34 | err = errors.New("':' not found when decode string") 35 | return 36 | } 37 | 38 | length, err := strconv.Atoi(string(data[start:i])) 39 | if err != nil { 40 | return 41 | } 42 | 43 | if length < 0 { 44 | err = errors.New("invalid length of string") 45 | return 46 | } 47 | 48 | index = i + 1 + length 49 | 50 | if index > len(data) || index < i+1 { 51 | err = errors.New("out of range") 52 | return 53 | } 54 | 55 | result = string(data[i+1 : index]) 56 | return 57 | } 58 | 59 | // DecodeInt decodes int value in the data. 60 | func DecodeInt(data []byte, start int) ( 61 | result interface{}, index int, err error) { 62 | 63 | if start >= len(data) || data[start] != 'i' { 64 | err = errors.New("invalid int bencode") 65 | return 66 | } 67 | 68 | index = find(data, start+1, 'e') 69 | 70 | if index == -1 { 71 | err = errors.New("':' not found when decode int") 72 | return 73 | } 74 | 75 | result, err = strconv.Atoi(string(data[start+1 : index])) 76 | if err != nil { 77 | return 78 | } 79 | index++ 80 | 81 | return 82 | } 83 | 84 | // decodeItem decodes an item of dict or list. 85 | func decodeItem(data []byte, i int) ( 86 | result interface{}, index int, err error) { 87 | 88 | var decodeFunc = []func([]byte, int) (interface{}, int, error){ 89 | DecodeString, DecodeInt, DecodeList, DecodeDict, 90 | } 91 | 92 | for _, f := range decodeFunc { 93 | result, index, err = f(data, i) 94 | if err == nil { 95 | return 96 | } 97 | } 98 | 99 | err = errors.New("invalid bencode when decode item") 100 | return 101 | } 102 | 103 | // DecodeList decodes a list value. 104 | func DecodeList(data []byte, start int) ( 105 | result interface{}, index int, err error) { 106 | 107 | if start >= len(data) || data[start] != 'l' { 108 | err = errors.New("invalid list bencode") 109 | return 110 | } 111 | 112 | var item interface{} 113 | r := make([]interface{}, 0, 8) 114 | 115 | index = start + 1 116 | for index < len(data) { 117 | char, _ := utf8.DecodeRune(data[index:]) 118 | if char == 'e' { 119 | break 120 | } 121 | 122 | item, index, err = decodeItem(data, index) 123 | if err != nil { 124 | return 125 | } 126 | r = append(r, item) 127 | } 128 | 129 | if index == len(data) { 130 | err = errors.New("'e' not found when decode list") 131 | return 132 | } 133 | index++ 134 | 135 | result = r 136 | return 137 | } 138 | 139 | // DecodeDict decodes a map value. 140 | func DecodeDict(data []byte, start int) ( 141 | result interface{}, index int, err error) { 142 | 143 | if start >= len(data) || data[start] != 'd' { 144 | err = errors.New("invalid dict bencode") 145 | return 146 | } 147 | 148 | var item, key interface{} 149 | r := make(map[string]interface{}) 150 | 151 | index = start + 1 152 | for index < len(data) { 153 | char, _ := utf8.DecodeRune(data[index:]) 154 | if char == 'e' { 155 | break 156 | } 157 | 158 | if !unicode.IsDigit(char) { 159 | err = errors.New("invalid dict bencode") 160 | return 161 | } 162 | 163 | key, index, err = DecodeString(data, index) 164 | if err != nil { 165 | return 166 | } 167 | 168 | if index >= len(data) { 169 | err = errors.New("out of range") 170 | return 171 | } 172 | 173 | item, index, err = decodeItem(data, index) 174 | if err != nil { 175 | return 176 | } 177 | 178 | r[key.(string)] = item 179 | } 180 | 181 | if index == len(data) { 182 | err = errors.New("'e' not found when decode dict") 183 | return 184 | } 185 | index++ 186 | 187 | result = r 188 | return 189 | } 190 | 191 | // Decode decodes a bencoded string to string, int, list or map. 192 | func Decode(data []byte) (result interface{}, err error) { 193 | result, _, err = decodeItem(data, 0) 194 | return 195 | } 196 | 197 | // EncodeString encodes a string value. 198 | func EncodeString(data string) string { 199 | return strings.Join([]string{strconv.Itoa(len(data)), data}, ":") 200 | } 201 | 202 | // EncodeInt encodes a int value. 203 | func EncodeInt(data int) string { 204 | return strings.Join([]string{"i", strconv.Itoa(data), "e"}, "") 205 | } 206 | 207 | // EncodeItem encodes an item of dict or list. 208 | func encodeItem(data interface{}) (item string) { 209 | switch v := data.(type) { 210 | case string: 211 | item = EncodeString(v) 212 | case int: 213 | item = EncodeInt(v) 214 | case []interface{}: 215 | item = EncodeList(v) 216 | case map[string]interface{}: 217 | item = EncodeDict(v) 218 | default: 219 | panic("invalid type when encode item") 220 | } 221 | return 222 | } 223 | 224 | // EncodeList encodes a list value. 225 | func EncodeList(data []interface{}) string { 226 | result := make([]string, len(data)) 227 | 228 | for i, item := range data { 229 | result[i] = encodeItem(item) 230 | } 231 | 232 | return strings.Join([]string{"l", strings.Join(result, ""), "e"}, "") 233 | } 234 | 235 | // EncodeDict encodes a dict value. 236 | func EncodeDict(data map[string]interface{}) string { 237 | result, i := make([]string, len(data)), 0 238 | 239 | for key, val := range data { 240 | result[i] = strings.Join( 241 | []string{EncodeString(key), encodeItem(val)}, 242 | "") 243 | i++ 244 | } 245 | 246 | return strings.Join([]string{"d", strings.Join(result, ""), "e"}, "") 247 | } 248 | 249 | // Encode encodes a string, int, dict or list value to a bencoded string. 250 | func Encode(data interface{}) string { 251 | switch v := data.(type) { 252 | case string: 253 | return EncodeString(v) 254 | case int: 255 | return EncodeInt(v) 256 | case []interface{}: 257 | return EncodeList(v) 258 | case map[string]interface{}: 259 | return EncodeDict(v) 260 | default: 261 | panic("invalid type when encode") 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /crawler/dht/container.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | ) 7 | 8 | type mapItem struct { 9 | key interface{} 10 | val interface{} 11 | } 12 | 13 | // syncedMap represents a goroutine-safe map. 14 | type syncedMap struct { 15 | *sync.RWMutex 16 | data map[interface{}]interface{} 17 | } 18 | 19 | // newSyncedMap returns a syncedMap pointer. 20 | func newSyncedMap() *syncedMap { 21 | return &syncedMap{ 22 | RWMutex: &sync.RWMutex{}, 23 | data: make(map[interface{}]interface{}), 24 | } 25 | } 26 | 27 | // Get returns the value mapped to key. 28 | func (smap *syncedMap) Get(key interface{}) (val interface{}, ok bool) { 29 | smap.RLock() 30 | defer smap.RUnlock() 31 | 32 | val, ok = smap.data[key] 33 | return 34 | } 35 | 36 | // Has returns whether the syncedMap contains the key. 37 | func (smap *syncedMap) Has(key interface{}) bool { 38 | _, ok := smap.Get(key) 39 | return ok 40 | } 41 | 42 | // Set sets pair {key: val}. 43 | func (smap *syncedMap) Set(key interface{}, val interface{}) { 44 | smap.Lock() 45 | defer smap.Unlock() 46 | 47 | smap.data[key] = val 48 | } 49 | 50 | // Delete deletes the key in the map. 51 | func (smap *syncedMap) Delete(key interface{}) { 52 | smap.Lock() 53 | defer smap.Unlock() 54 | 55 | delete(smap.data, key) 56 | } 57 | 58 | // DeleteMulti deletes keys in batch. 59 | func (smap *syncedMap) DeleteMulti(keys []interface{}) { 60 | smap.Lock() 61 | defer smap.Unlock() 62 | 63 | for _, key := range keys { 64 | delete(smap.data, key) 65 | } 66 | } 67 | 68 | // Clear resets the data. 69 | func (smap *syncedMap) Clear() { 70 | smap.Lock() 71 | defer smap.Unlock() 72 | 73 | smap.data = make(map[interface{}]interface{}) 74 | } 75 | 76 | // Iter returns a chan which output all items. 77 | func (smap *syncedMap) Iter() <-chan mapItem { 78 | ch := make(chan mapItem) 79 | go func() { 80 | smap.RLock() 81 | for key, val := range smap.data { 82 | ch <- mapItem{ 83 | key: key, 84 | val: val, 85 | } 86 | } 87 | smap.RUnlock() 88 | close(ch) 89 | }() 90 | return ch 91 | } 92 | 93 | // Len returns the length of syncedMap. 94 | func (smap *syncedMap) Len() int { 95 | smap.RLock() 96 | defer smap.RUnlock() 97 | 98 | return len(smap.data) 99 | } 100 | 101 | // syncedList represents a goroutine-safe list. 102 | type syncedList struct { 103 | *sync.RWMutex 104 | queue *list.List 105 | } 106 | 107 | // newSyncedList returns a syncedList pointer. 108 | func newSyncedList() *syncedList { 109 | return &syncedList{ 110 | RWMutex: &sync.RWMutex{}, 111 | queue: list.New(), 112 | } 113 | } 114 | 115 | // Front returns the first element of slist. 116 | func (slist *syncedList) Front() *list.Element { 117 | slist.RLock() 118 | defer slist.RUnlock() 119 | 120 | return slist.queue.Front() 121 | } 122 | 123 | // Back returns the last element of slist. 124 | func (slist *syncedList) Back() *list.Element { 125 | slist.RLock() 126 | defer slist.RUnlock() 127 | 128 | return slist.queue.Back() 129 | } 130 | 131 | // PushFront pushs an element to the head of slist. 132 | func (slist *syncedList) PushFront(v interface{}) *list.Element { 133 | slist.Lock() 134 | defer slist.Unlock() 135 | 136 | return slist.queue.PushFront(v) 137 | } 138 | 139 | // PushBack pushs an element to the tail of slist. 140 | func (slist *syncedList) PushBack(v interface{}) *list.Element { 141 | slist.Lock() 142 | defer slist.Unlock() 143 | 144 | return slist.queue.PushBack(v) 145 | } 146 | 147 | // InsertBefore inserts v before mark. 148 | func (slist *syncedList) InsertBefore( 149 | v interface{}, mark *list.Element) *list.Element { 150 | 151 | slist.Lock() 152 | defer slist.Unlock() 153 | 154 | return slist.queue.InsertBefore(v, mark) 155 | } 156 | 157 | // InsertAfter inserts v after mark. 158 | func (slist *syncedList) InsertAfter( 159 | v interface{}, mark *list.Element) *list.Element { 160 | 161 | slist.Lock() 162 | defer slist.Unlock() 163 | 164 | return slist.queue.InsertAfter(v, mark) 165 | } 166 | 167 | // Remove removes e from the slist. 168 | func (slist *syncedList) Remove(e *list.Element) interface{} { 169 | slist.Lock() 170 | defer slist.Unlock() 171 | 172 | return slist.queue.Remove(e) 173 | } 174 | 175 | // Clear resets the list queue. 176 | func (slist *syncedList) Clear() { 177 | slist.Lock() 178 | defer slist.Unlock() 179 | 180 | slist.queue.Init() 181 | } 182 | 183 | // Len returns length of the slist. 184 | func (slist *syncedList) Len() int { 185 | slist.RLock() 186 | defer slist.RUnlock() 187 | 188 | return slist.queue.Len() 189 | } 190 | 191 | // Iter returns a chan which output all elements. 192 | func (slist *syncedList) Iter() <-chan *list.Element { 193 | ch := make(chan *list.Element) 194 | go func() { 195 | slist.RLock() 196 | for e := slist.queue.Front(); e != nil; e = e.Next() { 197 | ch <- e 198 | } 199 | slist.RUnlock() 200 | close(ch) 201 | }() 202 | return ch 203 | } 204 | 205 | // KeyedDeque represents a keyed deque. 206 | type keyedDeque struct { 207 | *sync.RWMutex 208 | *syncedList 209 | index map[interface{}]*list.Element 210 | invertedIndex map[*list.Element]interface{} 211 | } 212 | 213 | // newKeyedDeque returns a newKeyedDeque pointer. 214 | func newKeyedDeque() *keyedDeque { 215 | return &keyedDeque{ 216 | RWMutex: &sync.RWMutex{}, 217 | syncedList: newSyncedList(), 218 | index: make(map[interface{}]*list.Element), 219 | invertedIndex: make(map[*list.Element]interface{}), 220 | } 221 | } 222 | 223 | // Push pushs a keyed-value to the end of deque. 224 | func (deque *keyedDeque) Push(key interface{}, val interface{}) { 225 | deque.Lock() 226 | defer deque.Unlock() 227 | 228 | if e, ok := deque.index[key]; ok { 229 | deque.syncedList.Remove(e) 230 | } 231 | deque.index[key] = deque.syncedList.PushBack(val) 232 | deque.invertedIndex[deque.index[key]] = key 233 | } 234 | 235 | // Get returns the keyed value. 236 | func (deque *keyedDeque) Get(key interface{}) (*list.Element, bool) { 237 | deque.RLock() 238 | defer deque.RUnlock() 239 | 240 | v, ok := deque.index[key] 241 | return v, ok 242 | } 243 | 244 | // Has returns whether key already exists. 245 | func (deque *keyedDeque) HasKey(key interface{}) bool { 246 | _, ok := deque.Get(key) 247 | return ok 248 | } 249 | 250 | // Delete deletes a value named key. 251 | func (deque *keyedDeque) Delete(key interface{}) (v interface{}) { 252 | deque.RLock() 253 | e, ok := deque.index[key] 254 | deque.RUnlock() 255 | 256 | deque.Lock() 257 | defer deque.Unlock() 258 | 259 | if ok { 260 | v = deque.syncedList.Remove(e) 261 | delete(deque.index, key) 262 | delete(deque.invertedIndex, e) 263 | } 264 | 265 | return 266 | } 267 | 268 | // Removes overwrites list.List.Remove. 269 | func (deque *keyedDeque) Remove(e *list.Element) (v interface{}) { 270 | deque.RLock() 271 | key, ok := deque.invertedIndex[e] 272 | deque.RUnlock() 273 | 274 | if ok { 275 | v = deque.Delete(key) 276 | } 277 | 278 | return 279 | } 280 | 281 | // Clear resets the deque. 282 | func (deque *keyedDeque) Clear() { 283 | deque.Lock() 284 | defer deque.Unlock() 285 | 286 | deque.syncedList.Clear() 287 | deque.index = make(map[interface{}]*list.Element) 288 | deque.invertedIndex = make(map[*list.Element]interface{}) 289 | } 290 | -------------------------------------------------------------------------------- /crawler/dht/dht.go: -------------------------------------------------------------------------------- 1 | // Package dht implements the bittorrent dht protocol. For more information 2 | // see http://www.bittorrent.org/beps/bep_0005.html. 3 | package dht 4 | 5 | import ( 6 | "encoding/hex" 7 | "errors" 8 | "math" 9 | "net" 10 | "time" 11 | ) 12 | 13 | const ( 14 | // StandardMode follows the standard protocol 15 | StandardMode = iota 16 | // CrawlMode for crawling the dht network. 17 | CrawlMode 18 | ) 19 | 20 | var ( 21 | // ErrNotReady is the error when DHT is not initialized. 22 | ErrNotReady = errors.New("dht is not ready") 23 | // ErrOnGetPeersResponseNotSet is the error that config 24 | // OnGetPeersResponseNotSet is not set when call dht.GetPeers. 25 | ErrOnGetPeersResponseNotSet = errors.New("OnGetPeersResponse is not set") 26 | ) 27 | 28 | // Config represents the configure of dht. 29 | type Config struct { 30 | // in mainline dht, k = 8 31 | K int 32 | // for crawling mode, we put all nodes in one bucket, so KBucketSize may 33 | // not be K 34 | KBucketSize int 35 | // candidates are udp, udp4, udp6 36 | Network string 37 | // format is `ip:port` 38 | Address string 39 | // the prime nodes through which we can join in dht network 40 | PrimeNodes []string 41 | // the kbucket expired duration 42 | KBucketExpiredAfter time.Duration 43 | // the node expired duration 44 | NodeExpriedAfter time.Duration 45 | // how long it checks whether the bucket is expired 46 | CheckKBucketPeriod time.Duration 47 | // peer token expired duration 48 | TokenExpiredAfter time.Duration 49 | // the max transaction id 50 | MaxTransactionCursor uint64 51 | // how many nodes routing table can hold 52 | MaxNodes int 53 | // callback when got get_peers request 54 | OnGetPeers func(string, string, int) 55 | // callback when receive get_peers response 56 | OnGetPeersResponse func(string, *Peer) 57 | // callback when got announce_peer request 58 | OnAnnouncePeer func(string, string, int) 59 | // blcoked ips 60 | BlockedIPs []string 61 | // blacklist size 62 | BlackListMaxSize int 63 | // StandardMode or CrawlMode 64 | Mode int 65 | // the times it tries when send fails 66 | Try int 67 | // the size of packet need to be dealt with 68 | PacketJobLimit int 69 | // the size of packet handler 70 | PacketWorkerLimit int 71 | // the nodes num to be fresh in a kbucket 72 | RefreshNodeNum int 73 | } 74 | 75 | // NewStandardConfig returns a Config pointer with default values. 76 | func NewStandardConfig() *Config { 77 | return &Config{ 78 | K: 8, 79 | KBucketSize: 8, 80 | Network: "udp4", 81 | Address: ":6881", 82 | PrimeNodes: []string{ 83 | "router.bittorrent.com:6881", 84 | "router.utorrent.com:6881", 85 | "dht.transmissionbt.com:6881", 86 | }, 87 | NodeExpriedAfter: time.Duration(time.Minute * 15), 88 | KBucketExpiredAfter: time.Duration(time.Minute * 15), 89 | CheckKBucketPeriod: time.Duration(time.Second * 30), 90 | TokenExpiredAfter: time.Duration(time.Minute * 10), 91 | MaxTransactionCursor: math.MaxUint32, 92 | MaxNodes: 5000, 93 | BlockedIPs: make([]string, 0), 94 | BlackListMaxSize: 65536, 95 | Try: 2, 96 | Mode: StandardMode, 97 | PacketJobLimit: 1024, 98 | PacketWorkerLimit: 256, 99 | RefreshNodeNum: 8, 100 | } 101 | } 102 | 103 | // NewCrawlConfig returns a config in crawling mode. 104 | func NewCrawlConfig() *Config { 105 | config := NewStandardConfig() 106 | config.NodeExpriedAfter = 0 107 | config.KBucketExpiredAfter = 0 108 | config.CheckKBucketPeriod = time.Second * 5 109 | config.KBucketSize = math.MaxInt32 110 | config.Mode = CrawlMode 111 | config.RefreshNodeNum = 256 112 | 113 | return config 114 | } 115 | 116 | // DHT represents a DHT node. 117 | type DHT struct { 118 | *Config 119 | node *node 120 | conn *net.UDPConn 121 | routingTable *routingTable 122 | transactionManager *transactionManager 123 | peersManager *peersManager 124 | tokenManager *tokenManager 125 | blackList *blackList 126 | Ready bool 127 | packets chan packet 128 | workerTokens chan struct{} 129 | } 130 | 131 | // New returns a DHT pointer. If config is nil, then config will be set to 132 | // the default config. 133 | func New(config *Config) *DHT { 134 | if config == nil { 135 | config = NewStandardConfig() 136 | } 137 | 138 | node, err := newNode(randomString(20), config.Network, config.Address) 139 | if err != nil { 140 | panic(err) 141 | } 142 | 143 | d := &DHT{ 144 | Config: config, 145 | node: node, 146 | blackList: newBlackList(config.BlackListMaxSize), 147 | packets: make(chan packet, config.PacketJobLimit), 148 | workerTokens: make(chan struct{}, config.PacketWorkerLimit), 149 | } 150 | 151 | for _, ip := range config.BlockedIPs { 152 | d.blackList.insert(ip, -1) 153 | } 154 | 155 | go func() { 156 | for _, ip := range getLocalIPs() { 157 | d.blackList.insert(ip, -1) 158 | } 159 | 160 | ip, err := getRemoteIP() 161 | if err != nil { 162 | d.blackList.insert(ip, -1) 163 | } 164 | }() 165 | 166 | return d 167 | } 168 | 169 | // IsStandardMode returns whether mode is StandardMode. 170 | func (dht *DHT) IsStandardMode() bool { 171 | return dht.Mode == StandardMode 172 | } 173 | 174 | // IsCrawlMode returns whether mode is CrawlMode. 175 | func (dht *DHT) IsCrawlMode() bool { 176 | return dht.Mode == CrawlMode 177 | } 178 | 179 | // init initializes global varables. 180 | func (dht *DHT) init() { 181 | listener, err := net.ListenPacket(dht.Network, dht.Address) 182 | if err != nil { 183 | panic(err) 184 | } 185 | 186 | dht.conn = listener.(*net.UDPConn) 187 | dht.routingTable = newRoutingTable(dht.KBucketSize, dht) 188 | dht.peersManager = newPeersManager(dht) 189 | dht.tokenManager = newTokenManager(dht.TokenExpiredAfter, dht) 190 | dht.transactionManager = newTransactionManager( 191 | dht.MaxTransactionCursor, dht) 192 | 193 | go dht.transactionManager.run() 194 | go dht.tokenManager.clear() 195 | go dht.blackList.clear() 196 | } 197 | 198 | // join makes current node join the dht network. 199 | func (dht *DHT) join() { 200 | for _, addr := range dht.PrimeNodes { 201 | raddr, err := net.ResolveUDPAddr(dht.Network, addr) 202 | if err != nil { 203 | continue 204 | } 205 | 206 | // NOTE: Temporary node has NOT node id. 207 | dht.transactionManager.findNode( 208 | &node{addr: raddr}, 209 | dht.node.id.RawString(), 210 | ) 211 | } 212 | } 213 | 214 | // listen receives message from udp. 215 | func (dht *DHT) listen() { 216 | go func() { 217 | buff := make([]byte, 8192) 218 | for { 219 | n, raddr, err := dht.conn.ReadFromUDP(buff) 220 | if err != nil { 221 | continue 222 | } 223 | 224 | dht.packets <- packet{buff[:n], raddr} 225 | } 226 | }() 227 | } 228 | 229 | // id returns a id near to target if target is not null, otherwise it returns 230 | // the dht's node id. 231 | func (dht *DHT) id(target string) string { 232 | if dht.IsStandardMode() || target == "" { 233 | return dht.node.id.RawString() 234 | } 235 | return target[:15] + dht.node.id.RawString()[15:] 236 | } 237 | 238 | // GetPeers returns peers who have announced having infoHash. 239 | func (dht *DHT) GetPeers(infoHash string) error { 240 | if !dht.Ready { 241 | return ErrNotReady 242 | } 243 | 244 | if dht.OnGetPeersResponse == nil { 245 | return ErrOnGetPeersResponseNotSet 246 | } 247 | 248 | if len(infoHash) == 40 { 249 | data, err := hex.DecodeString(infoHash) 250 | if err != nil { 251 | return err 252 | } 253 | infoHash = string(data) 254 | } 255 | 256 | neighbors := dht.routingTable.GetNeighbors( 257 | newBitmapFromString(infoHash), dht.routingTable.Len()) 258 | 259 | for _, no := range neighbors { 260 | dht.transactionManager.getPeers(no, infoHash) 261 | } 262 | 263 | return nil 264 | } 265 | 266 | // Run starts the dht. 267 | func (dht *DHT) Run() { 268 | dht.init() 269 | dht.listen() 270 | dht.join() 271 | 272 | dht.Ready = true 273 | 274 | var pkt packet 275 | tick := time.Tick(dht.CheckKBucketPeriod) 276 | 277 | for { 278 | select { 279 | case pkt = <-dht.packets: 280 | handle(dht, pkt) 281 | case <-tick: 282 | if dht.routingTable.Len() == 0 { 283 | dht.join() 284 | } else if dht.transactionManager.len() == 0 { 285 | go dht.routingTable.Fresh() 286 | } 287 | } 288 | } 289 | } 290 | -------------------------------------------------------------------------------- /crawler/dht/peerwire.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "encoding/binary" 7 | "errors" 8 | "io" 9 | "io/ioutil" 10 | "net" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | const ( 16 | // REQUEST represents request message type 17 | REQUEST = iota 18 | // DATA represents data message type 19 | DATA 20 | // REJECT represents reject message type 21 | REJECT 22 | ) 23 | 24 | const ( 25 | // BLOCK is 2 ^ 14 26 | BLOCK = 16384 27 | // MaxMetadataSize represents the max medata it can accept 28 | MaxMetadataSize = BLOCK * 1000 29 | // EXTENDED represents it is a extended message 30 | EXTENDED = 20 31 | // HANDSHAKE represents handshake bit 32 | HANDSHAKE = 0 33 | ) 34 | 35 | var handshakePrefix = []byte{ 36 | 19, 66, 105, 116, 84, 111, 114, 114, 101, 110, 116, 32, 112, 114, 37 | 111, 116, 111, 99, 111, 108, 0, 0, 0, 0, 0, 16, 0, 1, 38 | } 39 | 40 | // read reads size-length bytes from conn to data. 41 | func read(conn *net.TCPConn, size int, data *bytes.Buffer) error { 42 | conn.SetReadDeadline(time.Now().Add(time.Second * 15)) 43 | 44 | n, err := io.CopyN(data, conn, int64(size)) 45 | if err != nil || n != int64(size) { 46 | return errors.New("read error") 47 | } 48 | return nil 49 | } 50 | 51 | // readMessage gets a message from the tcp connection. 52 | func readMessage(conn *net.TCPConn, data *bytes.Buffer) ( 53 | length int, err error) { 54 | 55 | if err = read(conn, 4, data); err != nil { 56 | return 57 | } 58 | 59 | length = int(bytes2int(data.Next(4))) 60 | if length == 0 { 61 | return 62 | } 63 | 64 | if err = read(conn, length, data); err != nil { 65 | return 66 | } 67 | return 68 | } 69 | 70 | // sendMessage sends data to the connection. 71 | func sendMessage(conn *net.TCPConn, data []byte) error { 72 | length := int32(len(data)) 73 | 74 | buffer := bytes.NewBuffer(nil) 75 | binary.Write(buffer, binary.BigEndian, length) 76 | 77 | conn.SetWriteDeadline(time.Now().Add(time.Second * 10)) 78 | _, err := conn.Write(append(buffer.Bytes(), data...)) 79 | return err 80 | } 81 | 82 | // sendHandshake sends handshake message to conn. 83 | func sendHandshake(conn *net.TCPConn, infoHash, peerID []byte) error { 84 | data := make([]byte, 68) 85 | copy(data[:28], handshakePrefix) 86 | copy(data[28:48], infoHash) 87 | copy(data[48:], peerID) 88 | 89 | conn.SetWriteDeadline(time.Now().Add(time.Second * 10)) 90 | _, err := conn.Write(data) 91 | return err 92 | } 93 | 94 | // onHandshake handles the handshake response. 95 | func onHandshake(data []byte) (err error) { 96 | if !(bytes.Equal(handshakePrefix[:20], data[:20]) && data[25]&0x10 != 0) { 97 | err = errors.New("invalid handshake response") 98 | } 99 | return 100 | } 101 | 102 | // sendExtHandshake requests for the ut_metadata and metadata_size. 103 | func sendExtHandshake(conn *net.TCPConn) error { 104 | data := append( 105 | []byte{EXTENDED, HANDSHAKE}, 106 | Encode(map[string]interface{}{ 107 | "m": map[string]interface{}{"ut_metadata": 1}, 108 | })..., 109 | ) 110 | 111 | return sendMessage(conn, data) 112 | } 113 | 114 | // getUTMetaSize returns the ut_metadata and metadata_size. 115 | func getUTMetaSize(data []byte) ( 116 | utMetadata int, metadataSize int, err error) { 117 | 118 | v, err := Decode(data) 119 | if err != nil { 120 | return 121 | } 122 | 123 | dict, ok := v.(map[string]interface{}) 124 | if !ok { 125 | err = errors.New("invalid dict") 126 | return 127 | } 128 | 129 | if err = ParseKeys( 130 | dict, [][]string{{"metadata_size", "int"}, {"m", "map"}}); err != nil { 131 | return 132 | } 133 | 134 | m := dict["m"].(map[string]interface{}) 135 | if err = ParseKey(m, "ut_metadata", "int"); err != nil { 136 | return 137 | } 138 | 139 | utMetadata = m["ut_metadata"].(int) 140 | metadataSize = dict["metadata_size"].(int) 141 | 142 | if metadataSize > MaxMetadataSize { 143 | err = errors.New("metadata_size too long") 144 | } 145 | return 146 | } 147 | 148 | // Request represents the request context. 149 | type Request struct { 150 | InfoHash []byte 151 | IP string 152 | Port int 153 | } 154 | 155 | // Response contains the request context and the metadata info. 156 | type Response struct { 157 | Request 158 | MetadataInfo []byte 159 | } 160 | 161 | // Wire represents the wire protocol. 162 | type Wire struct { 163 | blackList *blackList 164 | queue *syncedMap 165 | requests chan Request 166 | responses chan Response 167 | workerTokens chan struct{} 168 | } 169 | 170 | // NewWire returns a Wire pointer. 171 | // - blackListSize: the blacklist size 172 | // - requestQueueSize: the max requests it can buffers 173 | // - workerQueueSize: the max goroutine downloading workers 174 | func NewWire(blackListSize, requestQueueSize, workerQueueSize int) *Wire { 175 | return &Wire{ 176 | blackList: newBlackList(blackListSize), 177 | queue: newSyncedMap(), 178 | requests: make(chan Request, requestQueueSize), 179 | responses: make(chan Response, 1024), 180 | workerTokens: make(chan struct{}, workerQueueSize), 181 | } 182 | } 183 | 184 | // Request pushes the request to the queue. 185 | func (wire *Wire) Request(infoHash []byte, ip string, port int) { 186 | wire.requests <- Request{InfoHash: infoHash, IP: ip, Port: port} 187 | } 188 | 189 | // Response returns a chan of Response. 190 | func (wire *Wire) Response() <-chan Response { 191 | return wire.responses 192 | } 193 | 194 | // isDone returns whether the wire get all pieces of the metadata info. 195 | func (wire *Wire) isDone(pieces [][]byte) bool { 196 | for _, piece := range pieces { 197 | if len(piece) == 0 { 198 | return false 199 | } 200 | } 201 | return true 202 | } 203 | 204 | func (wire *Wire) requestPieces( 205 | conn *net.TCPConn, utMetadata int, metadataSize int, piecesNum int) { 206 | 207 | buffer := make([]byte, 1024) 208 | for i := 0; i < piecesNum; i++ { 209 | buffer[0] = EXTENDED 210 | buffer[1] = byte(utMetadata) 211 | 212 | msg := Encode(map[string]interface{}{ 213 | "msg_type": REQUEST, 214 | "piece": i, 215 | }) 216 | 217 | length := len(msg) + 2 218 | copy(buffer[2:length], msg) 219 | 220 | sendMessage(conn, buffer[:length]) 221 | } 222 | buffer = nil 223 | } 224 | 225 | // fetchMetadata fetchs medata info accroding to infohash from dht. 226 | func (wire *Wire) fetchMetadata(r Request) { 227 | var ( 228 | length int 229 | msgType byte 230 | piecesNum int 231 | pieces [][]byte 232 | utMetadata int 233 | metadataSize int 234 | ) 235 | 236 | defer func() { 237 | pieces = nil 238 | recover() 239 | }() 240 | 241 | infoHash := r.InfoHash 242 | address := genAddress(r.IP, r.Port) 243 | 244 | dial, err := net.DialTimeout("tcp", address, time.Second*15) 245 | if err != nil { 246 | wire.blackList.insert(r.IP, r.Port) 247 | return 248 | } 249 | conn := dial.(*net.TCPConn) 250 | conn.SetLinger(0) 251 | defer conn.Close() 252 | 253 | data := bytes.NewBuffer(nil) 254 | data.Grow(BLOCK) 255 | 256 | if sendHandshake(conn, infoHash, []byte(randomString(20))) != nil || 257 | read(conn, 68, data) != nil || 258 | onHandshake(data.Next(68)) != nil || 259 | sendExtHandshake(conn) != nil { 260 | return 261 | } 262 | 263 | for { 264 | length, err = readMessage(conn, data) 265 | if err != nil { 266 | return 267 | } 268 | 269 | if length == 0 { 270 | continue 271 | } 272 | 273 | msgType, err = data.ReadByte() 274 | if err != nil { 275 | return 276 | } 277 | 278 | switch msgType { 279 | case EXTENDED: 280 | extendedID, err := data.ReadByte() 281 | if err != nil { 282 | return 283 | } 284 | 285 | payload, err := ioutil.ReadAll(data) 286 | if err != nil { 287 | return 288 | } 289 | 290 | if extendedID == 0 { 291 | if pieces != nil { 292 | return 293 | } 294 | 295 | utMetadata, metadataSize, err = getUTMetaSize(payload) 296 | if err != nil { 297 | return 298 | } 299 | 300 | piecesNum = metadataSize / BLOCK 301 | if metadataSize%BLOCK != 0 { 302 | piecesNum++ 303 | } 304 | 305 | pieces = make([][]byte, piecesNum) 306 | go wire.requestPieces(conn, utMetadata, metadataSize, piecesNum) 307 | 308 | continue 309 | } 310 | 311 | if pieces == nil { 312 | return 313 | } 314 | 315 | d, index, err := DecodeDict(payload, 0) 316 | if err != nil { 317 | return 318 | } 319 | dict := d.(map[string]interface{}) 320 | 321 | if err = ParseKeys(dict, [][]string{ 322 | {"msg_type", "int"}, 323 | {"piece", "int"}}); err != nil { 324 | return 325 | } 326 | 327 | if dict["msg_type"].(int) != DATA { 328 | continue 329 | } 330 | 331 | piece := dict["piece"].(int) 332 | pieceLen := length - 2 - index 333 | 334 | if (piece != piecesNum-1 && pieceLen != BLOCK) || 335 | (piece == piecesNum-1 && pieceLen != metadataSize%BLOCK) { 336 | return 337 | } 338 | 339 | pieces[piece] = payload[index:] 340 | 341 | if wire.isDone(pieces) { 342 | metadataInfo := bytes.Join(pieces, nil) 343 | 344 | info := sha1.Sum(metadataInfo) 345 | if !bytes.Equal(infoHash, info[:]) { 346 | return 347 | } 348 | 349 | wire.responses <- Response{ 350 | Request: r, 351 | MetadataInfo: metadataInfo, 352 | } 353 | return 354 | } 355 | default: 356 | data.Reset() 357 | } 358 | } 359 | } 360 | 361 | // Run starts the peer wire protocol. 362 | func (wire *Wire) Run() { 363 | go wire.blackList.clear() 364 | 365 | for r := range wire.requests { 366 | wire.workerTokens <- struct{}{} 367 | 368 | go func(r Request) { 369 | defer func() { 370 | <-wire.workerTokens 371 | }() 372 | 373 | key := strings.Join([]string{ 374 | string(r.InfoHash), genAddress(r.IP, r.Port), 375 | }, ":") 376 | 377 | if len(r.InfoHash) != 20 || wire.blackList.in(r.IP, r.Port) || 378 | wire.queue.Has(key) { 379 | return 380 | } 381 | 382 | wire.fetchMetadata(r) 383 | }(r) 384 | } 385 | } 386 | -------------------------------------------------------------------------------- /service/bittorrent.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.25.0 4 | // protoc v3.12.4 5 | // source: bittorrent.proto 6 | 7 | package service 8 | 9 | import ( 10 | proto "github.com/golang/protobuf/proto" 11 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 12 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 13 | reflect "reflect" 14 | sync "sync" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | // This is a compile-time assertion that a sufficiently up-to-date version 25 | // of the legacy proto package is being used. 26 | const _ = proto.ProtoPackageIsVersion4 27 | 28 | type Empty struct { 29 | state protoimpl.MessageState 30 | sizeCache protoimpl.SizeCache 31 | unknownFields protoimpl.UnknownFields 32 | } 33 | 34 | func (x *Empty) Reset() { 35 | *x = Empty{} 36 | if protoimpl.UnsafeEnabled { 37 | mi := &file_bittorrent_proto_msgTypes[0] 38 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 39 | ms.StoreMessageInfo(mi) 40 | } 41 | } 42 | 43 | func (x *Empty) String() string { 44 | return protoimpl.X.MessageStringOf(x) 45 | } 46 | 47 | func (*Empty) ProtoMessage() {} 48 | 49 | func (x *Empty) ProtoReflect() protoreflect.Message { 50 | mi := &file_bittorrent_proto_msgTypes[0] 51 | if protoimpl.UnsafeEnabled && x != nil { 52 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 53 | if ms.LoadMessageInfo() == nil { 54 | ms.StoreMessageInfo(mi) 55 | } 56 | return ms 57 | } 58 | return mi.MessageOf(x) 59 | } 60 | 61 | // Deprecated: Use Empty.ProtoReflect.Descriptor instead. 62 | func (*Empty) Descriptor() ([]byte, []int) { 63 | return file_bittorrent_proto_rawDescGZIP(), []int{0} 64 | } 65 | 66 | type BitTorrent struct { 67 | state protoimpl.MessageState 68 | sizeCache protoimpl.SizeCache 69 | unknownFields protoimpl.UnknownFields 70 | 71 | Infohash string `protobuf:"bytes,1,opt,name=infohash,proto3" json:"infohash,omitempty"` 72 | Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` 73 | Files []*File `protobuf:"bytes,3,rep,name=files,proto3" json:"files,omitempty"` 74 | Length int32 `protobuf:"varint,4,opt,name=length,proto3" json:"length,omitempty"` 75 | } 76 | 77 | func (x *BitTorrent) Reset() { 78 | *x = BitTorrent{} 79 | if protoimpl.UnsafeEnabled { 80 | mi := &file_bittorrent_proto_msgTypes[1] 81 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 82 | ms.StoreMessageInfo(mi) 83 | } 84 | } 85 | 86 | func (x *BitTorrent) String() string { 87 | return protoimpl.X.MessageStringOf(x) 88 | } 89 | 90 | func (*BitTorrent) ProtoMessage() {} 91 | 92 | func (x *BitTorrent) ProtoReflect() protoreflect.Message { 93 | mi := &file_bittorrent_proto_msgTypes[1] 94 | if protoimpl.UnsafeEnabled && x != nil { 95 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 96 | if ms.LoadMessageInfo() == nil { 97 | ms.StoreMessageInfo(mi) 98 | } 99 | return ms 100 | } 101 | return mi.MessageOf(x) 102 | } 103 | 104 | // Deprecated: Use BitTorrent.ProtoReflect.Descriptor instead. 105 | func (*BitTorrent) Descriptor() ([]byte, []int) { 106 | return file_bittorrent_proto_rawDescGZIP(), []int{1} 107 | } 108 | 109 | func (x *BitTorrent) GetInfohash() string { 110 | if x != nil { 111 | return x.Infohash 112 | } 113 | return "" 114 | } 115 | 116 | func (x *BitTorrent) GetName() string { 117 | if x != nil { 118 | return x.Name 119 | } 120 | return "" 121 | } 122 | 123 | func (x *BitTorrent) GetFiles() []*File { 124 | if x != nil { 125 | return x.Files 126 | } 127 | return nil 128 | } 129 | 130 | func (x *BitTorrent) GetLength() int32 { 131 | if x != nil { 132 | return x.Length 133 | } 134 | return 0 135 | } 136 | 137 | type File struct { 138 | state protoimpl.MessageState 139 | sizeCache protoimpl.SizeCache 140 | unknownFields protoimpl.UnknownFields 141 | 142 | Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` 143 | Length int32 `protobuf:"varint,2,opt,name=length,proto3" json:"length,omitempty"` 144 | } 145 | 146 | func (x *File) Reset() { 147 | *x = File{} 148 | if protoimpl.UnsafeEnabled { 149 | mi := &file_bittorrent_proto_msgTypes[2] 150 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 151 | ms.StoreMessageInfo(mi) 152 | } 153 | } 154 | 155 | func (x *File) String() string { 156 | return protoimpl.X.MessageStringOf(x) 157 | } 158 | 159 | func (*File) ProtoMessage() {} 160 | 161 | func (x *File) ProtoReflect() protoreflect.Message { 162 | mi := &file_bittorrent_proto_msgTypes[2] 163 | if protoimpl.UnsafeEnabled && x != nil { 164 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 165 | if ms.LoadMessageInfo() == nil { 166 | ms.StoreMessageInfo(mi) 167 | } 168 | return ms 169 | } 170 | return mi.MessageOf(x) 171 | } 172 | 173 | // Deprecated: Use File.ProtoReflect.Descriptor instead. 174 | func (*File) Descriptor() ([]byte, []int) { 175 | return file_bittorrent_proto_rawDescGZIP(), []int{2} 176 | } 177 | 178 | func (x *File) GetPath() string { 179 | if x != nil { 180 | return x.Path 181 | } 182 | return "" 183 | } 184 | 185 | func (x *File) GetLength() int32 { 186 | if x != nil { 187 | return x.Length 188 | } 189 | return 0 190 | } 191 | 192 | var File_bittorrent_proto protoreflect.FileDescriptor 193 | 194 | var file_bittorrent_proto_rawDesc = []byte{ 195 | 0x0a, 0x10, 0x62, 0x69, 0x74, 0x74, 0x6f, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 196 | 0x74, 0x6f, 0x12, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x07, 0x0a, 0x05, 0x45, 197 | 0x6d, 0x70, 0x74, 0x79, 0x22, 0x79, 0x0a, 0x0a, 0x42, 0x69, 0x74, 0x54, 0x6f, 0x72, 0x72, 0x65, 198 | 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x66, 0x6f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 199 | 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x69, 0x6e, 0x66, 0x6f, 0x68, 0x61, 0x73, 0x68, 0x12, 0x12, 200 | 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 201 | 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 202 | 0x0b, 0x32, 0x0d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 203 | 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 204 | 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 205 | 0x32, 0x0a, 0x04, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 206 | 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6c, 207 | 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6c, 0x65, 0x6e, 208 | 0x67, 0x74, 0x68, 0x32, 0x3b, 0x0a, 0x0a, 0x62, 0x69, 0x74, 0x54, 0x6f, 0x72, 0x72, 0x65, 0x6e, 209 | 0x74, 0x12, 0x2d, 0x0a, 0x04, 0x73, 0x65, 0x6e, 0x64, 0x12, 0x13, 0x2e, 0x73, 0x65, 0x72, 0x76, 210 | 0x69, 0x63, 0x65, 0x2e, 0x42, 0x69, 0x74, 0x54, 0x6f, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x1a, 0x0e, 211 | 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 212 | 0x42, 0x1b, 0x5a, 0x19, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x4f, 213 | 0x6c, 0x61, 0x6d, 0x65, 0x6e, 0x74, 0x2f, 0x67, 0x44, 0x48, 0x54, 0x70, 0x72, 0x62, 0x06, 0x70, 214 | 0x72, 0x6f, 0x74, 0x6f, 0x33, 215 | } 216 | 217 | var ( 218 | file_bittorrent_proto_rawDescOnce sync.Once 219 | file_bittorrent_proto_rawDescData = file_bittorrent_proto_rawDesc 220 | ) 221 | 222 | func file_bittorrent_proto_rawDescGZIP() []byte { 223 | file_bittorrent_proto_rawDescOnce.Do(func() { 224 | file_bittorrent_proto_rawDescData = protoimpl.X.CompressGZIP(file_bittorrent_proto_rawDescData) 225 | }) 226 | return file_bittorrent_proto_rawDescData 227 | } 228 | 229 | var file_bittorrent_proto_msgTypes = make([]protoimpl.MessageInfo, 3) 230 | var file_bittorrent_proto_goTypes = []interface{}{ 231 | (*Empty)(nil), // 0: service.Empty 232 | (*BitTorrent)(nil), // 1: service.BitTorrent 233 | (*File)(nil), // 2: service.File 234 | } 235 | var file_bittorrent_proto_depIdxs = []int32{ 236 | 2, // 0: service.BitTorrent.files:type_name -> service.File 237 | 1, // 1: service.bitTorrent.send:input_type -> service.BitTorrent 238 | 0, // 2: service.bitTorrent.send:output_type -> service.Empty 239 | 2, // [2:3] is the sub-list for method output_type 240 | 1, // [1:2] is the sub-list for method input_type 241 | 1, // [1:1] is the sub-list for extension type_name 242 | 1, // [1:1] is the sub-list for extension extendee 243 | 0, // [0:1] is the sub-list for field type_name 244 | } 245 | 246 | func init() { file_bittorrent_proto_init() } 247 | func file_bittorrent_proto_init() { 248 | if File_bittorrent_proto != nil { 249 | return 250 | } 251 | if !protoimpl.UnsafeEnabled { 252 | file_bittorrent_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 253 | switch v := v.(*Empty); i { 254 | case 0: 255 | return &v.state 256 | case 1: 257 | return &v.sizeCache 258 | case 2: 259 | return &v.unknownFields 260 | default: 261 | return nil 262 | } 263 | } 264 | file_bittorrent_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 265 | switch v := v.(*BitTorrent); i { 266 | case 0: 267 | return &v.state 268 | case 1: 269 | return &v.sizeCache 270 | case 2: 271 | return &v.unknownFields 272 | default: 273 | return nil 274 | } 275 | } 276 | file_bittorrent_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 277 | switch v := v.(*File); i { 278 | case 0: 279 | return &v.state 280 | case 1: 281 | return &v.sizeCache 282 | case 2: 283 | return &v.unknownFields 284 | default: 285 | return nil 286 | } 287 | } 288 | } 289 | type x struct{} 290 | out := protoimpl.TypeBuilder{ 291 | File: protoimpl.DescBuilder{ 292 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 293 | RawDescriptor: file_bittorrent_proto_rawDesc, 294 | NumEnums: 0, 295 | NumMessages: 3, 296 | NumExtensions: 0, 297 | NumServices: 1, 298 | }, 299 | GoTypes: file_bittorrent_proto_goTypes, 300 | DependencyIndexes: file_bittorrent_proto_depIdxs, 301 | MessageInfos: file_bittorrent_proto_msgTypes, 302 | }.Build() 303 | File_bittorrent_proto = out.File 304 | file_bittorrent_proto_rawDesc = nil 305 | file_bittorrent_proto_goTypes = nil 306 | file_bittorrent_proto_depIdxs = nil 307 | } 308 | -------------------------------------------------------------------------------- /server/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 5 | github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60= 6 | github.com/Olament/gDHT v0.0.0-20200812202725-3771e54dc061 h1:I7DggatDHF3xfnO3Nm84Ni04JQ9rrvOknLUVc9p6/+s= 7 | github.com/Olament/gDHT v0.0.0-20200812202725-3771e54dc061/go.mod h1:5tTJYHofb4mxDzXsTk1uQPB1nrOxFn0KgAaH9kry7/k= 8 | github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= 9 | github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= 10 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 11 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 12 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 13 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 14 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 17 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 18 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 19 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 20 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 21 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 22 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 23 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 24 | github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 25 | github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g= 26 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 27 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 28 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 29 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 30 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 31 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 32 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 33 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 34 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 35 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 36 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 37 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 38 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 39 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 40 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 41 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 42 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 43 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 44 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 45 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 46 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 47 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 48 | github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= 49 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 50 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 51 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 52 | github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= 53 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 54 | github.com/olivere/elastic v1.0.1 h1:UeafjZg+TifCVPhCJNPof0pUHig6vbXuJEbC/A+Ouo0= 55 | github.com/olivere/elastic v6.2.34+incompatible h1:GdvWBAqyIyEEUd+J2sSj6EnIaBywz7zZtN+Ps4JCv0g= 56 | github.com/olivere/elastic/v7 v7.0.19 h1:w4F6JpqOISadhYf/n0NR1cNj73xHqh4pzPwD1Gkidts= 57 | github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4= 58 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 59 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 60 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 61 | github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 62 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 63 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 64 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 65 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 66 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 67 | github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= 68 | github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= 69 | github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= 70 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 71 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 72 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 73 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 74 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 75 | go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 76 | go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo= 77 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 78 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 79 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 80 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 81 | golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= 82 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 83 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 84 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 85 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 86 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 87 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 88 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 89 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 90 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 91 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 92 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 93 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 94 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 95 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 96 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 97 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 98 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= 99 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 100 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 101 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 102 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 103 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 104 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 105 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 106 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 107 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 108 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 109 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 110 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 114 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 115 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 116 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 117 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 118 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 119 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 120 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 121 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 122 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 123 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 124 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 125 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 126 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 127 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 128 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 129 | google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= 130 | google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 131 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 132 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 133 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 134 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 135 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 136 | google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= 137 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 138 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 139 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 140 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 141 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 142 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 143 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 144 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 145 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 146 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 147 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 148 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 149 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 150 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 151 | gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 152 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 153 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 154 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 155 | -------------------------------------------------------------------------------- /crawler/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 5 | github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60= 6 | github.com/Olament/gDHT v0.0.0-20200812202725-3771e54dc061 h1:I7DggatDHF3xfnO3Nm84Ni04JQ9rrvOknLUVc9p6/+s= 7 | github.com/Olament/gDHT v0.0.0-20200812202725-3771e54dc061/go.mod h1:5tTJYHofb4mxDzXsTk1uQPB1nrOxFn0KgAaH9kry7/k= 8 | github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= 9 | github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= 10 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 11 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 12 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 13 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 14 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 15 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9 h1:h2Ul3Ym2iVZWMQGYmulVUJ4LSkBm1erp9mUkPwtMoLg= 18 | github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 19 | github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20200728120301-3819c84d1a28/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 20 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 21 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 22 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 23 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 24 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 25 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 26 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 27 | github.com/go-redis/redis/v8 v8.0.0-beta.7 h1:4HiY+qfsyz8OUr9zyAP2T1CJ0SFRY4mKFvm9TEznuv8= 28 | github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g= 29 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 30 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 31 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 32 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 33 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 34 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 35 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 36 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 37 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 38 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 39 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 40 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 41 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 42 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 43 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 44 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 45 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 46 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 47 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 48 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 49 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 50 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 51 | github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= 52 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 53 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 54 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 55 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 56 | github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4= 57 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 58 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 59 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 60 | github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 61 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 62 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 63 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 64 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 65 | github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= 66 | github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= 67 | github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= 68 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 69 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 70 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 71 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 72 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 73 | go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 74 | go.opentelemetry.io/otel v0.7.0 h1:u43jukpwqR8EsyeJOMgrsUgZwVI1e1eVw7yuzRkD1l0= 75 | go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo= 76 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 77 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 78 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 79 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 80 | golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= 81 | golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= 82 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 83 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 84 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 85 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 86 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 87 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 88 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 89 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 90 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 91 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 92 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 93 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 94 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= 95 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 96 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 97 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 98 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= 99 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 100 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= 101 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 102 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 103 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 104 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 105 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 106 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 107 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 108 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 109 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 110 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= 115 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 116 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 117 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 118 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 119 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 120 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 121 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 122 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 123 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 124 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 125 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 126 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 127 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 128 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 129 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 130 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 131 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 132 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 133 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 134 | google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= 135 | google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 136 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 137 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 138 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 139 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 140 | google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= 141 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 142 | google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= 143 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 144 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 145 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 146 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 147 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 148 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 149 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 150 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 151 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 152 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 153 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 154 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 155 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 156 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 157 | gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 158 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 159 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 160 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 161 | -------------------------------------------------------------------------------- /crawler/dht/routingtable.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "container/heap" 5 | "errors" 6 | "net" 7 | "strings" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // maxPrefixLength is the length of DHT node. 13 | const maxPrefixLength = 160 14 | 15 | // node represents a DHT node. 16 | type node struct { 17 | id *bitmap 18 | addr *net.UDPAddr 19 | lastActiveTime time.Time 20 | } 21 | 22 | // newNode returns a node pointer. 23 | func newNode(id, network, address string) (*node, error) { 24 | if len(id) != 20 { 25 | return nil, errors.New("node id should be a 20-length string") 26 | } 27 | 28 | addr, err := net.ResolveUDPAddr(network, address) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return &node{newBitmapFromString(id), addr, time.Now()}, nil 34 | } 35 | 36 | // newNodeFromCompactInfo parses compactNodeInfo and returns a node pointer. 37 | func newNodeFromCompactInfo( 38 | compactNodeInfo string, network string) (*node, error) { 39 | 40 | if len(compactNodeInfo) != 26 { 41 | return nil, errors.New("compactNodeInfo should be a 26-length string") 42 | } 43 | 44 | id := compactNodeInfo[:20] 45 | ip, port, _ := decodeCompactIPPortInfo(compactNodeInfo[20:]) 46 | 47 | return newNode(id, network, genAddress(ip.String(), port)) 48 | } 49 | 50 | // CompactIPPortInfo returns "Compact IP-address/port info". 51 | // See http://www.bittorrent.org/beps/bep_0005.html. 52 | func (node *node) CompactIPPortInfo() string { 53 | info, _ := encodeCompactIPPortInfo(node.addr.IP, node.addr.Port) 54 | return info 55 | } 56 | 57 | // CompactNodeInfo returns "Compact node info". 58 | // See http://www.bittorrent.org/beps/bep_0005.html. 59 | func (node *node) CompactNodeInfo() string { 60 | return strings.Join([]string{ 61 | node.id.RawString(), node.CompactIPPortInfo(), 62 | }, "") 63 | } 64 | 65 | // Peer represents a peer contact. 66 | type Peer struct { 67 | IP net.IP 68 | Port int 69 | token string 70 | } 71 | 72 | // newPeer returns a new peer pointer. 73 | func newPeer(ip net.IP, port int, token string) *Peer { 74 | return &Peer{ 75 | IP: ip, 76 | Port: port, 77 | token: token, 78 | } 79 | } 80 | 81 | // newPeerFromCompactIPPortInfo create a peer pointer by compact ip/port info. 82 | func newPeerFromCompactIPPortInfo(compactInfo, token string) (*Peer, error) { 83 | ip, port, err := decodeCompactIPPortInfo(compactInfo) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return newPeer(ip, port, token), nil 89 | } 90 | 91 | // CompactIPPortInfo returns "Compact node info". 92 | // See http://www.bittorrent.org/beps/bep_0005.html. 93 | func (p *Peer) CompactIPPortInfo() string { 94 | info, _ := encodeCompactIPPortInfo(p.IP, p.Port) 95 | return info 96 | } 97 | 98 | // peersManager represents a proxy that manipulates peers. 99 | type peersManager struct { 100 | sync.RWMutex 101 | table *syncedMap 102 | dht *DHT 103 | } 104 | 105 | // newPeersManager returns a new peersManager. 106 | func newPeersManager(dht *DHT) *peersManager { 107 | return &peersManager{ 108 | table: newSyncedMap(), 109 | dht: dht, 110 | } 111 | } 112 | 113 | // Insert adds a peer into peersManager. 114 | func (pm *peersManager) Insert(infoHash string, peer *Peer) { 115 | pm.Lock() 116 | if _, ok := pm.table.Get(infoHash); !ok { 117 | pm.table.Set(infoHash, newKeyedDeque()) 118 | } 119 | pm.Unlock() 120 | 121 | v, _ := pm.table.Get(infoHash) 122 | queue := v.(*keyedDeque) 123 | 124 | queue.Push(peer.CompactIPPortInfo(), peer) 125 | if queue.Len() > pm.dht.K { 126 | queue.Remove(queue.Front()) 127 | } 128 | } 129 | 130 | // GetPeers returns size-length peers who announces having infoHash. 131 | func (pm *peersManager) GetPeers(infoHash string, size int) []*Peer { 132 | peers := make([]*Peer, 0, size) 133 | 134 | v, ok := pm.table.Get(infoHash) 135 | if !ok { 136 | return peers 137 | } 138 | 139 | for e := range v.(*keyedDeque).Iter() { 140 | peers = append(peers, e.Value.(*Peer)) 141 | } 142 | 143 | if len(peers) > size { 144 | peers = peers[len(peers)-size:] 145 | } 146 | return peers 147 | } 148 | 149 | // kbucket represents a k-size bucket. 150 | type kbucket struct { 151 | sync.RWMutex 152 | nodes, candidates *keyedDeque 153 | lastChanged time.Time 154 | prefix *bitmap 155 | } 156 | 157 | // newKBucket returns a new kbucket pointer. 158 | func newKBucket(prefix *bitmap) *kbucket { 159 | bucket := &kbucket{ 160 | nodes: newKeyedDeque(), 161 | candidates: newKeyedDeque(), 162 | lastChanged: time.Now(), 163 | prefix: prefix, 164 | } 165 | return bucket 166 | } 167 | 168 | // LastChanged return the last time when it changes. 169 | func (bucket *kbucket) LastChanged() time.Time { 170 | bucket.RLock() 171 | defer bucket.RUnlock() 172 | 173 | return bucket.lastChanged 174 | } 175 | 176 | // RandomChildID returns a random id that has the same prefix with bucket. 177 | func (bucket *kbucket) RandomChildID() string { 178 | prefixLen := bucket.prefix.Size / 8 179 | 180 | return strings.Join([]string{ 181 | bucket.prefix.RawString()[:prefixLen], 182 | randomString(20 - prefixLen), 183 | }, "") 184 | } 185 | 186 | // UpdateTimestamp update bucket's last changed time.. 187 | func (bucket *kbucket) UpdateTimestamp() { 188 | bucket.Lock() 189 | defer bucket.Unlock() 190 | 191 | bucket.lastChanged = time.Now() 192 | } 193 | 194 | // Insert inserts node to the bucket. It returns whether the node is new in 195 | // the bucket. 196 | func (bucket *kbucket) Insert(no *node) bool { 197 | isNew := !bucket.nodes.HasKey(no.id.RawString()) 198 | 199 | bucket.nodes.Push(no.id.RawString(), no) 200 | bucket.UpdateTimestamp() 201 | 202 | return isNew 203 | } 204 | 205 | // Replace removes node, then put bucket.candidates.Back() to the right 206 | // place of bucket.nodes. 207 | func (bucket *kbucket) Replace(no *node) { 208 | bucket.nodes.Delete(no.id.RawString()) 209 | bucket.UpdateTimestamp() 210 | 211 | if bucket.candidates.Len() == 0 { 212 | return 213 | } 214 | 215 | no = bucket.candidates.Remove(bucket.candidates.Back()).(*node) 216 | 217 | inserted := false 218 | for e := range bucket.nodes.Iter() { 219 | if e.Value.(*node).lastActiveTime.After( 220 | no.lastActiveTime) && !inserted { 221 | 222 | bucket.nodes.InsertBefore(no, e) 223 | inserted = true 224 | } 225 | } 226 | 227 | if !inserted { 228 | bucket.nodes.PushBack(no) 229 | } 230 | } 231 | 232 | // Fresh pings the expired nodes in the bucket. 233 | func (bucket *kbucket) Fresh(dht *DHT) { 234 | for e := range bucket.nodes.Iter() { 235 | no := e.Value.(*node) 236 | if time.Since(no.lastActiveTime) > dht.NodeExpriedAfter { 237 | dht.transactionManager.ping(no) 238 | } 239 | } 240 | } 241 | 242 | // routingTableNode represents routing table tree node. 243 | type routingTableNode struct { 244 | sync.RWMutex 245 | children []*routingTableNode 246 | bucket *kbucket 247 | } 248 | 249 | // newRoutingTableNode returns a new routingTableNode pointer. 250 | func newRoutingTableNode(prefix *bitmap) *routingTableNode { 251 | return &routingTableNode{ 252 | children: make([]*routingTableNode, 2), 253 | bucket: newKBucket(prefix), 254 | } 255 | } 256 | 257 | // Child returns routingTableNode's left or right child. 258 | func (tableNode *routingTableNode) Child(index int) *routingTableNode { 259 | if index >= 2 { 260 | return nil 261 | } 262 | 263 | tableNode.RLock() 264 | defer tableNode.RUnlock() 265 | 266 | return tableNode.children[index] 267 | } 268 | 269 | // SetChild sets routingTableNode's left or right child. When index is 0, it's 270 | // the left child, if 1, it's the right child. 271 | func (tableNode *routingTableNode) SetChild(index int, c *routingTableNode) { 272 | tableNode.Lock() 273 | defer tableNode.Unlock() 274 | 275 | tableNode.children[index] = c 276 | } 277 | 278 | // KBucket returns the bucket routingTableNode holds. 279 | func (tableNode *routingTableNode) KBucket() *kbucket { 280 | tableNode.RLock() 281 | defer tableNode.RUnlock() 282 | 283 | return tableNode.bucket 284 | } 285 | 286 | // SetKBucket sets the bucket. 287 | func (tableNode *routingTableNode) SetKBucket(bucket *kbucket) { 288 | tableNode.Lock() 289 | defer tableNode.Unlock() 290 | 291 | tableNode.bucket = bucket 292 | } 293 | 294 | // Split splits current routingTableNode and sets it's two children. 295 | func (tableNode *routingTableNode) Split() { 296 | prefixLen := tableNode.KBucket().prefix.Size 297 | 298 | if prefixLen == maxPrefixLength { 299 | return 300 | } 301 | 302 | for i := 0; i < 2; i++ { 303 | tableNode.SetChild(i, newRoutingTableNode(newBitmapFrom( 304 | tableNode.KBucket().prefix, prefixLen+1))) 305 | } 306 | 307 | tableNode.Lock() 308 | tableNode.children[1].bucket.prefix.Set(prefixLen) 309 | tableNode.Unlock() 310 | 311 | for e := range tableNode.KBucket().nodes.Iter() { 312 | nd := e.Value.(*node) 313 | tableNode.Child(nd.id.Bit(prefixLen)).KBucket().nodes.PushBack(nd) 314 | } 315 | 316 | for e := range tableNode.KBucket().candidates.Iter() { 317 | nd := e.Value.(*node) 318 | tableNode.Child(nd.id.Bit(prefixLen)).KBucket().candidates.PushBack(nd) 319 | } 320 | 321 | for i := 0; i < 2; i++ { 322 | tableNode.Child(i).KBucket().UpdateTimestamp() 323 | } 324 | } 325 | 326 | // routingTable implements the routing table in DHT protocol. 327 | type routingTable struct { 328 | *sync.RWMutex 329 | k int 330 | root *routingTableNode 331 | cachedNodes *syncedMap 332 | cachedKBuckets *keyedDeque 333 | dht *DHT 334 | clearQueue *syncedList 335 | } 336 | 337 | // newRoutingTable returns a new routingTable pointer. 338 | func newRoutingTable(k int, dht *DHT) *routingTable { 339 | root := newRoutingTableNode(newBitmap(0)) 340 | 341 | rt := &routingTable{ 342 | RWMutex: &sync.RWMutex{}, 343 | k: k, 344 | root: root, 345 | cachedNodes: newSyncedMap(), 346 | cachedKBuckets: newKeyedDeque(), 347 | dht: dht, 348 | clearQueue: newSyncedList(), 349 | } 350 | 351 | rt.cachedKBuckets.Push(root.bucket.prefix.String(), root.bucket) 352 | return rt 353 | } 354 | 355 | // Insert adds a node to routing table. It returns whether the node is new 356 | // in the routingtable. 357 | func (rt *routingTable) Insert(nd *node) bool { 358 | rt.Lock() 359 | defer rt.Unlock() 360 | 361 | if rt.dht.blackList.in(nd.addr.IP.String(), nd.addr.Port) || 362 | rt.cachedNodes.Len() >= rt.dht.MaxNodes { 363 | return false 364 | } 365 | 366 | var ( 367 | next *routingTableNode 368 | bucket *kbucket 369 | ) 370 | root := rt.root 371 | 372 | for prefixLen := 1; prefixLen <= maxPrefixLength; prefixLen++ { 373 | next = root.Child(nd.id.Bit(prefixLen - 1)) 374 | 375 | if next != nil { 376 | // If next is not the leaf. 377 | root = next 378 | } else if root.KBucket().nodes.Len() < rt.k || 379 | root.KBucket().nodes.HasKey(nd.id.RawString()) { 380 | 381 | bucket = root.KBucket() 382 | isNew := bucket.Insert(nd) 383 | 384 | rt.cachedNodes.Set(nd.addr.String(), nd) 385 | rt.cachedKBuckets.Push(bucket.prefix.String(), bucket) 386 | 387 | return isNew 388 | } else if root.KBucket().prefix.Compare(nd.id, prefixLen-1) == 0 { 389 | // If node has the same prefix with bucket, split it. 390 | 391 | root.Split() 392 | 393 | rt.cachedKBuckets.Delete(root.KBucket().prefix.String()) 394 | root.SetKBucket(nil) 395 | 396 | for i := 0; i < 2; i++ { 397 | bucket = root.Child(i).KBucket() 398 | rt.cachedKBuckets.Push(bucket.prefix.String(), bucket) 399 | } 400 | 401 | root = root.Child(nd.id.Bit(prefixLen - 1)) 402 | } else { 403 | // Finally, store node as a candidate and fresh the bucket. 404 | root.KBucket().candidates.PushBack(nd) 405 | if root.KBucket().candidates.Len() > rt.k { 406 | root.KBucket().candidates.Remove( 407 | root.KBucket().candidates.Front()) 408 | } 409 | 410 | go root.KBucket().Fresh(rt.dht) 411 | return false 412 | } 413 | } 414 | return false 415 | } 416 | 417 | // GetNeighbors returns the size-length nodes closest to id. 418 | func (rt *routingTable) GetNeighbors(id *bitmap, size int) []*node { 419 | rt.RLock() 420 | nodes := make([]interface{}, 0, rt.cachedNodes.Len()) 421 | for item := range rt.cachedNodes.Iter() { 422 | nodes = append(nodes, item.val.(*node)) 423 | } 424 | rt.RUnlock() 425 | 426 | neighbors := getTopK(nodes, id, size) 427 | result := make([]*node, len(neighbors)) 428 | 429 | for i, nd := range neighbors { 430 | result[i] = nd.(*node) 431 | } 432 | return result 433 | } 434 | 435 | // GetNeighborIds return the size-length compact node info closest to id. 436 | func (rt *routingTable) GetNeighborCompactInfos(id *bitmap, size int) []string { 437 | neighbors := rt.GetNeighbors(id, size) 438 | infos := make([]string, len(neighbors)) 439 | 440 | for i, no := range neighbors { 441 | infos[i] = no.CompactNodeInfo() 442 | } 443 | 444 | return infos 445 | } 446 | 447 | // GetNodeKBucktById returns node whose id is `id` and the bucket it 448 | // belongs to. 449 | func (rt *routingTable) GetNodeKBucktByID(id *bitmap) ( 450 | nd *node, bucket *kbucket) { 451 | 452 | rt.RLock() 453 | defer rt.RUnlock() 454 | 455 | var next *routingTableNode 456 | root := rt.root 457 | 458 | for prefixLen := 1; prefixLen <= maxPrefixLength; prefixLen++ { 459 | next = root.Child(id.Bit(prefixLen - 1)) 460 | if next == nil { 461 | v, ok := root.KBucket().nodes.Get(id.RawString()) 462 | if !ok { 463 | return 464 | } 465 | nd, bucket = v.Value.(*node), root.KBucket() 466 | return 467 | } 468 | root = next 469 | } 470 | return 471 | } 472 | 473 | // GetNodeByAddress finds node by address. 474 | func (rt *routingTable) GetNodeByAddress(address string) (no *node, ok bool) { 475 | rt.RLock() 476 | defer rt.RUnlock() 477 | 478 | v, ok := rt.cachedNodes.Get(address) 479 | if ok { 480 | no = v.(*node) 481 | } 482 | return 483 | } 484 | 485 | // Remove deletes the node whose id is `id`. 486 | func (rt *routingTable) Remove(id *bitmap) { 487 | if nd, bucket := rt.GetNodeKBucktByID(id); nd != nil { 488 | bucket.Replace(nd) 489 | rt.cachedNodes.Delete(nd.addr.String()) 490 | rt.cachedKBuckets.Push(bucket.prefix.String(), bucket) 491 | } 492 | } 493 | 494 | // Remove deletes the node whose address is `ip:port`. 495 | func (rt *routingTable) RemoveByAddr(address string) { 496 | v, ok := rt.cachedNodes.Get(address) 497 | if ok { 498 | rt.Remove(v.(*node).id) 499 | } 500 | } 501 | 502 | // Fresh sends findNode to all nodes in the expired nodes. 503 | func (rt *routingTable) Fresh() { 504 | now := time.Now() 505 | 506 | for e := range rt.cachedKBuckets.Iter() { 507 | bucket := e.Value.(*kbucket) 508 | if now.Sub(bucket.LastChanged()) < rt.dht.KBucketExpiredAfter || 509 | bucket.nodes.Len() == 0 { 510 | continue 511 | } 512 | 513 | i := 0 514 | for e := range bucket.nodes.Iter() { 515 | if i < rt.dht.RefreshNodeNum { 516 | no := e.Value.(*node) 517 | rt.dht.transactionManager.findNode(no, bucket.RandomChildID()) 518 | rt.clearQueue.PushBack(no) 519 | } 520 | i++ 521 | } 522 | } 523 | 524 | if rt.dht.IsCrawlMode() { 525 | for e := range rt.clearQueue.Iter() { 526 | rt.Remove(e.Value.(*node).id) 527 | } 528 | } 529 | 530 | rt.clearQueue.Clear() 531 | } 532 | 533 | // Len returns the number of nodes in table. 534 | func (rt *routingTable) Len() int { 535 | rt.RLock() 536 | defer rt.RUnlock() 537 | 538 | return rt.cachedNodes.Len() 539 | } 540 | 541 | // Implementation of heap with heap.Interface. 542 | type heapItem struct { 543 | distance *bitmap 544 | value interface{} 545 | } 546 | 547 | type topKHeap []*heapItem 548 | 549 | func (kHeap topKHeap) Len() int { 550 | return len(kHeap) 551 | } 552 | 553 | func (kHeap topKHeap) Less(i, j int) bool { 554 | return kHeap[i].distance.Compare(kHeap[j].distance, maxPrefixLength) == 1 555 | } 556 | 557 | func (kHeap topKHeap) Swap(i, j int) { 558 | kHeap[i], kHeap[j] = kHeap[j], kHeap[i] 559 | } 560 | 561 | func (kHeap *topKHeap) Push(x interface{}) { 562 | *kHeap = append(*kHeap, x.(*heapItem)) 563 | } 564 | 565 | func (kHeap *topKHeap) Pop() interface{} { 566 | n := len(*kHeap) 567 | x := (*kHeap)[n-1] 568 | *kHeap = (*kHeap)[:n-1] 569 | return x 570 | } 571 | 572 | // getTopK solves the top-k problem with heap. It's time complexity is 573 | // O(n*log(k)). When n is large, time complexity will be too high, need to be 574 | // optimized. 575 | func getTopK(queue []interface{}, id *bitmap, k int) []interface{} { 576 | topkHeap := make(topKHeap, 0, k+1) 577 | 578 | for _, value := range queue { 579 | node := value.(*node) 580 | distance := id.Xor(node.id) 581 | if topkHeap.Len() == k { 582 | var last = topkHeap[topkHeap.Len() - 1] 583 | if last.distance.Compare(distance, maxPrefixLength) == 1 { 584 | item := &heapItem{ 585 | distance, 586 | value, 587 | } 588 | heap.Push(&topkHeap, item) 589 | heap.Pop(&topkHeap) 590 | } 591 | } else { 592 | item := &heapItem{ 593 | distance, 594 | value, 595 | } 596 | heap.Push(&topkHeap, item) 597 | } 598 | } 599 | 600 | tops := make([]interface{}, topkHeap.Len()) 601 | for i := len(tops) - 1; i >= 0; i-- { 602 | tops[i] = heap.Pop(&topkHeap).(*heapItem).value 603 | } 604 | 605 | return tops 606 | } 607 | -------------------------------------------------------------------------------- /crawler/dht/krpc.go: -------------------------------------------------------------------------------- 1 | package dht 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | "strings" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | const ( 12 | pingType = "ping" 13 | findNodeType = "find_node" 14 | getPeersType = "get_peers" 15 | announcePeerType = "announce_peer" 16 | ) 17 | 18 | const ( 19 | generalError = 201 + iota 20 | serverError 21 | protocolError 22 | unknownError 23 | ) 24 | 25 | // packet represents the information receive from udp. 26 | type packet struct { 27 | data []byte 28 | raddr *net.UDPAddr 29 | } 30 | 31 | // token represents the token when response getPeers request. 32 | type token struct { 33 | data string 34 | createTime time.Time 35 | } 36 | 37 | // tokenManager managers the tokens. 38 | type tokenManager struct { 39 | *syncedMap 40 | expiredAfter time.Duration 41 | dht *DHT 42 | } 43 | 44 | // newTokenManager returns a new tokenManager. 45 | func newTokenManager(expiredAfter time.Duration, dht *DHT) *tokenManager { 46 | return &tokenManager{ 47 | syncedMap: newSyncedMap(), 48 | expiredAfter: expiredAfter, 49 | dht: dht, 50 | } 51 | } 52 | 53 | // token returns a token. If it doesn't exist or is expired, it will add a 54 | // new token. 55 | func (tm *tokenManager) token(addr *net.UDPAddr) string { 56 | v, ok := tm.Get(addr.IP.String()) 57 | tk, _ := v.(token) 58 | 59 | if !ok || time.Now().Sub(tk.createTime) > tm.expiredAfter { 60 | tk = token{ 61 | data: randomString(5), 62 | createTime: time.Now(), 63 | } 64 | 65 | tm.Set(addr.IP.String(), tk) 66 | } 67 | 68 | return tk.data 69 | } 70 | 71 | // clear removes expired tokens. 72 | func (tm *tokenManager) clear() { 73 | for _ = range time.Tick(time.Minute * 3) { 74 | keys := make([]interface{}, 0, 100) 75 | 76 | for item := range tm.Iter() { 77 | if time.Now().Sub(item.val.(token).createTime) > tm.expiredAfter { 78 | keys = append(keys, item.key) 79 | } 80 | } 81 | 82 | tm.DeleteMulti(keys) 83 | } 84 | } 85 | 86 | // check returns whether the token is valid. 87 | func (tm *tokenManager) check(addr *net.UDPAddr, tokenString string) bool { 88 | key := addr.IP.String() 89 | v, ok := tm.Get(key) 90 | tk, _ := v.(token) 91 | 92 | if ok { 93 | tm.Delete(key) 94 | } 95 | 96 | return ok && tokenString == tk.data 97 | } 98 | 99 | // makeQuery returns a query-formed data. 100 | func makeQuery(t, q string, a map[string]interface{}) map[string]interface{} { 101 | return map[string]interface{}{ 102 | "t": t, 103 | "y": "q", 104 | "q": q, 105 | "a": a, 106 | } 107 | } 108 | 109 | // makeResponse returns a response-formed data. 110 | func makeResponse(t string, r map[string]interface{}) map[string]interface{} { 111 | return map[string]interface{}{ 112 | "t": t, 113 | "y": "r", 114 | "r": r, 115 | } 116 | } 117 | 118 | // makeError returns a err-formed data. 119 | func makeError(t string, errCode int, errMsg string) map[string]interface{} { 120 | return map[string]interface{}{ 121 | "t": t, 122 | "y": "e", 123 | "e": []interface{}{errCode, errMsg}, 124 | } 125 | } 126 | 127 | // send sends data to the udp. 128 | func send(dht *DHT, addr *net.UDPAddr, data map[string]interface{}) error { 129 | dht.conn.SetWriteDeadline(time.Now().Add(time.Second * 15)) 130 | 131 | _, err := dht.conn.WriteToUDP([]byte(Encode(data)), addr) 132 | if err != nil { 133 | dht.blackList.insert(addr.IP.String(), -1) 134 | } 135 | return err 136 | } 137 | 138 | // query represents the query data included queried node and query-formed data. 139 | type query struct { 140 | node *node 141 | data map[string]interface{} 142 | } 143 | 144 | // transaction implements transaction. 145 | type transaction struct { 146 | *query 147 | id string 148 | response chan struct{} 149 | } 150 | 151 | // transactionManager represents the manager of transactions. 152 | type transactionManager struct { 153 | *sync.RWMutex 154 | transactions *syncedMap 155 | index *syncedMap 156 | cursor uint64 157 | maxCursor uint64 158 | queryChan chan *query 159 | dht *DHT 160 | } 161 | 162 | // newTransactionManager returns new transactionManager pointer. 163 | func newTransactionManager(maxCursor uint64, dht *DHT) *transactionManager { 164 | return &transactionManager{ 165 | RWMutex: &sync.RWMutex{}, 166 | transactions: newSyncedMap(), 167 | index: newSyncedMap(), 168 | maxCursor: maxCursor, 169 | queryChan: make(chan *query, 1024), 170 | dht: dht, 171 | } 172 | } 173 | 174 | // genTransID generates a transaction id and returns it. 175 | func (tm *transactionManager) genTransID() string { 176 | tm.Lock() 177 | defer tm.Unlock() 178 | 179 | tm.cursor = (tm.cursor + 1) % tm.maxCursor 180 | return string(int2bytes(tm.cursor)) 181 | } 182 | 183 | // newTransaction creates a new transaction. 184 | func (tm *transactionManager) newTransaction(id string, q *query) *transaction { 185 | return &transaction{ 186 | id: id, 187 | query: q, 188 | response: make(chan struct{}, tm.dht.Try+1), 189 | } 190 | } 191 | 192 | // genIndexKey generates an indexed key which consists of queryType and 193 | // address. 194 | func (tm *transactionManager) genIndexKey(queryType, address string) string { 195 | return strings.Join([]string{queryType, address}, ":") 196 | } 197 | 198 | // genIndexKeyByTrans generates an indexed key by a transaction. 199 | func (tm *transactionManager) genIndexKeyByTrans(trans *transaction) string { 200 | return tm.genIndexKey(trans.data["q"].(string), trans.node.addr.String()) 201 | } 202 | 203 | // insert adds a transaction to transactionManager. 204 | func (tm *transactionManager) insert(trans *transaction) { 205 | tm.Lock() 206 | defer tm.Unlock() 207 | 208 | tm.transactions.Set(trans.id, trans) 209 | tm.index.Set(tm.genIndexKeyByTrans(trans), trans) 210 | } 211 | 212 | // delete removes a transaction from transactionManager. 213 | func (tm *transactionManager) delete(transID string) { 214 | v, ok := tm.transactions.Get(transID) 215 | if !ok { 216 | return 217 | } 218 | 219 | tm.Lock() 220 | defer tm.Unlock() 221 | 222 | trans := v.(*transaction) 223 | tm.transactions.Delete(trans.id) 224 | tm.index.Delete(tm.genIndexKeyByTrans(trans)) 225 | } 226 | 227 | // len returns how many transactions are requesting now. 228 | func (tm *transactionManager) len() int { 229 | return tm.transactions.Len() 230 | } 231 | 232 | // transaction returns a transaction. keyType should be one of 0, 1 which 233 | // represents transId and index each. 234 | func (tm *transactionManager) transaction( 235 | key string, keyType int) *transaction { 236 | 237 | sm := tm.transactions 238 | if keyType == 1 { 239 | sm = tm.index 240 | } 241 | 242 | v, ok := sm.Get(key) 243 | if !ok { 244 | return nil 245 | } 246 | 247 | return v.(*transaction) 248 | } 249 | 250 | // getByTransID returns a transaction by transID. 251 | func (tm *transactionManager) getByTransID(transID string) *transaction { 252 | return tm.transaction(transID, 0) 253 | } 254 | 255 | // getByIndex returns a transaction by indexed key. 256 | func (tm *transactionManager) getByIndex(index string) *transaction { 257 | return tm.transaction(index, 1) 258 | } 259 | 260 | // transaction gets the proper transaction with whose id is transId and 261 | // address is addr. 262 | func (tm *transactionManager) filterOne( 263 | transID string, addr *net.UDPAddr) *transaction { 264 | 265 | trans := tm.getByTransID(transID) 266 | if trans == nil || trans.node.addr.String() != addr.String() { 267 | return nil 268 | } 269 | 270 | return trans 271 | } 272 | 273 | // query sends the query-formed data to udp and wait for the response. 274 | // When timeout, it will retry `try - 1` times, which means it will query 275 | // `try` times totally. 276 | func (tm *transactionManager) query(q *query, try int) { 277 | transID := q.data["t"].(string) 278 | trans := tm.newTransaction(transID, q) 279 | 280 | tm.insert(trans) 281 | defer tm.delete(trans.id) 282 | 283 | success := false 284 | for i := 0; i < try; i++ { 285 | if err := send(tm.dht, q.node.addr, q.data); err != nil { 286 | break 287 | } 288 | 289 | select { 290 | case <-trans.response: 291 | success = true 292 | break 293 | case <-time.After(time.Second * 15): 294 | } 295 | } 296 | 297 | if !success && q.node.id != nil { 298 | tm.dht.blackList.insert(q.node.addr.IP.String(), q.node.addr.Port) 299 | tm.dht.routingTable.RemoveByAddr(q.node.addr.String()) 300 | } 301 | } 302 | 303 | // run starts to listen and consume the query chan. 304 | func (tm *transactionManager) run() { 305 | var q *query 306 | 307 | for { 308 | select { 309 | case q = <-tm.queryChan: 310 | go tm.query(q, tm.dht.Try) 311 | } 312 | } 313 | } 314 | 315 | // sendQuery send query-formed data to the chan. 316 | func (tm *transactionManager) sendQuery( 317 | no *node, queryType string, a map[string]interface{}) { 318 | 319 | // If the target is self, then stop. 320 | if no.id != nil && no.id.RawString() == tm.dht.node.id.RawString() || 321 | tm.getByIndex(tm.genIndexKey(queryType, no.addr.String())) != nil || 322 | tm.dht.blackList.in(no.addr.IP.String(), no.addr.Port) { 323 | return 324 | } 325 | 326 | data := makeQuery(tm.genTransID(), queryType, a) 327 | tm.queryChan <- &query{ 328 | node: no, 329 | data: data, 330 | } 331 | } 332 | 333 | // ping sends ping query to the chan. 334 | func (tm *transactionManager) ping(no *node) { 335 | tm.sendQuery(no, pingType, map[string]interface{}{ 336 | "id": tm.dht.id(no.id.RawString()), 337 | }) 338 | } 339 | 340 | // findNode sends find_node query to the chan. 341 | func (tm *transactionManager) findNode(no *node, target string) { 342 | tm.sendQuery(no, findNodeType, map[string]interface{}{ 343 | "id": tm.dht.id(target), 344 | "target": target, 345 | }) 346 | } 347 | 348 | // getPeers sends get_peers query to the chan. 349 | func (tm *transactionManager) getPeers(no *node, infoHash string) { 350 | tm.sendQuery(no, getPeersType, map[string]interface{}{ 351 | "id": tm.dht.id(infoHash), 352 | "info_hash": infoHash, 353 | }) 354 | } 355 | 356 | // announcePeer sends announce_peer query to the chan. 357 | func (tm *transactionManager) announcePeer( 358 | no *node, infoHash string, impliedPort, port int, token string) { 359 | 360 | tm.sendQuery(no, announcePeerType, map[string]interface{}{ 361 | "id": tm.dht.id(no.id.RawString()), 362 | "info_hash": infoHash, 363 | "implied_port": impliedPort, 364 | "port": port, 365 | "token": token, 366 | }) 367 | } 368 | 369 | // ParseKey parses the key in dict data. `t` is type of the keyed value. 370 | // It's one of "int", "string", "map", "list". 371 | func ParseKey(data map[string]interface{}, key string, t string) error { 372 | val, ok := data[key] 373 | if !ok { 374 | return errors.New("lack of key") 375 | } 376 | 377 | switch t { 378 | case "string": 379 | _, ok = val.(string) 380 | case "int": 381 | _, ok = val.(int) 382 | case "map": 383 | _, ok = val.(map[string]interface{}) 384 | case "list": 385 | _, ok = val.([]interface{}) 386 | default: 387 | panic("invalid type") 388 | } 389 | 390 | if !ok { 391 | return errors.New("invalid key type") 392 | } 393 | 394 | return nil 395 | } 396 | 397 | // ParseKeys parses keys. It just wraps ParseKey. 398 | func ParseKeys(data map[string]interface{}, pairs [][]string) error { 399 | for _, args := range pairs { 400 | key, t := args[0], args[1] 401 | if err := ParseKey(data, key, t); err != nil { 402 | return err 403 | } 404 | } 405 | return nil 406 | } 407 | 408 | // parseMessage parses the basic data received from udp. 409 | // It returns a map value. 410 | func parseMessage(data interface{}) (map[string]interface{}, error) { 411 | response, ok := data.(map[string]interface{}) 412 | if !ok { 413 | return nil, errors.New("response is not dict") 414 | } 415 | 416 | if err := ParseKeys( 417 | response, [][]string{{"t", "string"}, {"y", "string"}}); err != nil { 418 | return nil, err 419 | } 420 | 421 | return response, nil 422 | } 423 | 424 | // handleRequest handles the requests received from udp. 425 | func handleRequest(dht *DHT, addr *net.UDPAddr, 426 | response map[string]interface{}) (success bool) { 427 | 428 | t := response["t"].(string) 429 | 430 | if err := ParseKeys( 431 | response, [][]string{{"q", "string"}, {"a", "map"}}); err != nil { 432 | 433 | send(dht, addr, makeError(t, protocolError, err.Error())) 434 | return 435 | } 436 | 437 | q := response["q"].(string) 438 | a := response["a"].(map[string]interface{}) 439 | 440 | if err := ParseKey(a, "id", "string"); err != nil { 441 | send(dht, addr, makeError(t, protocolError, err.Error())) 442 | return 443 | } 444 | 445 | id := a["id"].(string) 446 | 447 | if id == dht.node.id.RawString() { 448 | return 449 | } 450 | 451 | if len(id) != 20 { 452 | send(dht, addr, makeError(t, protocolError, "invalid id")) 453 | return 454 | } 455 | 456 | if no, ok := dht.routingTable.GetNodeByAddress(addr.String()); ok && 457 | no.id.RawString() != id { 458 | 459 | dht.blackList.insert(addr.IP.String(), addr.Port) 460 | dht.routingTable.RemoveByAddr(addr.String()) 461 | 462 | send(dht, addr, makeError(t, protocolError, "invalid id")) 463 | return 464 | } 465 | 466 | switch q { 467 | case pingType: 468 | send(dht, addr, makeResponse(t, map[string]interface{}{ 469 | "id": dht.id(id), 470 | })) 471 | case findNodeType: 472 | if dht.IsStandardMode() { 473 | if err := ParseKey(a, "target", "string"); err != nil { 474 | send(dht, addr, makeError(t, protocolError, err.Error())) 475 | return 476 | } 477 | 478 | target := a["target"].(string) 479 | if len(target) != 20 { 480 | send(dht, addr, makeError(t, protocolError, "invalid target")) 481 | return 482 | } 483 | 484 | var nodes string 485 | targetID := newBitmapFromString(target) 486 | 487 | no, _ := dht.routingTable.GetNodeKBucktByID(targetID) 488 | if no != nil { 489 | nodes = no.CompactNodeInfo() 490 | } else { 491 | nodes = strings.Join( 492 | dht.routingTable.GetNeighborCompactInfos(targetID, dht.K), 493 | "", 494 | ) 495 | } 496 | 497 | send(dht, addr, makeResponse(t, map[string]interface{}{ 498 | "id": dht.id(target), 499 | "nodes": nodes, 500 | })) 501 | } 502 | case getPeersType: 503 | if err := ParseKey(a, "info_hash", "string"); err != nil { 504 | send(dht, addr, makeError(t, protocolError, err.Error())) 505 | return 506 | } 507 | 508 | infoHash := a["info_hash"].(string) 509 | 510 | if len(infoHash) != 20 { 511 | send(dht, addr, makeError(t, protocolError, "invalid info_hash")) 512 | return 513 | } 514 | 515 | if dht.IsCrawlMode() { 516 | send(dht, addr, makeResponse(t, map[string]interface{}{ 517 | "id": dht.id(infoHash), 518 | "token": dht.tokenManager.token(addr), 519 | "nodes": "", 520 | })) 521 | } else if peers := dht.peersManager.GetPeers( 522 | infoHash, dht.K); len(peers) > 0 { 523 | 524 | values := make([]interface{}, len(peers)) 525 | for i, p := range peers { 526 | values[i] = p.CompactIPPortInfo() 527 | } 528 | 529 | send(dht, addr, makeResponse(t, map[string]interface{}{ 530 | "id": dht.id(infoHash), 531 | "values": values, 532 | "token": dht.tokenManager.token(addr), 533 | })) 534 | } else { 535 | send(dht, addr, makeResponse(t, map[string]interface{}{ 536 | "id": dht.id(infoHash), 537 | "token": dht.tokenManager.token(addr), 538 | "nodes": strings.Join(dht.routingTable.GetNeighborCompactInfos( 539 | newBitmapFromString(infoHash), dht.K), ""), 540 | })) 541 | } 542 | 543 | if dht.OnGetPeers != nil { 544 | dht.OnGetPeers(infoHash, addr.IP.String(), addr.Port) 545 | } 546 | case announcePeerType: 547 | if err := ParseKeys(a, [][]string{ 548 | {"info_hash", "string"}, 549 | {"port", "int"}, 550 | {"token", "string"}}); err != nil { 551 | 552 | send(dht, addr, makeError(t, protocolError, err.Error())) 553 | return 554 | } 555 | 556 | infoHash := a["info_hash"].(string) 557 | port := a["port"].(int) 558 | token := a["token"].(string) 559 | 560 | if !dht.tokenManager.check(addr, token) { 561 | // send(dht, addr, makeError(t, protocolError, "invalid token")) 562 | return 563 | } 564 | 565 | if impliedPort, ok := a["implied_port"]; ok && 566 | impliedPort.(int) != 0 { 567 | 568 | port = addr.Port 569 | } 570 | 571 | if dht.IsStandardMode() { 572 | dht.peersManager.Insert(infoHash, newPeer(addr.IP, port, token)) 573 | 574 | send(dht, addr, makeResponse(t, map[string]interface{}{ 575 | "id": dht.id(id), 576 | })) 577 | } 578 | 579 | if dht.OnAnnouncePeer != nil { 580 | dht.OnAnnouncePeer(infoHash, addr.IP.String(), port) 581 | } 582 | default: 583 | // send(dht, addr, makeError(t, protocolError, "invalid q")) 584 | return 585 | } 586 | 587 | no, _ := newNode(id, addr.Network(), addr.String()) 588 | dht.routingTable.Insert(no) 589 | return true 590 | } 591 | 592 | // findOn puts nodes in the response to the routingTable, then if target is in 593 | // the nodes or all nodes are in the routingTable, it stops. Otherwise it 594 | // continues to findNode or getPeers. 595 | func findOn(dht *DHT, r map[string]interface{}, target *bitmap, 596 | queryType string) error { 597 | 598 | if err := ParseKey(r, "nodes", "string"); err != nil { 599 | return err 600 | } 601 | 602 | nodes := r["nodes"].(string) 603 | if len(nodes)%26 != 0 { 604 | return errors.New("the length of nodes should can be divided by 26") 605 | } 606 | 607 | hasNew, found := false, false 608 | for i := 0; i < len(nodes)/26; i++ { 609 | no, _ := newNodeFromCompactInfo( 610 | string(nodes[i*26:(i+1)*26]), dht.Network) 611 | 612 | if no.id.RawString() == target.RawString() { 613 | found = true 614 | } 615 | 616 | if dht.routingTable.Insert(no) { 617 | hasNew = true 618 | } 619 | } 620 | 621 | if found || !hasNew { 622 | return nil 623 | } 624 | 625 | targetID := target.RawString() 626 | for _, no := range dht.routingTable.GetNeighbors(target, dht.K) { 627 | switch queryType { 628 | case findNodeType: 629 | dht.transactionManager.findNode(no, targetID) 630 | case getPeersType: 631 | dht.transactionManager.getPeers(no, targetID) 632 | default: 633 | panic("invalid find type") 634 | } 635 | } 636 | return nil 637 | } 638 | 639 | // handleResponse handles responses received from udp. 640 | func handleResponse(dht *DHT, addr *net.UDPAddr, 641 | response map[string]interface{}) (success bool) { 642 | 643 | t := response["t"].(string) 644 | 645 | trans := dht.transactionManager.filterOne(t, addr) 646 | if trans == nil { 647 | return 648 | } 649 | 650 | // inform transManager to delete the transaction. 651 | if err := ParseKey(response, "r", "map"); err != nil { 652 | return 653 | } 654 | 655 | q := trans.data["q"].(string) 656 | a := trans.data["a"].(map[string]interface{}) 657 | r := response["r"].(map[string]interface{}) 658 | 659 | if err := ParseKey(r, "id", "string"); err != nil { 660 | return 661 | } 662 | 663 | id := r["id"].(string) 664 | 665 | // If response's node id is not the same with the node id in the 666 | // transaction, raise error. 667 | if trans.node.id != nil && trans.node.id.RawString() != r["id"].(string) { 668 | dht.blackList.insert(addr.IP.String(), addr.Port) 669 | dht.routingTable.RemoveByAddr(addr.String()) 670 | return 671 | } 672 | 673 | node, err := newNode(id, addr.Network(), addr.String()) 674 | if err != nil { 675 | return 676 | } 677 | 678 | switch q { 679 | case pingType: 680 | case findNodeType: 681 | if trans.data["q"].(string) != findNodeType { 682 | return 683 | } 684 | 685 | target := trans.data["a"].(map[string]interface{})["target"].(string) 686 | if findOn(dht, r, newBitmapFromString(target), findNodeType) != nil { 687 | return 688 | } 689 | case getPeersType: 690 | if err := ParseKey(r, "token", "string"); err != nil { 691 | return 692 | } 693 | 694 | token := r["token"].(string) 695 | infoHash := a["info_hash"].(string) 696 | 697 | if err := ParseKey(r, "values", "list"); err == nil { 698 | values := r["values"].([]interface{}) 699 | for _, v := range values { 700 | p, err := newPeerFromCompactIPPortInfo(v.(string), token) 701 | if err != nil { 702 | continue 703 | } 704 | dht.peersManager.Insert(infoHash, p) 705 | if dht.OnGetPeersResponse != nil { 706 | dht.OnGetPeersResponse(infoHash, p) 707 | } 708 | } 709 | } else if findOn( 710 | dht, r, newBitmapFromString(infoHash), getPeersType) != nil { 711 | return 712 | } 713 | case announcePeerType: 714 | default: 715 | return 716 | } 717 | 718 | // inform transManager to delete transaction. 719 | trans.response <- struct{}{} 720 | 721 | dht.blackList.delete(addr.IP.String(), addr.Port) 722 | dht.routingTable.Insert(node) 723 | 724 | return true 725 | } 726 | 727 | // handleError handles errors received from udp. 728 | func handleError(dht *DHT, addr *net.UDPAddr, 729 | response map[string]interface{}) (success bool) { 730 | 731 | if err := ParseKey(response, "e", "list"); err != nil { 732 | return 733 | } 734 | 735 | if e := response["e"].([]interface{}); len(e) != 2 { 736 | return 737 | } 738 | 739 | if trans := dht.transactionManager.filterOne( 740 | response["t"].(string), addr); trans != nil { 741 | 742 | trans.response <- struct{}{} 743 | } 744 | 745 | return true 746 | } 747 | 748 | var handlers = map[string]func(*DHT, *net.UDPAddr, map[string]interface{}) bool{ 749 | "q": handleRequest, 750 | "r": handleResponse, 751 | "e": handleError, 752 | } 753 | 754 | // handle handles packets received from udp. 755 | func handle(dht *DHT, pkt packet) { 756 | if len(dht.workerTokens) == dht.PacketWorkerLimit { 757 | return 758 | } 759 | 760 | dht.workerTokens <- struct{}{} 761 | 762 | go func() { 763 | defer func() { 764 | <-dht.workerTokens 765 | }() 766 | 767 | if dht.blackList.in(pkt.raddr.IP.String(), pkt.raddr.Port) { 768 | return 769 | } 770 | 771 | data, err := Decode(pkt.data) 772 | if err != nil { 773 | return 774 | } 775 | 776 | response, err := parseMessage(data) 777 | if err != nil { 778 | return 779 | } 780 | 781 | if f, ok := handlers[response["y"].(string)]; ok { 782 | f(dht, pkt.raddr, response) 783 | } 784 | }() 785 | } 786 | -------------------------------------------------------------------------------- /web/src/style.css: -------------------------------------------------------------------------------- 1 | /*! Tania Rascia's website style. Written with plain CSS in one file. Why not? */ 2 | 3 | /* Global variables */ 4 | 5 | @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@400;500;700&display=swap'); 6 | 7 | :root { 8 | --font-color: #495057; 9 | --heading-color: #343a40; 10 | --dark-font-color: #1b1d25; 11 | --medium-font-color: #60656c; 12 | --light-font-color: #787f87; 13 | --light-background: #f6f8fb; 14 | --lighter-background: #e6e8eb; 15 | --border: #d6d9de; 16 | --link-color: #5c7cfa; 17 | --accent-color: #fadb6b; 18 | --light-accent-color: #fcebb0; 19 | --code-font-family: Menlo, 'Roboto Mono', Courier New, monospace; 20 | } 21 | 22 | /* Reset */ 23 | 24 | *, 25 | *::before, 26 | *::after { 27 | box-sizing: border-box; 28 | } 29 | 30 | /* Scaffolding */ 31 | 32 | html { 33 | font-family: -apple-system, BlinkMacSystemFont, 'Helvetica Neue', 'Roboto', 34 | Roboto, Arial, 'Noto Sans', sans-serif, 'Apple Color Emoji', 'Segoe UI', 35 | 'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji'; 36 | color: var(--font-color); 37 | font-weight: 400; 38 | font-size: 1rem; 39 | line-height: 1.5; 40 | } 41 | 42 | body { 43 | margin: 0; 44 | padding: 0; 45 | } 46 | 47 | article { 48 | min-width: 0; 49 | } 50 | 51 | section { 52 | margin: 2rem 0; 53 | } 54 | 55 | section > h2 { 56 | font-size: 1.8rem; 57 | margin-top: 1rem; 58 | margin-bottom: 2rem; 59 | padding: 0.4rem 0; 60 | border-bottom: 1px solid var(--border); 61 | } 62 | 63 | @media screen and (min-width: 800px) { 64 | section { 65 | margin: 3rem 0; 66 | } 67 | section > h2 { 68 | font-size: 2.2rem; 69 | margin: 0 0 1rem; 70 | } 71 | } 72 | 73 | .container { 74 | max-width: 1150px; 75 | padding: 0 1.5rem; 76 | margin-left: auto; 77 | margin-right: auto; 78 | } 79 | 80 | .container.page p { 81 | max-width: 600px; 82 | } 83 | 84 | @media screen and (min-width: 800px) { 85 | .container { 86 | padding: 0 2rem; 87 | } 88 | } 89 | 90 | img { 91 | display: inline-block; 92 | max-width: 100%; 93 | height: auto; 94 | } 95 | 96 | p, 97 | ol, 98 | ul, 99 | dl, 100 | table, 101 | blockquote { 102 | font-size: 1.1rem; 103 | margin: 0 0 1.5rem 0; 104 | } 105 | 106 | ul { 107 | padding: 0 1rem; 108 | } 109 | 110 | @media screen and (min-width: 800px) { 111 | p, 112 | ol, 113 | ul, 114 | dl, 115 | table, 116 | blockquote { 117 | font-size: 1.15rem; 118 | } 119 | 120 | ul { 121 | padding: 0 2rem; 122 | } 123 | } 124 | 125 | ul li p { 126 | margin: 0; 127 | } 128 | 129 | ul li ul { 130 | padding-left: 1rem; 131 | margin: 0; 132 | } 133 | 134 | ul li ul li { 135 | margin: 0.25rem 0; 136 | } 137 | 138 | ol li ol { 139 | margin-bottom: 0; 140 | } 141 | 142 | .task-list-item [type='checkbox'] { 143 | margin-right: 0.5rem; 144 | } 145 | 146 | blockquote { 147 | margin: 2rem 0; 148 | padding: 1rem; 149 | background: #edf2ff; 150 | border-radius: 0.3rem; 151 | font-weight: 400; 152 | } 153 | 154 | blockquote p { 155 | margin: 0; 156 | } 157 | 158 | blockquote a { 159 | padding: 1px 4px; 160 | } 161 | 162 | blockquote :not(pre) > code[class*='language-'] { 163 | background: rgba(0, 0, 0, 0.1) !important; 164 | } 165 | 166 | blockquote a:hover { 167 | border-radius: 0.3rem; 168 | border-bottom: 1px solid #3b5bdb; 169 | background: #3b5bdb; 170 | color: white; 171 | } 172 | 173 | blockquote.quotation { 174 | margin: 2rem 0; 175 | padding: 0 1.5rem; 176 | background: white; 177 | border-radius: 0; 178 | border: none; 179 | border-left: 11px solid var(--border); 180 | } 181 | 182 | blockquote.quotation p { 183 | font-family: Georgia, serif; 184 | line-height: 1.6; 185 | color: var(--medium-font-color); 186 | font-size: 1.1rem; 187 | margin: 0 0 2rem 0; 188 | } 189 | 190 | blockquote.pull-quote { 191 | margin: 1.5rem 0; 192 | padding: 2rem; 193 | background: var(--light-accent-color); 194 | color: var(--dark-font-color); 195 | border: 0; 196 | } 197 | 198 | blockquote.pull-quote p { 199 | line-height: 1.5; 200 | font-size: 1.1rem; 201 | margin: 0 0 2rem 0; 202 | font-weight: 400; 203 | } 204 | 205 | blockquote.pull-quote p:last-of-type { 206 | margin: 0; 207 | } 208 | 209 | blockquote.pull-quote cite { 210 | font-size: 1rem; 211 | } 212 | 213 | blockquote.pull-quote a { 214 | background: linear-gradient(transparent 70%, var(--accent-color) 0); 215 | } 216 | 217 | blockquote.pull-quote a:hover { 218 | background: linear-gradient(transparent 70%, var(--accent-color) 0); 219 | border-bottom: 0; 220 | color: var(--dark-font-color); 221 | } 222 | 223 | @media screen and (min-width: 800px) { 224 | blockquote.pull-quote { 225 | margin: 1.5rem 2.5rem; 226 | padding: 2rem; 227 | min-width: 320px; 228 | } 229 | } 230 | 231 | @media screen and (min-width: 800px) { 232 | blockquote.quotation { 233 | margin: 2.5rem 0; 234 | overflow: auto; 235 | } 236 | blockquote.quotation cite { 237 | float: right; 238 | } 239 | blockquote.quotation p { 240 | font-size: 1.2rem; 241 | } 242 | } 243 | 244 | /* Headings */ 245 | 246 | h1, 247 | h2, 248 | h3, 249 | h4, 250 | h5 { 251 | margin: 0 0 1.5rem 0; 252 | font-weight: 700; 253 | line-height: 1.2; 254 | color: var(--heading-color); 255 | -webkit-font-smoothing: antialiased; 256 | -moz-osx-font-smoothing: grayscale; 257 | } 258 | 259 | h1:not(:first-child), 260 | h2:not(:first-child), 261 | h3:not(:first-child), 262 | h4:not(:first-child) { 263 | margin-top: 3rem; 264 | } 265 | 266 | h1 { 267 | font-size: 2.5rem; 268 | line-height: 1.1; 269 | } 270 | 271 | h2 { 272 | font-size: 1.75rem; 273 | } 274 | 275 | h2 code { 276 | font-size: 1.75rem !important; 277 | } 278 | 279 | h3 { 280 | font-size: 1.5rem; 281 | color: var(--font-color); 282 | font-weight: 600; 283 | margin-bottom: 1rem; 284 | } 285 | 286 | h3 code { 287 | font-size: 1.4rem !important; 288 | } 289 | 290 | h4 { 291 | font-size: 1.3rem; 292 | color: var(--font-color); 293 | font-weight: 500; 294 | margin-bottom: 1rem; 295 | border-bottom: 1px solid var(--border); 296 | padding-bottom: 0.25rem; 297 | } 298 | 299 | h5 { 300 | font-size: 1.2rem; 301 | margin-bottom: 1rem; 302 | } 303 | 304 | @media screen and (min-width: 800px) { 305 | h1 { 306 | font-size: 3rem; 307 | } 308 | 309 | h2 { 310 | font-size: 2rem; 311 | } 312 | 313 | h2 code { 314 | font-size: 2rem !important; 315 | } 316 | 317 | h3 { 318 | font-size: 1.7rem; 319 | color: var(--font-color); 320 | font-weight: 600; 321 | } 322 | 323 | h3 code { 324 | font-size: 1.6rem !important; 325 | } 326 | } 327 | 328 | /* Sidebar */ 329 | 330 | .aside-content { 331 | background: var(--light-background); 332 | padding: 1.5rem; 333 | border-radius: 0.3rem; 334 | } 335 | 336 | aside { 337 | min-width: 300px; 338 | } 339 | 340 | aside p, 341 | aside ul, 342 | aside a { 343 | font-size: 0.95rem; 344 | } 345 | 346 | aside ul { 347 | padding-left: 1rem; 348 | } 349 | 350 | aside h3 { 351 | margin-top: 0.5rem; 352 | margin-bottom: 1rem; 353 | padding-bottom: 0.25rem; 354 | font-size: 1.1rem; 355 | color: var(--dark-font-color); 356 | border-bottom: 1px solid var(--font-color); 357 | } 358 | 359 | aside p a { 360 | color: var(--dark-font-color); 361 | background: linear-gradient( 362 | transparent 70%, 363 | var(--accent-color) 0 364 | ) !important; 365 | } 366 | 367 | aside section { 368 | margin: 3.5rem 0; 369 | } 370 | 371 | .aside-content section:first-child { 372 | margin-top: 0; 373 | } 374 | .aside-content section:last-child { 375 | margin-bottom: 0; 376 | } 377 | 378 | .avatar { 379 | display: block; 380 | border-radius: 50%; 381 | max-width: 180px; 382 | margin: 0 auto 1rem; 383 | } 384 | 385 | a.link { 386 | display: block; 387 | padding: 0.25rem 0; 388 | margin: 0.25rem 0; 389 | border-radius: 0.3rem; 390 | font-weight: 600; 391 | background: transparent; 392 | color: var(--dark-font-color); 393 | } 394 | 395 | a.link:hover { 396 | background: transparent; 397 | color: #3b5bdb; 398 | } 399 | 400 | /* Main */ 401 | 402 | .lead { 403 | padding: 3rem 0; 404 | background: white; 405 | margin: 0; 406 | } 407 | 408 | .lead .container { 409 | display: flex; 410 | flex-direction: column; 411 | } 412 | 413 | .lead h1 { 414 | font-weight: 600; 415 | font-size: 1.8rem; 416 | margin-bottom: 2rem; 417 | line-height: 1.2; 418 | } 419 | 420 | .lead p { 421 | color: rgba(0, 0, 0, 0.8); 422 | font-size: 1.1rem; 423 | font-weight: 400; 424 | margin-bottom: 2.5rem; 425 | } 426 | 427 | .lead p:last-child { 428 | margin-bottom: 0; 429 | } 430 | 431 | .copy { 432 | max-width: 600px; 433 | } 434 | 435 | .lead img { 436 | display: block; 437 | border-radius: 16px; 438 | height: 150px; 439 | width: 150px; 440 | margin-bottom: 2rem; 441 | } 442 | 443 | .lead a { 444 | color: var(--link-color); 445 | background: transparent; 446 | font-weight: 700; 447 | } 448 | 449 | .lead a:hover { 450 | background: linear-gradient(transparent 70%, #dbe4ff 0); 451 | } 452 | 453 | .lead a.button { 454 | display: block; 455 | border: none; 456 | padding: 0.85rem 1.5rem; 457 | margin-right: 0.75rem; 458 | font-weight: 500; 459 | background: var(--light-background); 460 | color: var(--dark-font-color); 461 | border: 1px solid transparent; 462 | } 463 | 464 | .lead a.button:hover { 465 | border: 1px solid #3b5bdb; 466 | background: #3b5bdb; 467 | color: white; 468 | } 469 | 470 | .lead .image { 471 | order: 1; 472 | } 473 | 474 | .lead .copy { 475 | order: 2; 476 | } 477 | 478 | .lead b { 479 | color: white; 480 | } 481 | 482 | @media screen and (min-width: 800px) { 483 | .lead { 484 | padding: 2rem 0; 485 | } 486 | 487 | .lead .container { 488 | flex-direction: row; 489 | justify-content: space-between; 490 | align-items: center; 491 | } 492 | 493 | .lead h1 { 494 | font-size: 2.6rem; 495 | line-height: 1.1; 496 | } 497 | 498 | .lead p { 499 | font-size: 1.2rem; 500 | } 501 | 502 | .lead img { 503 | height: auto; 504 | width: auto; 505 | max-width: 200px; 506 | margin-bottom: 0; 507 | margin-left: 2.5rem; 508 | } 509 | 510 | .lead .image { 511 | order: 2; 512 | } 513 | 514 | .lead .copy { 515 | order: 1; 516 | } 517 | } 518 | 519 | @media screen and (min-width: 1000px) { 520 | .lead { 521 | align-items: center; 522 | } 523 | .lead img { 524 | max-width: 350px; 525 | } 526 | } 527 | 528 | p.subtitle { 529 | color: var(--medium-font-color); 530 | font-size: 1.3rem; 531 | font-weight: 300; 532 | } 533 | 534 | @media screen and (min-width: 800px) { 535 | p.subtitle { 536 | font-size: 1.5rem; 537 | } 538 | } 539 | 540 | /* Links */ 541 | 542 | a { 543 | color: var(--dark-font-color); 544 | background: linear-gradient(transparent 70%, #dbe4ff 0); 545 | text-decoration: none; 546 | font-weight: 700; 547 | } 548 | 549 | a:hover { 550 | color: var(--dark-font-color); 551 | background: linear-gradient(transparent 70%, #bac8ff 0); 552 | } 553 | 554 | .anchor.before { 555 | background: none; 556 | } 557 | 558 | a.gatsby-resp-image-link:hover { 559 | border-bottom: none; 560 | } 561 | 562 | a code[class*='language-'] { 563 | color: var(--link-color) !important; 564 | } 565 | 566 | a code[class*='language-']:hover { 567 | background: var(--link-color) !important; 568 | color: white !important; 569 | } 570 | 571 | /* Navbar */ 572 | 573 | main { 574 | margin-top: 3.5rem; 575 | } 576 | 577 | .emoji { 578 | margin: 0 0.25rem 0 0.1rem; 579 | } 580 | 581 | .navbar { 582 | width: 100%; 583 | padding: 0.5rem; 584 | position: fixed; 585 | top: 0; 586 | left: 0; 587 | background: #4c6ef5; 588 | border-bottom: 1px solid #3b5bdb; 589 | box-shadow: 0 3px 13px rgba(100, 110, 140, 0.1), 590 | 0 2px 4px rgba(100, 110, 140, 0.15); 591 | z-index: 2; 592 | } 593 | 594 | .navbar .flex { 595 | justify-content: space-between; 596 | } 597 | 598 | .navbar a, 599 | .footer a { 600 | background: transparent; 601 | border-radius: 0.3rem; 602 | padding: 0.5rem; 603 | margin: 0 0.1rem; 604 | color: white; 605 | font-weight: 400; 606 | font-size: 0.95rem; 607 | border: 1px solid transparent; 608 | } 609 | 610 | .navbar a:hover, 611 | .navbar a[aria-current='page'] { 612 | background: rgba(0, 0, 0, 0.2); 613 | color: white; 614 | } 615 | 616 | .navbar a .emoji { 617 | display: none; 618 | } 619 | 620 | .footer a { 621 | color: rgba(0, 0, 0, 0.7); 622 | } 623 | 624 | .footer a:hover { 625 | background: rgba(0, 0, 0, 0.05); 626 | } 627 | 628 | .navbar a:first-of-type { 629 | margin-left: -1rem; 630 | } 631 | .navbar a:last-of-type { 632 | margin-right: -1rem; 633 | } 634 | 635 | .navbar a.brand { 636 | font-size: 1.05rem; 637 | font-weight: 500; 638 | color: white; 639 | white-space: nowrap; 640 | background: transparent; 641 | border: none; 642 | } 643 | 644 | .navbar a.brand .emoji { 645 | display: inline-block !important; 646 | } 647 | 648 | @media screen and (min-width: 800px) { 649 | .emoji { 650 | margin: 0 0.5rem 0 0.1rem; 651 | } 652 | .navbar { 653 | padding: 0.5rem 0; 654 | position: static; 655 | box-shadow: none; 656 | } 657 | 658 | .navbar .flex { 659 | justify-content: space-between; 660 | } 661 | 662 | .navbar a, 663 | .footer a { 664 | padding: 1rem 1.5rem; 665 | margin: 0 0.25rem; 666 | font-size: 1.1rem; 667 | } 668 | .navbar a .emoji { 669 | /* display: inline-block; */ 670 | } 671 | .navbar a.brand { 672 | font-size: 1.3rem; 673 | margin-right: 3rem; 674 | border: none; 675 | } 676 | .navbar a:first-of-type { 677 | margin-left: -1.5rem; 678 | } 679 | } 680 | 681 | /* Footer */ 682 | 683 | .footer { 684 | padding: 2rem 0; 685 | } 686 | 687 | .footer > .flex { 688 | flex-direction: column; 689 | align-items: center; 690 | justify-content: center; 691 | } 692 | 693 | .footer-links { 694 | display: flex; 695 | align-items: center; 696 | justify-content: center; 697 | flex-wrap: wrap; 698 | } 699 | 700 | .flex nav { 701 | padding: 1rem 0; 702 | } 703 | 704 | .footer img { 705 | height: 30px; 706 | width: 30px; 707 | } 708 | 709 | .footer a.img { 710 | display: flex; 711 | align-items: center; 712 | padding: 0; 713 | margin: 0 0.75rem; 714 | background: none; 715 | } 716 | 717 | .footer img { 718 | height: 30px; 719 | width: 30px; 720 | } 721 | 722 | @media screen and (min-width: 800px) { 723 | .footer > .flex { 724 | align-items: flex-start; 725 | margin-left: -1.5rem; 726 | margin-right: -1.5rem; 727 | } 728 | 729 | .footer a.img { 730 | padding: 0 0.5rem; 731 | margin: 0 1rem; 732 | } 733 | } 734 | 735 | /* Tables */ 736 | 737 | table { 738 | border-collapse: collapse; 739 | border-spacing: 0; 740 | width: 100%; 741 | max-width: 100%; 742 | overflow-x: auto; 743 | } 744 | 745 | th { 746 | border-bottom: 2px solid var(--light-background); 747 | } 748 | 749 | tfoot th { 750 | border-top: 1px solid var(--light-background); 751 | } 752 | 753 | td { 754 | border-bottom: 1px solid var(--light-background); 755 | } 756 | 757 | th, 758 | td { 759 | text-align: left; 760 | padding: 0.75rem !important; 761 | hyphens: auto; 762 | word-break: break-word; 763 | } 764 | 765 | tbody tr:nth-child(even) { 766 | background-color: var(--light-background); 767 | } 768 | 769 | /* Grid and flex */ 770 | 771 | .flex { 772 | display: flex; 773 | align-items: center; 774 | } 775 | 776 | .flex-row { 777 | display: flex; 778 | flex-direction: column; 779 | } 780 | 781 | @media screen and (min-width: 800px) { 782 | .flex-row { 783 | flex-direction: row; 784 | } 785 | } 786 | 787 | .flex-col { 788 | flex: 1; 789 | } 790 | 791 | .flex-two-thirds { 792 | flex: 2; 793 | } 794 | 795 | .justify-center { 796 | justify-content: center; 797 | } 798 | 799 | .grid { 800 | display: grid; 801 | align-items: stretch; 802 | } 803 | 804 | .grid.projects { 805 | grid-template-columns: 40px auto; 806 | } 807 | 808 | .row .cell.description { 809 | grid-column: 1 / span 2; 810 | margin-bottom: 2rem; 811 | line-height: 1.3; 812 | font-size: 1.1rem; 813 | } 814 | 815 | @media screen and (min-width: 800px) { 816 | .grid { 817 | margin-left: -1rem; 818 | margin-right: -1rem; 819 | } 820 | 821 | .grid.projects { 822 | grid-template-columns: 50px 1fr 4fr; 823 | } 824 | 825 | .row .cell.tags { 826 | justify-content: flex-end; 827 | } 828 | 829 | .row .cell.description { 830 | grid-column: auto; 831 | margin-bottom: 0; 832 | } 833 | 834 | .grid.posts { 835 | grid-template-columns: 200px 4fr; 836 | } 837 | 838 | .grid.lists { 839 | grid-template-columns: 2fr 4fr; 840 | } 841 | 842 | .grid.posts.with-tags { 843 | grid-template-columns: 6fr 1fr 1fr; 844 | } 845 | } 846 | 847 | .row { 848 | display: contents; 849 | } 850 | 851 | a.cell { 852 | background: transparent; 853 | } 854 | 855 | .posts .row, 856 | .lists .row { 857 | display: block; 858 | margin-bottom: 1rem; 859 | padding-bottom: 1rem; 860 | } 861 | 862 | .row .cell { 863 | font-weight: 600; 864 | color: var(--dark-font-color); 865 | font-size: 1.1rem; 866 | line-height: 1.2; 867 | padding: 0.25rem 0; 868 | } 869 | 870 | .row .cell.simple { 871 | border-bottom: none; 872 | } 873 | 874 | .row .cell time { 875 | display: block; 876 | font-size: 0.75rem; 877 | margin-bottom: 0.25rem; 878 | } 879 | 880 | @media screen and (min-width: 800px) { 881 | .posts .row, 882 | .lists .row { 883 | margin-bottom: 2rem; 884 | } 885 | 886 | .row .cell { 887 | display: flex; 888 | align-items: center; 889 | font-size: 1.2rem; 890 | padding: 0.5rem 0; 891 | } 892 | .row .cell time { 893 | font-size: 0.8rem; 894 | } 895 | .row .cell.simple { 896 | padding: 0.5rem 0; 897 | } 898 | .posts .row, 899 | .lists .row { 900 | display: contents; 901 | } 902 | 903 | .row:hover > * { 904 | background: var(--light-background); 905 | border-color: var(--border); 906 | } 907 | 908 | .row .cell:first-child, 909 | .row .cell.simple:first-child { 910 | padding-left: 1rem; 911 | border-top-left-radius: 0.3rem; 912 | border-bottom-left-radius: 0.3rem; 913 | } 914 | 915 | .row .cell:last-child, 916 | .row .cell.simple:last-child { 917 | padding-right: 1rem; 918 | padding-left: 1rem; 919 | border-top-right-radius: 0.3rem; 920 | border-bottom-right-radius: 0.3rem; 921 | } 922 | } 923 | 924 | .row .cell.light { 925 | font-weight: 400; 926 | color: var(--light-font-color); 927 | } 928 | 929 | /* Post */ 930 | 931 | .grid.post { 932 | grid-gap: 1rem; 933 | margin-left: 0; 934 | margin-right: 0; 935 | } 936 | 937 | header { 938 | padding: 3rem 0 0; 939 | } 940 | 941 | header h1 { 942 | font-size: 2rem; 943 | display: inline-block; 944 | font-weight: 600; 945 | margin-bottom: 1rem; 946 | } 947 | 948 | header u { 949 | display: inline-block; 950 | text-decoration: none; 951 | padding: 0.4rem 0; 952 | } 953 | 954 | .article-header { 955 | margin-bottom: 2rem; 956 | } 957 | 958 | .article-header .container { 959 | padding-left: 0; 960 | padding-right: 0; 961 | } 962 | 963 | .article-header .thumb { 964 | display: flex; 965 | flex-direction: column; 966 | } 967 | 968 | .article-header h1 { 969 | font-weight: 700; 970 | font-size: 1.8rem; 971 | margin: 0; 972 | } 973 | 974 | @media screen and (min-width: 800px) { 975 | header { 976 | padding: 0; 977 | } 978 | 979 | header h1 { 980 | font-size: 3rem; 981 | } 982 | 983 | header u { 984 | background: linear-gradient(transparent 85%, #bac8ff 0); 985 | } 986 | 987 | .article-header { 988 | padding-top: 0; 989 | margin-bottom: 3.5rem; 990 | } 991 | 992 | .article-header .thumb { 993 | flex-direction: row; 994 | align-items: center; 995 | } 996 | } 997 | 998 | @media screen and (min-width: 1100px) { 999 | .grid.post { 1000 | margin-top: 5rem; 1001 | grid-gap: 5rem; 1002 | grid-template-columns: 3fr 1fr; 1003 | } 1004 | 1005 | .article-header h1 { 1006 | font-size: 2.8rem; 1007 | } 1008 | } 1009 | 1010 | .post-thumbnail { 1011 | display: block !important; 1012 | } 1013 | 1014 | .post-thumbnail.gatsby-image-wrapper { 1015 | min-width: 75px; 1016 | height: 75px !important; 1017 | width: 75px !important; 1018 | margin-bottom: 0.5rem; 1019 | } 1020 | 1021 | @media screen and (min-width: 800px) { 1022 | .post-thumbnail.gatsby-image-wrapper { 1023 | min-width: 100px; 1024 | height: 100px !important; 1025 | width: 100px !important; 1026 | margin-right: 1.5rem; 1027 | margin-bottom: 0; 1028 | } 1029 | } 1030 | 1031 | /* Search */ 1032 | 1033 | .search-form { 1034 | display: flex; 1035 | } 1036 | 1037 | [type='search'] { 1038 | display: block; 1039 | padding: 0.8rem 1rem; 1040 | border: 1px solid var(--border); 1041 | width: 100%; 1042 | max-width: 400px; 1043 | border-radius: 0.3rem; 1044 | font-size: 1rem; 1045 | -webkit-appearance: none; 1046 | } 1047 | 1048 | .search-form [type='search'] { 1049 | border-right: none; 1050 | border-top-right-radius: 0; 1051 | border-bottom-right-radius: 0; 1052 | } 1053 | 1054 | button { 1055 | font-weight: 500; 1056 | font-size: 1.3rem; 1057 | border: 1px solid var(--border); 1058 | padding: 0.5rem 1rem; 1059 | border-top-right-radius: 0.3rem; 1060 | border-bottom-right-radius: 0.3rem; 1061 | border-top-left-radius: 0.3rem; 1062 | border-bottom-left-radius: 0.3rem; 1063 | cursor: pointer; 1064 | background: #f9f9f9; 1065 | } 1066 | 1067 | button:hover { 1068 | background: #eee; 1069 | } 1070 | 1071 | ::placeholder, 1072 | ::-webkit-input-placeholder, 1073 | ::-moz-placeholder, 1074 | :-moz-placeholder, 1075 | :-ms-input-placeholder { 1076 | color: var(--light-font-color); 1077 | } 1078 | 1079 | .button { 1080 | font-weight: 500; 1081 | font-size: 1.1rem; 1082 | border: 1px solid var(--link-color); 1083 | padding: 0.5rem 1rem; 1084 | cursor: pointer; 1085 | background: var(--link-color); 1086 | color: white; 1087 | border-radius: 0.3rem; 1088 | } 1089 | 1090 | .button:hover { 1091 | border: 1px solid #364fc7; 1092 | background: #364fc7; 1093 | color: white; 1094 | } 1095 | 1096 | /* Suggested */ 1097 | 1098 | .suggested { 1099 | flex-direction: column; 1100 | align-items: stretch; 1101 | margin-left: -1rem; 1102 | margin-right: -1rem; 1103 | padding: 0; 1104 | } 1105 | 1106 | .suggested a { 1107 | background: none; 1108 | text-align: center; 1109 | margin: 1rem; 1110 | border-bottom: none; 1111 | transition: all 0.2s ease; 1112 | padding: 1.5rem; 1113 | border-radius: 0.3rem; 1114 | background: var(--light-background); 1115 | } 1116 | 1117 | .suggested a:hover { 1118 | transform: translate3D(0, -1px, 0); 1119 | background: var(--lighter-background); 1120 | } 1121 | 1122 | @media screen and (min-width: 800px) { 1123 | .suggested { 1124 | flex-direction: row; 1125 | } 1126 | .suggested a { 1127 | flex: 0 0 calc(50% - 2rem); 1128 | } 1129 | } 1130 | 1131 | /* Helpers */ 1132 | 1133 | .small { 1134 | max-width: 600px; 1135 | } 1136 | 1137 | .medium { 1138 | max-width: 800px; 1139 | } 1140 | 1141 | time, 1142 | .meta { 1143 | color: var(--light-font-color); 1144 | font-size: 0.9rem; 1145 | white-space: nowrap; 1146 | font-weight: 400; 1147 | } 1148 | 1149 | .meta { 1150 | color: var(--medium-font-color); 1151 | font-size: 1rem; 1152 | } 1153 | 1154 | .text-center { 1155 | text-align: center; 1156 | } 1157 | 1158 | /** react auto-suggest */ 1159 | 1160 | .react-autosuggest__container { 1161 | position: relative; 1162 | width: 100%; 1163 | max-width: 400px; 1164 | margin: 0px 1rem; 1165 | } 1166 | 1167 | .react-autosuggest__suggestions-container--open { 1168 | display: block; 1169 | position: absolute; 1170 | width: 100%; 1171 | max-width: 400px; 1172 | border: 1px solid #aaa; 1173 | background-color: #fff; 1174 | border-radius: 0.3rem; 1175 | z-index: 2; 1176 | } 1177 | 1178 | .react-autosuggest__suggestions-list { 1179 | margin: 0 0; 1180 | padding: 0 0; 1181 | font-size: 1rem; 1182 | list-style-type: none; 1183 | } 1184 | 1185 | .react-autosuggest__suggestion { 1186 | padding: 0.25rem 1rem; 1187 | overflow-wrap: break-word; 1188 | } 1189 | 1190 | .react-autosuggest__suggestion--highlighted { 1191 | background-color: #ddd; 1192 | } 1193 | 1194 | /* tags */ 1195 | 1196 | .tags { 1197 | display: flex !important; 1198 | flex-wrap: wrap; 1199 | align-items: center; 1200 | margin-left: -0.5rem; 1201 | margin-right: -0.5rem; 1202 | } 1203 | 1204 | /* list bar */ 1205 | .list-bar { 1206 | display: flex; 1207 | } 1208 | 1209 | .list-bar .item { 1210 | display: flex; 1211 | justify-content: center; 1212 | align-items: center; 1213 | width: 33%; 1214 | 1215 | font-size: 1.25rem; 1216 | font-weight: 600; 1217 | margin: 0.2rem; 1218 | line-height: 1.5; 1219 | /*padding: 0.5rem 1rem;*/ 1220 | } 1221 | 1222 | .list-bar .item > button { 1223 | display: block; 1224 | font-weight: 600; 1225 | color: #343a40; 1226 | border-radius: 0; 1227 | border: 0; 1228 | background: linear-gradient(transparent 80%, #dbe4ff 0); 1229 | white-space: nowrap; 1230 | Outline: none; 1231 | } 1232 | 1233 | .list-bar .item > button:active { 1234 | border-radius: 0; 1235 | transform: translateY(2px); 1236 | } --------------------------------------------------------------------------------