├── .gitattributes ├── .gitignore ├── README.md ├── crawler ├── config │ └── config.go ├── engine │ ├── concurrent.go │ ├── simple.go │ ├── types.go │ └── worker.go ├── fetcher │ └── fetcher.go ├── frontend │ ├── Dockerfile │ ├── Makefile │ ├── controller │ │ └── searchresult.go │ ├── model │ │ └── page.go │ ├── starter.go │ └── view │ │ ├── css │ │ └── style.css │ │ ├── index.html │ │ ├── js │ │ └── index.js │ │ ├── logo.png │ │ ├── searchresult.go │ │ ├── searchresult_test.go │ │ └── template.html ├── main.go ├── persist │ ├── itemsaver.go │ └── itemsaver_test.go ├── scheduler │ ├── queued.go │ └── simple.go ├── xcar │ └── parser │ │ ├── cardetail.go │ │ ├── cardetail_test.go │ │ ├── cardetail_test_data.html │ │ ├── carlist.go │ │ ├── carlist_test.go │ │ ├── carlist_test_data.html │ │ ├── carmodel.go │ │ ├── carmodel_test.go │ │ └── carmodel_test_data.html └── zhenai │ └── parser │ ├── city.go │ ├── citylist.go │ ├── citylist_test.go │ ├── citylist_test_data.html │ ├── profile.go │ ├── profile_test.go │ └── profile_test_data.html ├── crawler_distributed ├── Dockerfile ├── Makefile ├── bloom │ ├── bloom.go │ └── bloom_test.go ├── config │ └── config.go ├── consul │ └── consul.go ├── data │ └── services.json ├── gredis │ └── gredis.go ├── main.go ├── persist │ ├── client │ │ └── itemsaver.go │ └── server │ │ ├── Dockerfile │ │ ├── Makefile │ │ ├── client_test.go │ │ └── itemsaver.go ├── proto │ ├── crawler.pb.go │ └── crawler.proto ├── rpcsupport │ ├── handle.go │ └── rpc.go └── worker │ ├── client │ └── worker.go │ ├── server │ ├── Dockerfile │ ├── Makefile │ ├── client_test.go │ └── worker.go │ └── types.go ├── docker-compose.yml ├── go.mod └── go.sum /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-language=Go 2 | *.css linguist-language=Go 3 | *.html linguist-language=Go 4 | *.go linguist-language=Go 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | .DS_Store 17 | .DS_Store? 18 | ._* 19 | .Spotlight-V100 20 | .Trashes 21 | Icon? 22 | ehthumbs.db 23 | Thumbs.db 24 | .idea 25 | *.iml 26 | crawler_distributed/client 27 | kompose 28 | crawler_distributed/worker/server/work 29 | crawler_distributed/persist/server/item 30 | crawler_distributed/data/redis 31 | crawler/frontend/frontend 32 | 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Go Distributed Reptile 2 | - 相信这个项目，对于学习分布式系统或者爬虫的你来说，帮助是巨大的。 3 | - 项目的思路来自于慕课网的ccmouse老师[链接](https://coding.imooc.com/learn/list/180.html) 4 | - 项目视频[介绍](https://www.bilibili.com/video/BV1n7411W7BB) 5 | - 微服务教程[学习吧](https://study.163.com/course/courseMain.htm?courseId=1209482821) 6 | ## 技术栈 7 | **Go, Protobuf, Consul, Docker, Elasticsearch, BloomFilter** 8 | ### 必须要做的 9 | - git clone https://github.com/apple-han/island.git 10 | - cd island 11 | - 全局搜索192.168.31.231 换成你主机的IP地址(这里因为有json文件,不好做全局的配置) 12 | 13 | ### Docker的方式部署 14 | - cd crawler_distributed/persist/server 15 | 16 | ``` 17 | make build 18 | make docker 19 | ``` 20 | 21 | - cd crawler_distributed/worker/server 22 | ``` 23 | make build 24 | make docker 25 | ``` 26 | 27 | - cd crawler_distributed 28 | ``` 29 | make build 30 | make docker 31 | ``` 32 | 33 | - cd crawler/frontend 34 | ``` 35 | make build 36 | make docker 37 | ``` 38 | 39 | 40 | - cd island 41 | ``` 42 | docker-compose up -d 43 | ``` 44 | - http://192.168.31.231:8888/search?q=大众(自己的ip) 45 | 46 | 47 | ### 小贴士 48 | 1. 由于系统是一个分布式的,所以整体下来 还是有一点难度 49 | 2. 大家好好看一下吧,欢迎大家多多的PR 50 | 51 | -------------------------------------------------------------------------------- /crawler/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | const ( 4 | // Parser names 5 | ParseCity = "ParseCity" 6 | ParseCityList = "ParseCityList" 7 | ParseProfile = "ParseProfile" 8 | 9 | ParseCarDetail = "ParseCarDetail" 10 | ParseCarList = "ParseCarList" 11 | ParseCarModel = "ParseCarModel" 12 | 13 | NilParser = "NilParser" 14 | 15 | // ElasticSearch 16 | ElasticIndex = "car_profile" 17 | 18 | // Rate limiting 19 | Qps = 2 20 | ) 21 | -------------------------------------------------------------------------------- /crawler/engine/concurrent.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "island/crawler_distributed/bloom" 5 | pb "island/crawler_distributed/proto" 6 | "log" 7 | ) 8 | 9 | type ConcurrentEngine struct { 10 | Scheduler Scheduler 11 | WorkerCount int 12 | ItemChan chan pb.Item 13 | RequestProcessor Processor 14 | } 15 | 16 | type Processor func(Request) (ParseResult, error) 17 | 18 | type Scheduler interface { 19 | ReadyNotifier 20 | Submit(Request) 21 | WorkerChan() chan Request 22 | Run() 23 | } 24 | 25 | type ReadyNotifier interface { 26 | WorkerReady(chan Request) 27 | } 28 | 29 | func (e *ConcurrentEngine) Run(seeds ...Request) { 30 | out := make(chan ParseResult) 31 | e.Scheduler.Run() 32 | for i := 0; i < e.WorkerCount; i++ { 33 | e.createWorker(e.Scheduler.WorkerChan(), 34 | out, e.Scheduler) 35 | } 36 | 37 | for _, r := range seeds { 38 | // 去重 39 | if isDuplicate(r.Url) { 40 | continue 41 | } 42 | e.Scheduler.Submit(r) 43 | } 44 | 45 | for { 46 | result := <-out 47 | for _, item := range result.Items { 48 | go func(i pb.Item) { 49 | e.ItemChan <- i 50 | }(*item) 51 | } 52 | 53 | for _, request := range result.Requests { 54 | if isDuplicate(request.Url) { 55 | continue 56 | } 57 | e.Scheduler.Submit(request) 58 | } 59 | } 60 | } 61 | 62 | func (e *ConcurrentEngine) createWorker( 63 | in chan Request, 64 | out chan ParseResult, ready ReadyNotifier) { 65 | go func() { 66 | for { 67 | ready.WorkerReady(in) 68 | request := <-in 69 | result, err := e.RequestProcessor( 70 | request) 71 | if err != nil { 72 | continue 73 | } 74 | // 这里是获取的结果 75 | out <- result 76 | } 77 | }() 78 | } 79 | 80 | var visitedUrls = make(map[string]bool) 81 | 82 | //func isDuplicate(url string) bool { 83 | // if visitedUrls[url] { 84 | // return true 85 | // } 86 | // 87 | // visitedUrls[url] = true 88 | // return false 89 | //} 90 | 91 | func isDuplicate(url string) bool { 92 | b, err := bloom.NewBloomFilter().IsContains(url) 93 | if err != nil{ 94 | log.Println("IsContains failed:", err.Error()) 95 | return false 96 | } 97 | if b == 1{ 98 | return true 99 | } 100 | err = bloom.NewBloomFilter().Insert(url) 101 | if err != nil{ 102 | log.Println("Insert failed:%s", err.Error()) 103 | return false 104 | } 105 | return false 106 | } 107 | -------------------------------------------------------------------------------- /crawler/engine/simple.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | type SimpleEngine struct{} 8 | 9 | func (e SimpleEngine) Run(seeds ...Request) { 10 | var requests []Request 11 | for _, r := range seeds { 12 | requests = append(requests, r) 13 | } 14 | 15 | for len(requests) > 0 { 16 | r := requests[0] 17 | requests = requests[1:] 18 | 19 | parseResult, err := Worker(r) 20 | if err != nil { 21 | continue 22 | } 23 | 24 | requests = append(requests, 25 | parseResult.Requests...) 26 | 27 | for _, item := range parseResult.Items { 28 | log.Printf("Got item: %v", item) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /crawler/engine/types.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "island/crawler/config" 5 | pb "island/crawler_distributed/proto" 6 | ) 7 | 8 | type ParserFunc func( 9 | contents []byte, url string) ParseResult 10 | 11 | type Parser interface { 12 | Parse(contents []byte, url string) ParseResult 13 | Serialize() (name string, args string) 14 | } 15 | 16 | type Request struct { 17 | Url string 18 | Parser Parser 19 | } 20 | 21 | type ParseResult struct { 22 | Requests []Request 23 | Items []*pb.Item 24 | } 25 | 26 | type NilParser struct{} 27 | 28 | func (NilParser) Parse( 29 | _ []byte, _ string) ParseResult { 30 | return ParseResult{} 31 | } 32 | 33 | func (NilParser) Serialize() ( 34 | name string, args string) { 35 | return config.NilParser, "" 36 | } 37 | 38 | type FuncParser struct { 39 | parser ParserFunc 40 | name string 41 | } 42 | 43 | func (f *FuncParser) Parse( 44 | contents []byte, url string) ParseResult { 45 | return f.parser(contents, url) 46 | } 47 | 48 | func (f *FuncParser) Serialize() ( 49 | name string, args string) { 50 | return f.name, "" 51 | } 52 | 53 | func NewFuncParser( 54 | p ParserFunc, name string) *FuncParser { 55 | return &FuncParser{ 56 | parser: p, 57 | name: name, 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /crawler/engine/worker.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "log" 5 | 6 | "island/crawler/fetcher" 7 | ) 8 | 9 | func Worker(r Request) (ParseResult, error) { 10 | body, err := fetcher.Fetch(r.Url) 11 | if err != nil { 12 | log.Printf("Fetcher: error "+ 13 | "fetching url %s: %v", 14 | r.Url, err) 15 | return ParseResult{}, err 16 | } 17 | 18 | return r.Parser.Parse(body, r.Url), nil 19 | } 20 | -------------------------------------------------------------------------------- /crawler/fetcher/fetcher.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "log" 10 | 11 | "time" 12 | 13 | "golang.org/x/net/html/charset" 14 | "golang.org/x/text/encoding" 15 | "golang.org/x/text/encoding/unicode" 16 | "golang.org/x/text/transform" 17 | "island/crawler/config" 18 | ) 19 | 20 | var ( 21 | rateLimiter = time.Tick( 22 | time.Second / config.Qps) 23 | verboseLogging = false 24 | ) 25 | 26 | func SetVerboseLogging() { 27 | verboseLogging = true 28 | } 29 | 30 | func Fetch(url string) ([]byte, error) { 31 | <-rateLimiter 32 | if verboseLogging { 33 | log.Printf("Fetching url %s", url) 34 | } 35 | resp, err := http.Get(url) 36 | if err != nil { 37 | return nil, err 38 | } 39 | defer resp.Body.Close() 40 | 41 | if resp.StatusCode != http.StatusOK { 42 | return nil, 43 | fmt.Errorf("wrong status code: %d", 44 | resp.StatusCode) 45 | } 46 | 47 | bodyReader := bufio.NewReader(resp.Body) 48 | e := determineEncoding(bodyReader) 49 | utf8Reader := transform.NewReader(bodyReader, 50 | e.NewDecoder()) 51 | return ioutil.ReadAll(utf8Reader) 52 | } 53 | 54 | func determineEncoding( 55 | r *bufio.Reader) encoding.Encoding { 56 | bytes, err := r.Peek(1024) 57 | if err != nil { 58 | log.Printf("Fetcher error: %v", err) 59 | return unicode.UTF8 60 | } 61 | e, _, _ := charset.DetermineEncoding( 62 | bytes, "") 63 | return e 64 | } 65 | -------------------------------------------------------------------------------- /crawler/frontend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | WORKDIR /frontend 3 | COPY . /frontend 4 | ENTRYPOINT [ "./frontend"] 5 | -------------------------------------------------------------------------------- /crawler/frontend/Makefile: -------------------------------------------------------------------------------- 1 | 2 | build: 3 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o frontend starter.go 4 | docker: 5 | docker build . -t frontend:latest 6 | 7 | -------------------------------------------------------------------------------- /crawler/frontend/controller/searchresult.go: -------------------------------------------------------------------------------- 1 | package controller 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "reflect" 7 | "regexp" 8 | c "island/crawler_distributed/config" 9 | pb "island/crawler_distributed/proto" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/olivere/elastic/v7" 14 | "island/crawler/config" 15 | "island/crawler/frontend/model" 16 | "island/crawler/frontend/view" 17 | ) 18 | 19 | type SearchResultHandler struct { 20 | view view.SearchResultView 21 | client *elastic.Client 22 | } 23 | 24 | func CreateSearchResultHandler( 25 | template string) SearchResultHandler { 26 | client, err := elastic.NewClient( 27 | elastic.SetURL(c.ElasticHost), 28 | elastic.SetSniff(false)) 29 | if err != nil { 30 | panic(err) 31 | } 32 | 33 | return SearchResultHandler{ 34 | view: view.CreateSearchResultView( 35 | template), 36 | client: client, 37 | } 38 | } 39 | 40 | func (h SearchResultHandler) ServeHTTP( 41 | w http.ResponseWriter, req *http.Request) { 42 | q := strings.TrimSpace(req.FormValue("q")) 43 | 44 | from, err := strconv.Atoi( 45 | req.FormValue("from")) 46 | if err != nil { 47 | from = 0 48 | } 49 | 50 | page, err := h.getSearchResult(q, from) 51 | if err != nil { 52 | http.Error(w, err.Error(), 53 | http.StatusBadRequest) 54 | return 55 | } 56 | 57 | err = h.view.Render(w, page) 58 | if err != nil { 59 | http.Error(w, err.Error(), 60 | http.StatusBadRequest) 61 | return 62 | } 63 | } 64 | 65 | const pageSize = 10 66 | 67 | func (h SearchResultHandler) getSearchResult( 68 | q string, from int) (model.SearchResult, error) { 69 | var result model.SearchResult 70 | result.Query = q 71 | 72 | resp, err := h.client. 73 | Search(config.ElasticIndex). 74 | Query(elastic.NewQueryStringQuery( 75 | rewriteQueryString(q))). 76 | From(from). 77 | Do(context.Background()) 78 | 79 | if err != nil { 80 | return result, err 81 | } 82 | 83 | result.Hits = resp.TotalHits() 84 | result.Start = from 85 | result.Items = resp.Each( 86 | reflect.TypeOf(pb.Item{})) 87 | if result.Start == 0 { 88 | result.PrevFrom = -1 89 | } else { 90 | result.PrevFrom = 91 | (result.Start - 1) / 92 | pageSize * pageSize 93 | } 94 | result.NextFrom = 95 | result.Start + len(result.Items) 96 | 97 | return result, nil 98 | } 99 | 100 | // Rewrites query string. Replaces field names 101 | // like "Age" to "Payload.Age" 102 | func rewriteQueryString(q string) string { 103 | re := regexp.MustCompile(`([A-Z][a-z]*):`) 104 | return re.ReplaceAllString(q, "Payload.$1:") 105 | } 106 | -------------------------------------------------------------------------------- /crawler/frontend/model/page.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type SearchResult struct { 4 | Hits int64 5 | Start int 6 | Query string 7 | PrevFrom int 8 | NextFrom int 9 | Items []interface{} 10 | } 11 | -------------------------------------------------------------------------------- /crawler/frontend/starter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | "island/crawler/frontend/controller" 7 | ) 8 | 9 | func main() { 10 | http.Handle("/", http.FileServer( 11 | http.Dir("view"))) 12 | http.Handle("/search", 13 | controller.CreateSearchResultHandler( 14 | "view/template.html")) 15 | err := http.ListenAndServe(":8888", nil) 16 | if err != nil { 17 | panic(err) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /crawler/frontend/view/css/style.css: -------------------------------------------------------------------------------- 1 | /* -- import Roboto Font ---------------------------- */ 2 | @import "https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic&subset=latin,cyrillic"; 3 | /* -- You can use this tables in Bootstrap (v3) projects. -- */ 4 | /* -- Box model ------------------------------- */ 5 | *, 6 | *:after, 7 | *:before { 8 | -webkit-box-sizing: border-box; 9 | -moz-box-sizing: border-box; 10 | box-sizing: border-box; 11 | } 12 | /* -- Demo style ------------------------------- */ 13 | html, 14 | body { 15 | position: relative; 16 | min-height: 100%; 17 | height: 100%; 18 | } 19 | html { 20 | position: relative; 21 | overflow-x: hidden; 22 | margin: 16px; 23 | padding: 0; 24 | min-height: 100%; 25 | font-size: 62.5%; 26 | } 27 | body { 28 | font-family: 'RobotoDraft', 'Roboto', 'Helvetica Neue, Helvetica, Arial', sans-serif; 29 | font-style: normal; 30 | font-weight: 300; 31 | font-size: 1.4rem; 32 | line-height: 2rem; 33 | letter-spacing: 0.01rem; 34 | color: #212121; 35 | background-color: #f5f5f5; 36 | -webkit-font-smoothing: antialiased; 37 | -moz-osx-font-smoothing: grayscale; 38 | text-rendering: optimizeLegibility; 39 | } 40 | #demo { 41 | margin: 20px auto; 42 | max-width: 1200px; 43 | } 44 | #demo h1 { 45 | font-size: 2.4rem; 46 | line-height: 3.2rem; 47 | letter-spacing: 0; 48 | font-weight: 300; 49 | color: #212121; 50 | text-transform: inherit; 51 | margin-bottom: 1rem; 52 | text-align: center; 53 | } 54 | #demo h2 { 55 | font-size: 1.5rem; 56 | line-height: 2.8rem; 57 | letter-spacing: 0.01rem; 58 | font-weight: 400; 59 | color: #212121; 60 | text-align: center; 61 | } 62 | .shadow-z-1 { 63 | -webkit-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); 64 | -moz-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); 65 | box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); 66 | } 67 | /* -- Material Design Table style -------------- */ 68 | .table { 69 | width: 100%; 70 | max-width: 100%; 71 | margin-bottom: 2rem; 72 | background-color: #fff; 73 | } 74 | .table > thead > tr, 75 | .table > tbody > tr, 76 | .table > tfoot > tr { 77 | -webkit-transition: all 0.3s ease; 78 | -o-transition: all 0.3s ease; 79 | transition: all 0.3s ease; 80 | } 81 | .table > thead > tr > th, 82 | .table > tbody > tr > th, 83 | .table > tfoot > tr > th, 84 | .table > thead > tr > td, 85 | .table > tbody > tr > td, 86 | .table > tfoot > tr > td { 87 | text-align: left; 88 | padding: 1.6rem; 89 | vertical-align: top; 90 | border-top: 0; 91 | -webkit-transition: all 0.3s ease; 92 | -o-transition: all 0.3s ease; 93 | transition: all 0.3s ease; 94 | } 95 | .table > thead > tr > th { 96 | font-weight: 400; 97 | color: #757575; 98 | vertical-align: bottom; 99 | border-bottom: 1px solid rgba(0, 0, 0, 0.12); 100 | } 101 | .table > caption + thead > tr:first-child > th, 102 | .table > colgroup + thead > tr:first-child > th, 103 | .table > thead:first-child > tr:first-child > th, 104 | .table > caption + thead > tr:first-child > td, 105 | .table > colgroup + thead > tr:first-child > td, 106 | .table > thead:first-child > tr:first-child > td { 107 | border-top: 0; 108 | } 109 | .table > tbody + tbody { 110 | border-top: 1px solid rgba(0, 0, 0, 0.12); 111 | } 112 | .table .table { 113 | background-color: #fff; 114 | } 115 | .table .no-border { 116 | border: 0; 117 | } 118 | .table-condensed > thead > tr > th, 119 | .table-condensed > tbody > tr > th, 120 | .table-condensed > tfoot > tr > th, 121 | .table-condensed > thead > tr > td, 122 | .table-condensed > tbody > tr > td, 123 | .table-condensed > tfoot > tr > td { 124 | padding: 0.8rem; 125 | } 126 | .table-bordered { 127 | border: 0; 128 | } 129 | .table-bordered > thead > tr > th, 130 | .table-bordered > tbody > tr > th, 131 | .table-bordered > tfoot > tr > th, 132 | .table-bordered > thead > tr > td, 133 | .table-bordered > tbody > tr > td, 134 | .table-bordered > tfoot > tr > td { 135 | border: 0; 136 | border-bottom: 1px solid #e0e0e0; 137 | } 138 | .table-bordered > thead > tr > th, 139 | .table-bordered > thead > tr > td { 140 | border-bottom-width: 2px; 141 | } 142 | .table-striped > tbody > tr:nth-child(odd) > td, 143 | .table-striped > tbody > tr:nth-child(odd) > th { 144 | background-color: #f5f5f5; 145 | } 146 | .table-hover > tbody > tr:hover > td, 147 | .table-hover > tbody > tr:hover > th { 148 | background-color: rgba(0, 0, 0, 0.12); 149 | } 150 | @media screen and (max-width: 768px) { 151 | .table-responsive-vertical > .table { 152 | margin-bottom: 0; 153 | background-color: transparent; 154 | } 155 | .table-responsive-vertical > .table > thead, 156 | .table-responsive-vertical > .table > tfoot { 157 | display: none; 158 | } 159 | .table-responsive-vertical > .table > tbody { 160 | display: block; 161 | } 162 | .table-responsive-vertical > .table > tbody > tr { 163 | display: block; 164 | border: 1px solid #e0e0e0; 165 | border-radius: 2px; 166 | margin-bottom: 1.6rem; 167 | } 168 | .table-responsive-vertical > .table > tbody > tr > td { 169 | background-color: #fff; 170 | display: block; 171 | vertical-align: middle; 172 | text-align: right; 173 | } 174 | .table-responsive-vertical > .table > tbody > tr > td[data-title]:before { 175 | content: attr(data-title); 176 | float: left; 177 | font-size: inherit; 178 | font-weight: 400; 179 | color: #757575; 180 | } 181 | .table-responsive-vertical.shadow-z-1 { 182 | -webkit-box-shadow: none; 183 | -moz-box-shadow: none; 184 | box-shadow: none; 185 | } 186 | .table-responsive-vertical.shadow-z-1 > .table > tbody > tr { 187 | border: none; 188 | -webkit-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); 189 | -moz-box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); 190 | box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.12), 0 1px 2px 0 rgba(0, 0, 0, 0.24); 191 | } 192 | .table-responsive-vertical > .table-bordered { 193 | border: 0; 194 | } 195 | .table-responsive-vertical > .table-bordered > tbody > tr > td { 196 | border: 0; 197 | border-bottom: 1px solid #e0e0e0; 198 | } 199 | .table-responsive-vertical > .table-bordered > tbody > tr > td:last-child { 200 | border-bottom: 0; 201 | } 202 | .table-responsive-vertical > .table-striped > tbody > tr > td, 203 | .table-responsive-vertical > .table-striped > tbody > tr:nth-child(odd) { 204 | background-color: #fff; 205 | } 206 | .table-responsive-vertical > .table-striped > tbody > tr > td:nth-child(odd) { 207 | background-color: #f5f5f5; 208 | } 209 | .table-responsive-vertical > .table-hover > tbody > tr:hover > td, 210 | .table-responsive-vertical > .table-hover > tbody > tr:hover { 211 | background-color: #fff; 212 | } 213 | .table-responsive-vertical > .table-hover > tbody > tr > td:hover { 214 | background-color: rgba(0, 0, 0, 0.12); 215 | } 216 | } 217 | .table-striped.table-mc-red > tbody > tr:nth-child(odd) > td, 218 | .table-striped.table-mc-red > tbody > tr:nth-child(odd) > th { 219 | background-color: #fde0dc; 220 | } 221 | .table-hover.table-mc-red > tbody > tr:hover > td, 222 | .table-hover.table-mc-red > tbody > tr:hover > th { 223 | background-color: #f9bdbb; 224 | } 225 | @media screen and (max-width: 767px) { 226 | .table-responsive-vertical .table-striped.table-mc-red > tbody > tr > td, 227 | .table-responsive-vertical .table-striped.table-mc-red > tbody > tr:nth-child(odd) { 228 | background-color: #fff; 229 | } 230 | .table-responsive-vertical .table-striped.table-mc-red > tbody > tr > td:nth-child(odd) { 231 | background-color: #fde0dc; 232 | } 233 | .table-responsive-vertical .table-hover.table-mc-red > tbody > tr:hover > td, 234 | .table-responsive-vertical .table-hover.table-mc-red > tbody > tr:hover { 235 | background-color: #fff; 236 | } 237 | .table-responsive-vertical .table-hover.table-mc-red > tbody > tr > td:hover { 238 | background-color: #f9bdbb; 239 | } 240 | } 241 | .table-striped.table-mc-pink > tbody > tr:nth-child(odd) > td, 242 | .table-striped.table-mc-pink > tbody > tr:nth-child(odd) > th { 243 | background-color: #fce4ec; 244 | } 245 | .table-hover.table-mc-pink > tbody > tr:hover > td, 246 | .table-hover.table-mc-pink > tbody > tr:hover > th { 247 | background-color: #f8bbd0; 248 | } 249 | @media screen and (max-width: 767px) { 250 | .table-responsive-vertical .table-striped.table-mc-pink > tbody > tr > td, 251 | .table-responsive-vertical .table-striped.table-mc-pink > tbody > tr:nth-child(odd) { 252 | background-color: #fff; 253 | } 254 | .table-responsive-vertical .table-striped.table-mc-pink > tbody > tr > td:nth-child(odd) { 255 | background-color: #fce4ec; 256 | } 257 | .table-responsive-vertical .table-hover.table-mc-pink > tbody > tr:hover > td, 258 | .table-responsive-vertical .table-hover.table-mc-pink > tbody > tr:hover { 259 | background-color: #fff; 260 | } 261 | .table-responsive-vertical .table-hover.table-mc-pink > tbody > tr > td:hover { 262 | background-color: #f8bbd0; 263 | } 264 | } 265 | .table-striped.table-mc-purple > tbody > tr:nth-child(odd) > td, 266 | .table-striped.table-mc-purple > tbody > tr:nth-child(odd) > th { 267 | background-color: #f3e5f5; 268 | } 269 | .table-hover.table-mc-purple > tbody > tr:hover > td, 270 | .table-hover.table-mc-purple > tbody > tr:hover > th { 271 | background-color: #e1bee7; 272 | } 273 | @media screen and (max-width: 767px) { 274 | .table-responsive-vertical .table-striped.table-mc-purple > tbody > tr > td, 275 | .table-responsive-vertical .table-striped.table-mc-purple > tbody > tr:nth-child(odd) { 276 | background-color: #fff; 277 | } 278 | .table-responsive-vertical .table-striped.table-mc-purple > tbody > tr > td:nth-child(odd) { 279 | background-color: #f3e5f5; 280 | } 281 | .table-responsive-vertical .table-hover.table-mc-purple > tbody > tr:hover > td, 282 | .table-responsive-vertical .table-hover.table-mc-purple > tbody > tr:hover { 283 | background-color: #fff; 284 | } 285 | .table-responsive-vertical .table-hover.table-mc-purple > tbody > tr > td:hover { 286 | background-color: #e1bee7; 287 | } 288 | } 289 | .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) > td, 290 | .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) > th { 291 | background-color: #ede7f6; 292 | } 293 | .table-hover.table-mc-deep-purple > tbody > tr:hover > td, 294 | .table-hover.table-mc-deep-purple > tbody > tr:hover > th { 295 | background-color: #d1c4e9; 296 | } 297 | @media screen and (max-width: 767px) { 298 | .table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr > td, 299 | .table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr:nth-child(odd) { 300 | background-color: #fff; 301 | } 302 | .table-responsive-vertical .table-striped.table-mc-deep-purple > tbody > tr > td:nth-child(odd) { 303 | background-color: #ede7f6; 304 | } 305 | .table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr:hover > td, 306 | .table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr:hover { 307 | background-color: #fff; 308 | } 309 | .table-responsive-vertical .table-hover.table-mc-deep-purple > tbody > tr > td:hover { 310 | background-color: #d1c4e9; 311 | } 312 | } 313 | .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) > td, 314 | .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) > th { 315 | background-color: #e8eaf6; 316 | } 317 | .table-hover.table-mc-indigo > tbody > tr:hover > td, 318 | .table-hover.table-mc-indigo > tbody > tr:hover > th { 319 | background-color: #c5cae9; 320 | } 321 | @media screen and (max-width: 767px) { 322 | .table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr > td, 323 | .table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr:nth-child(odd) { 324 | background-color: #fff; 325 | } 326 | .table-responsive-vertical .table-striped.table-mc-indigo > tbody > tr > td:nth-child(odd) { 327 | background-color: #e8eaf6; 328 | } 329 | .table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr:hover > td, 330 | .table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr:hover { 331 | background-color: #fff; 332 | } 333 | .table-responsive-vertical .table-hover.table-mc-indigo > tbody > tr > td:hover { 334 | background-color: #c5cae9; 335 | } 336 | } 337 | .table-striped.table-mc-blue > tbody > tr:nth-child(odd) > td, 338 | .table-striped.table-mc-blue > tbody > tr:nth-child(odd) > th { 339 | background-color: #e7e9fd; 340 | } 341 | .table-hover.table-mc-blue > tbody > tr:hover > td, 342 | .table-hover.table-mc-blue > tbody > tr:hover > th { 343 | background-color: #d0d9ff; 344 | } 345 | @media screen and (max-width: 767px) { 346 | .table-responsive-vertical .table-striped.table-mc-blue > tbody > tr > td, 347 | .table-responsive-vertical .table-striped.table-mc-blue > tbody > tr:nth-child(odd) { 348 | background-color: #fff; 349 | } 350 | .table-responsive-vertical .table-striped.table-mc-blue > tbody > tr > td:nth-child(odd) { 351 | background-color: #e7e9fd; 352 | } 353 | .table-responsive-vertical .table-hover.table-mc-blue > tbody > tr:hover > td, 354 | .table-responsive-vertical .table-hover.table-mc-blue > tbody > tr:hover { 355 | background-color: #fff; 356 | } 357 | .table-responsive-vertical .table-hover.table-mc-blue > tbody > tr > td:hover { 358 | background-color: #d0d9ff; 359 | } 360 | } 361 | .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) > td, 362 | .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) > th { 363 | background-color: #e1f5fe; 364 | } 365 | .table-hover.table-mc-light-blue > tbody > tr:hover > td, 366 | .table-hover.table-mc-light-blue > tbody > tr:hover > th { 367 | background-color: #b3e5fc; 368 | } 369 | @media screen and (max-width: 767px) { 370 | .table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr > td, 371 | .table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr:nth-child(odd) { 372 | background-color: #fff; 373 | } 374 | .table-responsive-vertical .table-striped.table-mc-light-blue > tbody > tr > td:nth-child(odd) { 375 | background-color: #e1f5fe; 376 | } 377 | .table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr:hover > td, 378 | .table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr:hover { 379 | background-color: #fff; 380 | } 381 | .table-responsive-vertical .table-hover.table-mc-light-blue > tbody > tr > td:hover { 382 | background-color: #b3e5fc; 383 | } 384 | } 385 | .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) > td, 386 | .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) > th { 387 | background-color: #e0f7fa; 388 | } 389 | .table-hover.table-mc-cyan > tbody > tr:hover > td, 390 | .table-hover.table-mc-cyan > tbody > tr:hover > th { 391 | background-color: #b2ebf2; 392 | } 393 | @media screen and (max-width: 767px) { 394 | .table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr > td, 395 | .table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr:nth-child(odd) { 396 | background-color: #fff; 397 | } 398 | .table-responsive-vertical .table-striped.table-mc-cyan > tbody > tr > td:nth-child(odd) { 399 | background-color: #e0f7fa; 400 | } 401 | .table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr:hover > td, 402 | .table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr:hover { 403 | background-color: #fff; 404 | } 405 | .table-responsive-vertical .table-hover.table-mc-cyan > tbody > tr > td:hover { 406 | background-color: #b2ebf2; 407 | } 408 | } 409 | .table-striped.table-mc-teal > tbody > tr:nth-child(odd) > td, 410 | .table-striped.table-mc-teal > tbody > tr:nth-child(odd) > th { 411 | background-color: #e0f2f1; 412 | } 413 | .table-hover.table-mc-teal > tbody > tr:hover > td, 414 | .table-hover.table-mc-teal > tbody > tr:hover > th { 415 | background-color: #b2dfdb; 416 | } 417 | @media screen and (max-width: 767px) { 418 | .table-responsive-vertical .table-striped.table-mc-teal > tbody > tr > td, 419 | .table-responsive-vertical .table-striped.table-mc-teal > tbody > tr:nth-child(odd) { 420 | background-color: #fff; 421 | } 422 | .table-responsive-vertical .table-striped.table-mc-teal > tbody > tr > td:nth-child(odd) { 423 | background-color: #e0f2f1; 424 | } 425 | .table-responsive-vertical .table-hover.table-mc-teal > tbody > tr:hover > td, 426 | .table-responsive-vertical .table-hover.table-mc-teal > tbody > tr:hover { 427 | background-color: #fff; 428 | } 429 | .table-responsive-vertical .table-hover.table-mc-teal > tbody > tr > td:hover { 430 | background-color: #b2dfdb; 431 | } 432 | } 433 | .table-striped.table-mc-green > tbody > tr:nth-child(odd) > td, 434 | .table-striped.table-mc-green > tbody > tr:nth-child(odd) > th { 435 | background-color: #d0f8ce; 436 | } 437 | .table-hover.table-mc-green > tbody > tr:hover > td, 438 | .table-hover.table-mc-green > tbody > tr:hover > th { 439 | background-color: #a3e9a4; 440 | } 441 | @media screen and (max-width: 767px) { 442 | .table-responsive-vertical .table-striped.table-mc-green > tbody > tr > td, 443 | .table-responsive-vertical .table-striped.table-mc-green > tbody > tr:nth-child(odd) { 444 | background-color: #fff; 445 | } 446 | .table-responsive-vertical .table-striped.table-mc-green > tbody > tr > td:nth-child(odd) { 447 | background-color: #d0f8ce; 448 | } 449 | .table-responsive-vertical .table-hover.table-mc-green > tbody > tr:hover > td, 450 | .table-responsive-vertical .table-hover.table-mc-green > tbody > tr:hover { 451 | background-color: #fff; 452 | } 453 | .table-responsive-vertical .table-hover.table-mc-green > tbody > tr > td:hover { 454 | background-color: #a3e9a4; 455 | } 456 | } 457 | .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) > td, 458 | .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) > th { 459 | background-color: #f1f8e9; 460 | } 461 | .table-hover.table-mc-light-green > tbody > tr:hover > td, 462 | .table-hover.table-mc-light-green > tbody > tr:hover > th { 463 | background-color: #dcedc8; 464 | } 465 | @media screen and (max-width: 767px) { 466 | .table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr > td, 467 | .table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr:nth-child(odd) { 468 | background-color: #fff; 469 | } 470 | .table-responsive-vertical .table-striped.table-mc-light-green > tbody > tr > td:nth-child(odd) { 471 | background-color: #f1f8e9; 472 | } 473 | .table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr:hover > td, 474 | .table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr:hover { 475 | background-color: #fff; 476 | } 477 | .table-responsive-vertical .table-hover.table-mc-light-green > tbody > tr > td:hover { 478 | background-color: #dcedc8; 479 | } 480 | } 481 | .table-striped.table-mc-lime > tbody > tr:nth-child(odd) > td, 482 | .table-striped.table-mc-lime > tbody > tr:nth-child(odd) > th { 483 | background-color: #f9fbe7; 484 | } 485 | .table-hover.table-mc-lime > tbody > tr:hover > td, 486 | .table-hover.table-mc-lime > tbody > tr:hover > th { 487 | background-color: #f0f4c3; 488 | } 489 | @media screen and (max-width: 767px) { 490 | .table-responsive-vertical .table-striped.table-mc-lime > tbody > tr > td, 491 | .table-responsive-vertical .table-striped.table-mc-lime > tbody > tr:nth-child(odd) { 492 | background-color: #fff; 493 | } 494 | .table-responsive-vertical .table-striped.table-mc-lime > tbody > tr > td:nth-child(odd) { 495 | background-color: #f9fbe7; 496 | } 497 | .table-responsive-vertical .table-hover.table-mc-lime > tbody > tr:hover > td, 498 | .table-responsive-vertical .table-hover.table-mc-lime > tbody > tr:hover { 499 | background-color: #fff; 500 | } 501 | .table-responsive-vertical .table-hover.table-mc-lime > tbody > tr > td:hover { 502 | background-color: #f0f4c3; 503 | } 504 | } 505 | .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) > td, 506 | .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) > th { 507 | background-color: #fffde7; 508 | } 509 | .table-hover.table-mc-yellow > tbody > tr:hover > td, 510 | .table-hover.table-mc-yellow > tbody > tr:hover > th { 511 | background-color: #fff9c4; 512 | } 513 | @media screen and (max-width: 767px) { 514 | .table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr > td, 515 | .table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr:nth-child(odd) { 516 | background-color: #fff; 517 | } 518 | .table-responsive-vertical .table-striped.table-mc-yellow > tbody > tr > td:nth-child(odd) { 519 | background-color: #fffde7; 520 | } 521 | .table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr:hover > td, 522 | .table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr:hover { 523 | background-color: #fff; 524 | } 525 | .table-responsive-vertical .table-hover.table-mc-yellow > tbody > tr > td:hover { 526 | background-color: #fff9c4; 527 | } 528 | } 529 | .table-striped.table-mc-amber > tbody > tr:nth-child(odd) > td, 530 | .table-striped.table-mc-amber > tbody > tr:nth-child(odd) > th { 531 | background-color: #fff8e1; 532 | } 533 | .table-hover.table-mc-amber > tbody > tr:hover > td, 534 | .table-hover.table-mc-amber > tbody > tr:hover > th { 535 | background-color: #ffecb3; 536 | } 537 | @media screen and (max-width: 767px) { 538 | .table-responsive-vertical .table-striped.table-mc-amber > tbody > tr > td, 539 | .table-responsive-vertical .table-striped.table-mc-amber > tbody > tr:nth-child(odd) { 540 | background-color: #fff; 541 | } 542 | .table-responsive-vertical .table-striped.table-mc-amber > tbody > tr > td:nth-child(odd) { 543 | background-color: #fff8e1; 544 | } 545 | .table-responsive-vertical .table-hover.table-mc-amber > tbody > tr:hover > td, 546 | .table-responsive-vertical .table-hover.table-mc-amber > tbody > tr:hover { 547 | background-color: #fff; 548 | } 549 | .table-responsive-vertical .table-hover.table-mc-amber > tbody > tr > td:hover { 550 | background-color: #ffecb3; 551 | } 552 | } 553 | .table-striped.table-mc-orange > tbody > tr:nth-child(odd) > td, 554 | .table-striped.table-mc-orange > tbody > tr:nth-child(odd) > th { 555 | background-color: #fff3e0; 556 | } 557 | .table-hover.table-mc-orange > tbody > tr:hover > td, 558 | .table-hover.table-mc-orange > tbody > tr:hover > th { 559 | background-color: #ffe0b2; 560 | } 561 | @media screen and (max-width: 767px) { 562 | .table-responsive-vertical .table-striped.table-mc-orange > tbody > tr > td, 563 | .table-responsive-vertical .table-striped.table-mc-orange > tbody > tr:nth-child(odd) { 564 | background-color: #fff; 565 | } 566 | .table-responsive-vertical .table-striped.table-mc-orange > tbody > tr > td:nth-child(odd) { 567 | background-color: #fff3e0; 568 | } 569 | .table-responsive-vertical .table-hover.table-mc-orange > tbody > tr:hover > td, 570 | .table-responsive-vertical .table-hover.table-mc-orange > tbody > tr:hover { 571 | background-color: #fff; 572 | } 573 | .table-responsive-vertical .table-hover.table-mc-orange > tbody > tr > td:hover { 574 | background-color: #ffe0b2; 575 | } 576 | } 577 | .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) > td, 578 | .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) > th { 579 | background-color: #fbe9e7; 580 | } 581 | .table-hover.table-mc-deep-orange > tbody > tr:hover > td, 582 | .table-hover.table-mc-deep-orange > tbody > tr:hover > th { 583 | background-color: #ffccbc; 584 | } 585 | @media screen and (max-width: 767px) { 586 | .table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr > td, 587 | .table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr:nth-child(odd) { 588 | background-color: #fff; 589 | } 590 | .table-responsive-vertical .table-striped.table-mc-deep-orange > tbody > tr > td:nth-child(odd) { 591 | background-color: #fbe9e7; 592 | } 593 | .table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr:hover > td, 594 | .table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr:hover { 595 | background-color: #fff; 596 | } 597 | .table-responsive-vertical .table-hover.table-mc-deep-orange > tbody > tr > td:hover { 598 | background-color: #ffccbc; 599 | } 600 | } 601 | -------------------------------------------------------------------------------- /crawler/frontend/view/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 |
20 |
21 | 22 |
23 | 24 | 25 |
26 |
27 | 28 |
29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /crawler/frontend/view/js/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by Kupletsky Sergey on 05.11.14. 3 | * 4 | * Material Design Responsive Table 5 | * Tested on Win8.1 with browsers: Chrome 37, Firefox 32, Opera 25, IE 11, Safari 5.1.7 6 | * You can use this table in Bootstrap (v3) projects. Material Design Responsive Table CSS-style will override basic bootstrap style. 7 | * JS used only for table constructor: you don't need it in your project 8 | */ 9 | 10 | $(document).ready(function() { 11 | 12 | var table = $('#table'); 13 | 14 | // Table bordered 15 | $('#table-bordered').change(function() { 16 | var value = $( this ).val(); 17 | table.removeClass('table-bordered').addClass(value); 18 | }); 19 | 20 | // Table striped 21 | $('#table-striped').change(function() { 22 | var value = $( this ).val(); 23 | table.removeClass('table-striped').addClass(value); 24 | }); 25 | 26 | // Table hover 27 | $('#table-hover').change(function() { 28 | var value = $( this ).val(); 29 | table.removeClass('table-hover').addClass(value); 30 | }); 31 | 32 | // Table color 33 | $('#table-color').change(function() { 34 | var value = $(this).val(); 35 | table.removeClass(/^table-mc-/).addClass(value); 36 | }); 37 | }); 38 | 39 | // jQuery’s hasClass and removeClass on steroids 40 | // by Nikita Vasilyev 41 | // https://github.com/NV/jquery-regexp-classes 42 | (function(removeClass) { 43 | 44 | jQuery.fn.removeClass = function( value ) { 45 | if ( value && typeof value.test === "function" ) { 46 | for ( var i = 0, l = this.length; i < l; i++ ) { 47 | var elem = this[i]; 48 | if ( elem.nodeType === 1 && elem.className ) { 49 | var classNames = elem.className.split( /\s+/ ); 50 | 51 | for ( var n = classNames.length; n--; ) { 52 | if ( value.test(classNames[n]) ) { 53 | classNames.splice(n, 1); 54 | } 55 | } 56 | elem.className = jQuery.trim( classNames.join(" ") ); 57 | } 58 | } 59 | } else { 60 | removeClass.call(this, value); 61 | } 62 | return this; 63 | } 64 | 65 | })(jQuery.fn.removeClass); 66 | -------------------------------------------------------------------------------- /crawler/frontend/view/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apple-han/island/7b6bdc72998d8e099c5a8698cc1801596306d9f0/crawler/frontend/view/logo.png -------------------------------------------------------------------------------- /crawler/frontend/view/searchresult.go: -------------------------------------------------------------------------------- 1 | package view 2 | 3 | import ( 4 | "html/template" 5 | "io" 6 | 7 | "island/crawler/frontend/model" 8 | ) 9 | 10 | type SearchResultView struct { 11 | template *template.Template 12 | } 13 | 14 | func CreateSearchResultView( 15 | filename string) SearchResultView { 16 | return SearchResultView{ 17 | template: template.Must( 18 | template.ParseFiles(filename)), 19 | } 20 | } 21 | 22 | func (s SearchResultView) Render( 23 | w io.Writer, data model.SearchResult) error { 24 | return s.template.Execute(w, data) 25 | } 26 | -------------------------------------------------------------------------------- /crawler/frontend/view/searchresult_test.go: -------------------------------------------------------------------------------- 1 | package view 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "island/crawler/engine" 8 | "island/crawler/frontend/model" 9 | common "island/crawler/model" 10 | ) 11 | 12 | func TestSearchResultView_Render(t *testing.T) { 13 | view := CreateSearchResultView( 14 | "template.html") 15 | 16 | out, err := os.Create("template.test.html") 17 | if err != nil { 18 | panic(err) 19 | } 20 | defer out.Close() 21 | 22 | page := model.SearchResult{} 23 | page.Hits = 123 24 | item := engine.Item{ 25 | Url: "http://album.zhenai.com/u/108906739", 26 | Type: "zhenai", 27 | Id: "108906739", 28 | Payload: common.Profile{ 29 | Age: 34, 30 | Height: 162, 31 | Weight: 57, 32 | Income: "3001-5000元", 33 | Gender: "女", 34 | Name: "安静的雪", 35 | Xinzuo: "牡羊座", 36 | Occupation: "人事/行政", 37 | Marriage: "离异", 38 | House: "已购房", 39 | Hokou: "山东菏泽", 40 | Education: "大学本科", 41 | Car: "未购车", 42 | }, 43 | } 44 | for i := 0; i < 10; i++ { 45 | page.Items = append(page.Items, item) 46 | } 47 | 48 | err = view.Render(out, page) 49 | if err != nil { 50 | t.Error(err) 51 | } 52 | 53 | // TODO: verify contents in template.test.html 54 | } 55 | -------------------------------------------------------------------------------- /crawler/frontend/view/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 |
20 |
21 |
22 | 23 | 24 |
25 |
26 |

共为你找到相关结果约为{{.Hits}}个。显示从{{.Start}}起共{{len .Items}}个。

27 | 28 | 29 | 30 |
31 | 32 | 33 | 34 | {{range .Items}} 35 | 36 | 37 | {{with .Car}} 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | {{end}} 48 | 49 | {{else}} 50 |
没有找到相关用户
51 | {{end}} 52 | 53 |
{{.Car.Name}}{{.Name}}{{.Price}}万元{{.Size}}{{.Fuel}}L/100km{{.Transmission}}{{.Engine}}{{.Displacement}}L{{.MaxSpeed}}km/h{{.Acceleration}}s,0-100km/h
54 |
55 | {{if ge .PrevFrom 0}} 56 | 上一页 57 | {{end}} 58 | 下一页 59 |
60 |
61 | 62 |
63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /crawler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "island/crawler/config" 5 | "island/crawler/engine" 6 | "island/crawler/persist" 7 | "island/crawler/scheduler" 8 | "island/crawler/xcar/parser" 9 | ) 10 | 11 | func main() { 12 | itemChan, err := persist.ItemSaver( 13 | config.ElasticIndex) 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | e := engine.ConcurrentEngine{ 19 | Scheduler: &scheduler.QueuedScheduler{}, 20 | WorkerCount: 100, 21 | ItemChan: itemChan, 22 | RequestProcessor: engine.Worker, 23 | } 24 | e.Run(engine.Request{ 25 | Url: "http://www.starter.url.here", 26 | Parser: engine.NewFuncParser( 27 | parser.ParseCarList, 28 | config.ParseCarList), 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /crawler/persist/itemsaver.go: -------------------------------------------------------------------------------- 1 | package persist 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | pb "island/crawler_distributed/proto" 8 | "island/crawler_distributed/config" 9 | "github.com/olivere/elastic/v7" 10 | ) 11 | 12 | func ItemSaver( 13 | index string) (chan pb.Item, error) { 14 | client, err := elastic.NewClient( 15 | // Must turn off sniff in docker 16 | elastic.SetURL(config.ElasticHost), 17 | elastic.SetSniff(false)) 18 | 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | out := make(chan pb.Item) 24 | go func() { 25 | itemCount := 0 26 | for { 27 | item := <-out 28 | log.Printf("Item Saver: got item "+ 29 | "#%d: %v", itemCount, item) 30 | itemCount++ 31 | 32 | err := Save(client, index, &item) 33 | if err != nil { 34 | log.Printf("Item Saver: error "+ 35 | "saving item %v: %v", 36 | item, err) 37 | } 38 | } 39 | }() 40 | 41 | return out, nil 42 | } 43 | 44 | func Save( 45 | client *elastic.Client, index string, 46 | item *pb.Item) error { 47 | 48 | if item.Type == "" { 49 | return errors.New("must supply Type") 50 | } 51 | 52 | indexService := client.Index(). 53 | Index(index). 54 | Type(item.Type). 55 | BodyJson(item) 56 | if item.Id != "" { 57 | indexService.Id(item.Id) 58 | } 59 | 60 | _, err := indexService. 61 | Do(context.Background()) 62 | 63 | return err 64 | } 65 | -------------------------------------------------------------------------------- /crawler/persist/itemsaver_test.go: -------------------------------------------------------------------------------- 1 | package persist 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "testing" 7 | 8 | "github.com/olivere/elastic/v7" 9 | "island/crawler/engine" 10 | "island/crawler/model" 11 | ) 12 | 13 | func TestSave(t *testing.T) { 14 | expected := engine.Item{ 15 | Url: "http://album.zhenai.com/u/108906739", 16 | Type: "zhenai", 17 | Id: "108906739", 18 | Payload: model.Profile{ 19 | Age: 34, 20 | Height: 162, 21 | Weight: 57, 22 | Income: "3001-5000元", 23 | Gender: "女", 24 | Name: "安静的雪", 25 | Xinzuo: "牡羊座", 26 | Occupation: "人事/行政", 27 | Marriage: "离异", 28 | House: "已购房", 29 | Hokou: "山东菏泽", 30 | Education: "大学本科", 31 | Car: "未购车", 32 | }, 33 | } 34 | 35 | // TODO: Try to start up elastic search 36 | // here using docker go client. 37 | client, err := elastic.NewClient( 38 | elastic.SetSniff(false)) 39 | 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | const index = "dating_test" 45 | // Save expected item 46 | err = Save(client, index, expected) 47 | 48 | if err != nil { 49 | panic(err) 50 | } 51 | 52 | // Fetch saved item 53 | resp, err := client.Get(). 54 | Index(index). 55 | Type(expected.Type). 56 | Id(expected.Id). 57 | Do(context.Background()) 58 | 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | t.Logf("%s", resp.Source) 64 | 65 | var actual engine.Item 66 | json.Unmarshal(resp.Source, &actual) 67 | 68 | actualProfile, _ := model.FromJsonObj( 69 | actual.Payload) 70 | actual.Payload = actualProfile 71 | 72 | // Verify result 73 | if actual != expected { 74 | t.Errorf("got %v; expected %v", 75 | actual, expected) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /crawler/scheduler/queued.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import "island/crawler/engine" 4 | 5 | type QueuedScheduler struct { 6 | requestChan chan engine.Request 7 | workerChan chan chan engine.Request 8 | } 9 | 10 | func (s *QueuedScheduler) WorkerChan() chan engine.Request { 11 | return make(chan engine.Request) 12 | } 13 | 14 | func (s *QueuedScheduler) Submit(r engine.Request) { 15 | s.requestChan <- r 16 | } 17 | 18 | func (s *QueuedScheduler) WorkerReady( 19 | w chan engine.Request) { 20 | s.workerChan <- w 21 | } 22 | 23 | func (s *QueuedScheduler) Run() { 24 | s.workerChan = make(chan chan engine.Request) 25 | s.requestChan = make(chan engine.Request) 26 | go func() { 27 | var requestQ []engine.Request 28 | var workerQ []chan engine.Request 29 | for { 30 | var activeRequest engine.Request 31 | var activeWorker chan engine.Request 32 | if len(requestQ) > 0 && 33 | len(workerQ) > 0 { 34 | activeWorker = workerQ[0] 35 | activeRequest = requestQ[0] 36 | } 37 | 38 | select { 39 | case r := <-s.requestChan: 40 | requestQ = append(requestQ, r) 41 | case w := <-s.workerChan: 42 | workerQ = append(workerQ, w) 43 | case activeWorker <- activeRequest: 44 | workerQ = workerQ[1:] 45 | requestQ = requestQ[1:] 46 | } 47 | } 48 | }() 49 | } 50 | -------------------------------------------------------------------------------- /crawler/scheduler/simple.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import "island/crawler/engine" 4 | 5 | type SimpleScheduler struct { 6 | workerChan chan engine.Request 7 | } 8 | 9 | func (s *SimpleScheduler) WorkerChan() chan engine.Request { 10 | return s.workerChan 11 | } 12 | 13 | func (s *SimpleScheduler) WorkerReady(chan engine.Request) { 14 | } 15 | 16 | func (s *SimpleScheduler) Run() { 17 | s.workerChan = make(chan engine.Request) 18 | } 19 | 20 | func (s *SimpleScheduler) Submit( 21 | r engine.Request) { 22 | go func() { s.workerChan <- r }() 23 | } 24 | -------------------------------------------------------------------------------- /crawler/xcar/parser/cardetail.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | pb "island/crawler_distributed/proto" 7 | "strconv" 8 | 9 | "island/crawler/engine" 10 | ) 11 | 12 | var priceReTmpl = `(\d+\.\d+)` 13 | 14 | var nameRe = regexp.MustCompile(`【(.*)报价_图片_参数】.*`) 15 | var carImageRe = regexp.MustCompile(`(\d+[^\d]\d+[^\d]\d+mm)`) 17 | var fuelRe = regexp.MustCompile(`(\d+\.\d+)L/100km`) 18 | var transmissionRe = regexp.MustCompile(`(.+)`) 19 | var engineRe = regexp.MustCompile(`发\s*动\s*机.*\s*.*<.*>(\d+kW[^<]*)<`) 20 | var displacementRe = regexp.MustCompile(`]*>(\d+)`) 22 | var accelRe = regexp.MustCompile(`]*>([\d\.]+)`) 23 | var urlRe = regexp.MustCompile(`http://newcar.xcar.com.cn/(m\d+)/`) 24 | 25 | func ParseCarDetail(contents []byte, url string) engine.ParseResult { 26 | id := extractString([]byte(url), urlRe) 27 | car := pb.Car{ 28 | Name: extractString(contents, nameRe), 29 | ImageURL: "http:" + extractString(contents, carImageRe), 30 | Size: extractString(contents, sizeRe), 31 | Fuel: extractFloat(contents, fuelRe), 32 | Transmission: extractString(contents, transmissionRe), 33 | Engine: extractString(contents, engineRe), 34 | Displacement: extractFloat(contents, displacementRe), 35 | MaxSpeed: extractFloat(contents, maxSpeedRe), 36 | Acceleration: extractFloat(contents, accelRe), 37 | } 38 | 39 | priceRe, err := regexp.Compile( 40 | fmt.Sprintf(priceReTmpl, regexp.QuoteMeta(id))) 41 | if err == nil { 42 | car.Price = extractFloat(contents, priceRe) 43 | } 44 | 45 | result := engine.ParseResult{ 46 | Items: []*pb.Item{ 47 | { 48 | Id: id, 49 | Url: url, 50 | Type: "xcar", 51 | Car: &car, 52 | }, 53 | }, 54 | } 55 | carModelResult := ParseCarModel(contents, url) 56 | result.Requests = carModelResult.Requests 57 | return result 58 | } 59 | 60 | func extractString( 61 | contents []byte, re *regexp.Regexp) string { 62 | match := re.FindSubmatch(contents) 63 | 64 | if len(match) >= 2 { 65 | return string(match[1]) 66 | } else { 67 | return "" 68 | } 69 | } 70 | 71 | func extractFloat(contents []byte, re *regexp.Regexp) float32 { 72 | f, err := strconv.ParseFloat(extractString(contents, re), 64) 73 | if err != nil { 74 | return 0 75 | } 76 | return float32(f) 77 | } 78 | -------------------------------------------------------------------------------- /crawler/xcar/parser/cardetail_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "island/crawler/engine" 8 | "island/crawler/model" 9 | ) 10 | 11 | func TestParseCarDetail(t *testing.T) { 12 | contents, err := ioutil.ReadFile( 13 | "cardetail_test_data.html") 14 | 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | expectedItem := engine.Item{ 20 | Url: "http://newcar.xcar.com.cn/m35001/", 21 | Type: "xcar", 22 | Id: "m35001", 23 | Payload: model.Car{ 24 | Name: "奥迪TT双门2017款45 TFSI", 25 | Price: 47.18, 26 | ImageURL: "http://img1.xcarimg.com/b63/s8386/m_20170616000036181753843373443.jpg-280x210.jpg", 27 | Size: "4191×1832×1353mm", 28 | Fuel: 16.7, 29 | Transmission: "6挡双离合", 30 | Engine: "169kW(2.0L涡轮增压)", 31 | Displacement: 2, 32 | MaxSpeed: 250, 33 | Acceleration: 5.9, 34 | }, 35 | } 36 | 37 | result := ParseCarDetail(contents, "http://newcar.xcar.com.cn/m35001/") 38 | 39 | if len(result.Items) != 1 { 40 | t.Errorf("Must only return one item, but %d returned", 41 | len(result.Items)) 42 | } 43 | 44 | actualItem := result.Items[0] 45 | if actualItem != expectedItem { 46 | t.Errorf("expected item: %+v, but was %+v", 47 | expectedItem, actualItem) 48 | } 49 | 50 | const resultSize = 8 51 | expectedUrls := []string{ 52 | "http://newcar.xcar.com.cn/m45776/", 53 | "http://newcar.xcar.com.cn/m45776/", 54 | "http://newcar.xcar.com.cn/m32946/", 55 | } 56 | 57 | if len(result.Requests) != resultSize { 58 | t.Errorf("result should have %d "+ 59 | "requests; but had %d", 60 | resultSize, len(result.Requests)) 61 | } 62 | for i, url := range expectedUrls { 63 | if result.Requests[i].Url != url { 64 | t.Errorf("expected url #%d: %s; but "+ 65 | "was %s", 66 | i, url, result.Requests[i].Url) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /crawler/xcar/parser/carlist.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "regexp" 5 | 6 | "island/crawler/config" 7 | "island/crawler/engine" 8 | ) 9 | 10 | const host = "http://newcar.xcar.com.cn" 11 | 12 | var carModelRe = regexp.MustCompile(``) 13 | var carListRe = regexp.MustCompile(`]*>([^<]+)`) 13 | cityUrlRe = regexp.MustCompile( 14 | `href="(http://www.zhenai.com/zhenghun/[^"]+)"`) 15 | ) 16 | 17 | func ParseCity( 18 | contents []byte, _ string) engine.ParseResult { 19 | matches := profileRe.FindAllSubmatch( 20 | contents, -1) 21 | 22 | result := engine.ParseResult{} 23 | for _, m := range matches { 24 | result.Requests = append( 25 | result.Requests, engine.Request{ 26 | Url: string(m[1]), 27 | Parser: NewProfileParser( 28 | string(m[2])), 29 | }) 30 | } 31 | 32 | matches = cityUrlRe.FindAllSubmatch( 33 | contents, -1) 34 | for _, m := range matches { 35 | result.Requests = append(result.Requests, 36 | engine.Request{ 37 | Url: string(m[1]), 38 | Parser: engine.NewFuncParser( 39 | ParseCity, config.ParseCity), 40 | }) 41 | } 42 | 43 | return result 44 | } 45 | -------------------------------------------------------------------------------- /crawler/zhenai/parser/citylist.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "regexp" 5 | 6 | "island/crawler/config" 7 | "island/crawler/engine" 8 | ) 9 | 10 | const cityListRe = `]*>([^<]+)` 11 | 12 | func ParseCityList( 13 | contents []byte, _ string) engine.ParseResult { 14 | re := regexp.MustCompile(cityListRe) 15 | matches := re.FindAllSubmatch(contents, -1) 16 | 17 | result := engine.ParseResult{} 18 | for _, m := range matches { 19 | result.Requests = append( 20 | result.Requests, engine.Request{ 21 | Url: string(m[1]), 22 | Parser: engine.NewFuncParser( 23 | ParseCity, config.ParseCity), 24 | }) 25 | } 26 | 27 | return result 28 | } 29 | -------------------------------------------------------------------------------- /crawler/zhenai/parser/citylist_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | 6 | "io/ioutil" 7 | ) 8 | 9 | func TestParseCityList(t *testing.T) { 10 | contents, err := ioutil.ReadFile( 11 | "citylist_test_data.html") 12 | 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | result := ParseCityList(contents, "") 18 | 19 | const resultSize = 470 20 | expectedUrls := []string{ 21 | "http://www.zhenai.com/zhenghun/aba", 22 | "http://www.zhenai.com/zhenghun/akesu", 23 | "http://www.zhenai.com/zhenghun/alashanmeng", 24 | } 25 | 26 | if len(result.Requests) != resultSize { 27 | t.Errorf("result should have %d "+ 28 | "requests; but had %d", 29 | resultSize, len(result.Requests)) 30 | } 31 | for i, url := range expectedUrls { 32 | if result.Requests[i].Url != url { 33 | t.Errorf("expected url #%d: %s; but "+ 34 | "was %s", 35 | i, url, result.Requests[i].Url) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /crawler/zhenai/parser/profile.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "regexp" 5 | pb "island/crawler_distributed/proto" 6 | "strconv" 7 | 8 | "island/crawler/config" 9 | "island/crawler/engine" 10 | ) 11 | 12 | var ageRe = regexp.MustCompile( 13 | `年龄：(\d+)岁`) 14 | var heightRe = regexp.MustCompile( 15 | `身高：(\d+)CM`) 16 | var incomeRe = regexp.MustCompile( 17 | `月收入：([^<]+)`) 18 | var weightRe = regexp.MustCompile( 19 | `体重：(\d+)KG`) 20 | var genderRe = regexp.MustCompile( 21 | `性别：([^<]+)`) 22 | var xinzuoRe = regexp.MustCompile( 23 | `星座：([^<]+)`) 24 | var marriageRe = regexp.MustCompile( 25 | `婚况：([^<]+)`) 26 | var educationRe = regexp.MustCompile( 27 | `学历：([^<]+)`) 28 | var occupationRe = regexp.MustCompile( 29 | `职业：([^<]+)`) 30 | var hokouRe = regexp.MustCompile( 31 | `籍贯：([^<]+)`) 32 | var houseRe = regexp.MustCompile( 33 | `住房条件：([^<]+)`) 34 | var carRe = regexp.MustCompile( 35 | `是否购车：([^<]+)`) 36 | var guessRe = regexp.MustCompile( 37 | `]*href="(http://album.zhenai.com/u/[\d]+)">([^<]+)`) 38 | var idUrlRe = regexp.MustCompile( 39 | `http://album.zhenai.com/u/([\d]+)`) 40 | 41 | func parseProfile( 42 | contents []byte, url string, 43 | name string) engine.ParseResult { 44 | profile := pb.Profile{} 45 | profile.Name = name 46 | 47 | age, err := strconv.Atoi( 48 | extractString(contents, ageRe)) 49 | if err == nil { 50 | profile.Age = int32(age) 51 | } 52 | 53 | height, err := strconv.Atoi( 54 | extractString(contents, heightRe)) 55 | if err == nil { 56 | profile.Height = int32(height) 57 | } 58 | 59 | weight, err := strconv.Atoi( 60 | extractString(contents, weightRe)) 61 | if err == nil { 62 | profile.Weight = int32(weight) 63 | } 64 | 65 | profile.Income = extractString( 66 | contents, incomeRe) 67 | profile.Gender = extractString( 68 | contents, genderRe) 69 | profile.Car = extractString( 70 | contents, carRe) 71 | profile.Education = extractString( 72 | contents, educationRe) 73 | profile.Hokou = extractString( 74 | contents, hokouRe) 75 | profile.House = extractString( 76 | contents, houseRe) 77 | profile.Marriage = extractString( 78 | contents, marriageRe) 79 | profile.Occupation = extractString( 80 | contents, occupationRe) 81 | profile.Xinzuo = extractString( 82 | contents, xinzuoRe) 83 | 84 | result := engine.ParseResult{ 85 | Items: []*pb.Item{ 86 | { 87 | Url: url, 88 | Type: "zhenai", 89 | Id: extractString( 90 | []byte(url), idUrlRe), 91 | Payload: &profile, 92 | }, 93 | }, 94 | } 95 | 96 | matches := guessRe.FindAllSubmatch( 97 | contents, -1) 98 | for _, m := range matches { 99 | result.Requests = append(result.Requests, 100 | engine.Request{ 101 | Url: string(m[1]), 102 | Parser: NewProfileParser( 103 | string(m[2])), 104 | }) 105 | } 106 | 107 | return result 108 | } 109 | 110 | func extractString( 111 | contents []byte, re *regexp.Regexp) string { 112 | match := re.FindSubmatch(contents) 113 | 114 | if len(match) >= 2 { 115 | return string(match[1]) 116 | } else { 117 | return "" 118 | } 119 | } 120 | 121 | type ProfileParser struct { 122 | userName string 123 | } 124 | 125 | func (p *ProfileParser) Parse( 126 | contents []byte, 127 | url string) engine.ParseResult { 128 | return parseProfile(contents, url, p.userName) 129 | } 130 | 131 | func (p *ProfileParser) Serialize() ( 132 | name string, args string) { 133 | return config.ParseProfile, p.userName 134 | } 135 | 136 | func NewProfileParser( 137 | name string) *ProfileParser { 138 | return &ProfileParser{ 139 | userName: name, 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /crawler/zhenai/parser/profile_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "island/crawler/engine" 8 | "island/crawler/model" 9 | ) 10 | 11 | func TestParseProfile(t *testing.T) { 12 | contents, err := ioutil.ReadFile( 13 | "profile_test_data.html") 14 | 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | result := parseProfile(contents, 20 | "http://album.zhenai.com/u/108906739", 21 | "安静的雪") 22 | 23 | if len(result.Items) != 1 { 24 | t.Errorf("Items should contain 1 "+ 25 | "element; but was %v", result.Items) 26 | } 27 | 28 | actual := result.Items[0] 29 | 30 | expected := engine.Item{ 31 | Url: "http://album.zhenai.com/u/108906739", 32 | Type: "zhenai", 33 | Id: "108906739", 34 | Payload: model.Profile{ 35 | Age: 34, 36 | Height: 162, 37 | Weight: 57, 38 | Income: "3001-5000元", 39 | Gender: "女", 40 | Name: "安静的雪", 41 | Xinzuo: "牡羊座", 42 | Occupation: "人事/行政", 43 | Marriage: "离异", 44 | House: "已购房", 45 | Hokou: "山东菏泽", 46 | Education: "大学本科", 47 | Car: "未购车", 48 | }, 49 | } 50 | 51 | if actual != expected { 52 | t.Errorf("expected %v; but was %v", 53 | expected, actual) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /crawler/zhenai/parser/profile_test_data.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 安静的雪资料照片_新疆阿勒泰征婚交友_珍爱网 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 |
120 |
121 | 122 |

相亲无难事，珍爱有红娘

123 |
124 | 132 |
133 |
134 |
135 |
136 | 137 | 153 | 154 | 164 |
165 | 166 | 175 |
176 | 177 | 178 |
179 |
180 |
181 | 182 |
183 |
184 |
185 | 安静的雪照片 186 |
187 |
188 |
189 |

安静的雪

190 |

34岁，新疆阿勒泰，162cm，3001-5000元

191 |
192 |
193 | 发邮件 194 | 195 | 196 | 197 | 打招呼 198 | 199 | 200 | 201 | 202 | 问问题 203 | 204 | 205 | 206 | 207 | 208 | 委托红娘 209 | 210 | 211 |
212 |
213 |
214 |
215 | 216 | 217 |
218 | 219 | 220 | 221 | 222 | 223 | 224 |
225 |
226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 |
240 |
241 |
242 |
243 | 244 |
245 |
246 |
247 |
248 |
249 | 安静的雪照片 250 | 251 |
    252 | 253 |
254 | 255 |
256 |
257 |
258 |
    259 | 260 | 261 | 262 |
  • 263 |

    264 | 265 | 266 |

    267 |
    • 268 | 269 |
270 |
271 |
272 |
273 | 274 | 275 |
276 |
277 |
278 |

279 | 安静的雪 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 698分 294 | 295 | 304 | 305 |

306 |

ID：108906739 307 | 诚信值：15 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 |

317 |
318 |
319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 |
年龄：34岁身高：162CM月收入：3001-5000元
婚况：离异学历：大学本科工作地：新疆阿勒泰
职业： 人事/行政有无孩子：有，我们住在一起籍贯：山东菏泽
338 |
339 |
340 |
341 | 342 | 343 | 发邮件 344 | 345 | 346 | 打招呼 347 | 348 | 349 | 350 | 351 | 问问题 352 | 353 | 354 | 355 | 356 | 357 | 委托红娘 358 | 359 | 360 |
361 | 362 |
363 | 364 | 关注 365 | 送礼物 366 | 367 | 在线聊天 368 | 369 | 情敌动态 370 | 371 | 举报/拉黑 372 | 373 |
374 |
375 |
376 | 377 | 378 |
379 | 380 |
381 |
382 | 386 |
387 |
388 | 389 |
390 |

不完美又何妨？万物皆有裂隙，那是光进来的地方。随缘

391 |
392 |
393 | 赞一下 394 | 395 |
396 | 397 |
398 |
399 | 400 |
401 |

想要听听TA对自己的想法？来邀请TA补充自我描述吧

402 | 403 |
404 |
405 |
406 | 407 |
408 |
409 |
410 |
411 | 412 |
413 | 414 |

详细资料

415 |
416 |
417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 |
性别：女生肖：--
身高：162CM星座：牡羊座
体重：57KG血型：--
体型：--职业：人事/行政
民族：汉族公司：--
信仰：--
443 |
444 |
445 |
446 | 447 |
448 | 449 |

生活状况

450 |
451 |
452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 |
住房条件：已购房想何时结婚：--
是否购车：未购车婚后与父母住吗：--
是否吸烟：不吸烟与对方父母同住：--
是否喝酒：不喝酒较大的消费：--
厨艺：--喜欢怎样的约会：--
家务：--
478 |
479 |
480 |
481 | 482 |
483 | 484 |

兴趣爱好

485 |
486 |
487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 |
喜欢的活动：--喜欢的食物：--
喜欢的体育运动：--喜欢的地方：--
喜欢的音乐：--喜欢的宠物：--
喜欢的影视节目：--
505 |
506 |
507 | 508 | 509 | 510 |
511 | 512 |
513 | 514 |

择偶条件

515 |
516 |
517 | 518 | 519 | 520 | 521 | 522 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 530 | 531 | 532 | 533 | 534 | 535 | 536 | 537 | 538 | 539 | 540 | 541 | 542 | 543 | 544 | 545 | 546 |
性别：男体型：不限
年龄：35 - 43岁职业：不限
身高：173 - 185厘米是否抽烟：不吸烟
学历：不限是否喝酒：不限
月收入：5000元以上有没有孩子：不限
婚况： 不限是否想要孩子：不限
工作地区：新疆阿勒泰是否有照片：不限
547 |
548 | 549 |
550 | TA的资料完整度只有55%，邀请完善资料 551 | 552 |
553 | 554 |
555 |
556 | 557 | 637 |
638 |
639 |
640 |
641 |
642 |
643 | 644 | 645 | 646 | 647 | 648 | 654 | 655 | 656 | 712 | 713 | 714 | 715 | 716 |
717 |
718 |
719 | 720 | 721 |

722 | 当前位置：珍爱首页>阿勒泰征婚>阿勒泰市征婚>阿勒泰市女士征婚 723 |

724 | 725 | 726 | 727 | 728 |

周边征婚：新疆征婚阿勒泰征婚阿勒泰市征婚博尔塔拉征婚巴音郭楞征婚克孜勒苏柯尔克孜征婚伊犁征婚喀什征婚乌鲁木齐征婚昌吉征婚阿克苏征婚哈密征婚克拉玛依征婚和田征婚

729 | 730 |
731 |
732 |
733 | 734 | 735 | 760 | 761 | 791 | 792 | 821 | 822 | 823 | -------------------------------------------------------------------------------- /crawler_distributed/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | ADD client /client 3 | ENTRYPOINT [ "/client"] 4 | -------------------------------------------------------------------------------- /crawler_distributed/Makefile: -------------------------------------------------------------------------------- 1 | 2 | build: 3 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o client ./main.go 4 | docker: 5 | docker build . -t client:latest 6 | 7 | -------------------------------------------------------------------------------- /crawler_distributed/bloom/bloom.go: -------------------------------------------------------------------------------- 1 | package bloom 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "errors" 7 | "github.com/gomodule/redigo/redis" 8 | "island/crawler_distributed/gredis" 9 | "strconv" 10 | ) 11 | var ( 12 | BitSize uint = 1<<31 13 | Seeds = []uint{1,2,3,5} 14 | BlockNum uint = 1 15 | Key = "bloomfilter" 16 | Ret = 1 17 | ) 18 | type SimpleHash struct { 19 | Cap uint 20 | Seed uint 21 | } 22 | 23 | type BloomFilter struct { 24 | BitSize uint 25 | Seeds []uint 26 | Key string 27 | BlockNum uint 28 | HashFunc []*SimpleHash 29 | } 30 | 31 | func NewBloomFilter() *BloomFilter{ 32 | var HashMiddleFunc []*SimpleHash 33 | for _, v := range Seeds{ 34 | HashMiddleFunc = append(HashMiddleFunc, &SimpleHash{ 35 | Cap: BitSize, 36 | Seed: v, 37 | }) 38 | } 39 | return &BloomFilter{ 40 | BitSize: BitSize, 41 | Seeds: Seeds, 42 | Key: Key, 43 | BlockNum: BlockNum, 44 | HashFunc: HashMiddleFunc, 45 | } 46 | } 47 | 48 | // IsContains int 1 represent has already existed 49 | func (b *BloomFilter) IsContains(s string) (int, error){ 50 | if len(s) == 0{ 51 | return 0, errors.New("input is must") 52 | } 53 | input := b.MD5(s) 54 | n, err := strconv.ParseUint(input[0:2], 16, 64) 55 | if err != nil { 56 | return 0, errors.New("string to uint64 fail") 57 | } 58 | name := b.Key + strconv.Itoa(int(uint(n) % b.BlockNum)) 59 | for _, f := range b.HashFunc{ 60 | loc := f.Hash(input) 61 | r, err := redis.Int(gredis.GetBit(name, loc)) 62 | if err != nil{ 63 | return 0, err 64 | } 65 | Ret = Ret & r 66 | } 67 | 68 | return Ret, nil 69 | } 70 | 71 | func (b *BloomFilter) Insert(s string) error{ 72 | if len(s) == 0{ 73 | return errors.New("input is must") 74 | } 75 | input := b.MD5(s) 76 | n, err := strconv.ParseUint(input[0:2], 16, 64) 77 | if err != nil { 78 | return errors.New("string to uint64 fail") 79 | } 80 | name := b.Key + strconv.Itoa(int(uint(n) % b.BlockNum)) 81 | for _, f := range b.HashFunc{ 82 | loc := f.Hash(input) 83 | gredis.SetBit(name, loc, 1) 84 | } 85 | return nil 86 | } 87 | 88 | func (s *SimpleHash) Hash(value string) uint{ 89 | var ret uint = 0 90 | for _, v := range value{ 91 | ret += s.Seed * ret + uint(v) 92 | } 93 | var bz = BitSize-1 94 | r := bz & ret 95 | return r 96 | } 97 | 98 | func (b *BloomFilter) MD5(value string) string { 99 | m := md5.New() 100 | m.Write([]byte(value)) 101 | return hex.EncodeToString(m.Sum(nil)) 102 | } 103 | -------------------------------------------------------------------------------- /crawler_distributed/bloom/bloom_test.go: -------------------------------------------------------------------------------- 1 | package bloom 2 | 3 | import ( 4 | "island/crawler_distributed/gredis" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestBloomFilter_Insert(t *testing.T) { 10 | gredis.Setup() 11 | i := 0 12 | for { 13 | err := NewBloomFilter().Insert("http://www.baidu.com" + strconv.Itoa(i)) 14 | if err != nil{ 15 | t.Errorf("err: %s", err.Error()) 16 | } 17 | i++ 18 | if i==1000{ 19 | break 20 | } 21 | } 22 | 23 | } 24 | 25 | func TestBloomFilter_IsContains(t *testing.T) { 26 | gredis.Setup() 27 | i := 0 28 | for { 29 | b, err := NewBloomFilter().IsContains("http://www.baidu.com" + strconv.Itoa(i)) 30 | if b == 0 || err != nil{ 31 | t.Errorf("result: %v; err: %s; i: %d", 32 | b, "fail", i) 33 | } 34 | i++ 35 | if i==1000{ 36 | break 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /crawler_distributed/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | const ( 4 | ConsulHost = "http://192.168.31.231:8500" 5 | ElasticHost = "http://192.168.31.231:9200" 6 | RedisHost = "192.168.31.231:6379" 7 | MaxIdle = 30 8 | MaxActive = 1000 9 | IdleTimeout = 10 10 | ) 11 | 12 | -------------------------------------------------------------------------------- /crawler_distributed/consul/consul.go: -------------------------------------------------------------------------------- 1 | package consul 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/consul/api" 6 | "log" 7 | "net" 8 | c "island/crawler_distributed/config" 9 | "strconv" 10 | ) 11 | 12 | func Find(name string) []string{ 13 | var lastIndex uint64 14 | config := api.DefaultConfig() 15 | config.Address = c.ConsulHost 16 | 17 | client, err := api.NewClient(config) 18 | if err != nil { 19 | fmt.Println("api new client is failed, err:", err) 20 | return nil 21 | } 22 | services, metainfo, err := client.Health().Service(name, "", true, &api.QueryOptions{ 23 | WaitIndex: lastIndex, 24 | }) 25 | if err != nil { 26 | log.Println("error retrieving instances from Consul: ", err) 27 | } 28 | lastIndex = metainfo.LastIndex 29 | 30 | addrs := make([]string, 0) 31 | for _, service := range services { 32 | addrs = append(addrs, net.JoinHostPort(service.Service.Address, strconv.Itoa(service.Service.Port))) 33 | } 34 | return addrs 35 | } 36 | -------------------------------------------------------------------------------- /crawler_distributed/data/services.json: -------------------------------------------------------------------------------- 1 | { 2 | "services": [ 3 | { 4 | "id": "item", 5 | "name": "item", 6 | "tags": [ 7 | "saveItem" 8 | ], 9 | "address": "192.168.31.231", 10 | "port": 1234, 11 | "checks": [ 12 | { 13 | "http": "http://192.168.31.231:8080/ping", 14 | "tls_skip_verify": false, 15 | "method": "GET", 16 | "interval": "10s", 17 | "timeout": "1s" 18 | } 19 | ] 20 | },{ 21 | "id": "car", 22 | "name": "car", 23 | "tags": [ 24 | "carOne" 25 | ], 26 | "address": "192.168.31.231", 27 | "port": 9001, 28 | "checks": [ 29 | { 30 | "http": "http://192.168.31.231:8081/ping", 31 | "tls_skip_verify": false, 32 | "method": "GET", 33 | "interval": "10s", 34 | "timeout": "1s" 35 | } 36 | ] 37 | },{ 38 | "id": "cartwo", 39 | "name": "car", 40 | "tags": [ 41 | "carTwo" 42 | ], 43 | "address": "192.168.31.231", 44 | "port": 9002, 45 | "checks": [ 46 | { 47 | "http": "http://192.168.31.231:8082/ping", 48 | "tls_skip_verify": false, 49 | "method": "GET", 50 | "interval": "10s", 51 | "timeout": "1s" 52 | } 53 | ] 54 | },{ 55 | "id": "carthree", 56 | "name": "car", 57 | "tags": [ 58 | "carThree" 59 | ], 60 | "address": "192.168.31.231", 61 | "port": 9003, 62 | "checks": [ 63 | { 64 | "http": "http://192.168.31.231:8083/ping", 65 | "tls_skip_verify": false, 66 | "method": "GET", 67 | "interval": "10s", 68 | "timeout": "1s" 69 | } 70 | ] 71 | } 72 | 73 | ] 74 | } 75 | 76 | -------------------------------------------------------------------------------- /crawler_distributed/gredis/gredis.go: -------------------------------------------------------------------------------- 1 | package gredis 2 | 3 | import ( 4 | "github.com/gomodule/redigo/redis" 5 | "github.com/prometheus/common/log" 6 | "island/crawler_distributed/config" 7 | "time" 8 | ) 9 | 10 | var RedisConn *redis.Pool 11 | 12 | // Setup Initialize the Redis instance 13 | func Setup() { 14 | RedisConn = &redis.Pool{ 15 | MaxIdle: config.MaxIdle, 16 | MaxActive: config.MaxActive, 17 | IdleTimeout: config.IdleTimeout, 18 | Dial: func() (redis.Conn, error) { 19 | c, err := redis.Dial("tcp", config.RedisHost, 20 | redis.DialConnectTimeout(time.Duration(3000)*time.Millisecond)) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return c, err 25 | }, 26 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 27 | _, err := c.Do("PING") 28 | return err 29 | }, 30 | } 31 | 32 | return 33 | } 34 | 35 | 36 | // Set redis hash表的使用 37 | func SetBit(key string, field, value interface{}) { 38 | conn := RedisConn.Get() 39 | defer conn.Close() 40 | _, err := conn.Do("SETBIT", key, field, value) 41 | if err != nil { 42 | log.Error("pkg.gredis.SETBIT Do error is:", err) 43 | return 44 | } 45 | return 46 | } 47 | 48 | // GetBit redis hash表的使用 49 | func GetBit(key string, field interface{}) (interface{}, error) { 50 | conn := RedisConn.Get() 51 | defer conn.Close() 52 | reply, err := conn.Do("GetBit", key, field) 53 | if err != nil { 54 | log.Error("pkg.gredis.HGet Do error is:", err) 55 | return nil, err 56 | } 57 | return reply, nil 58 | } 59 | -------------------------------------------------------------------------------- /crawler_distributed/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "island/crawler_distributed/consul" 6 | "island/crawler_distributed/gredis" 7 | pb "island/crawler_distributed/proto" 8 | 9 | "log" 10 | 11 | "island/crawler/config" 12 | "island/crawler/engine" 13 | "island/crawler/scheduler" 14 | "island/crawler/xcar/parser" 15 | itemsaver "island/crawler_distributed/persist/client" 16 | "island/crawler_distributed/rpcsupport" 17 | worker "island/crawler_distributed/worker/client" 18 | ) 19 | 20 | func main() { 21 | var ( 22 | itemChan chan pb.Item 23 | err error 24 | ) 25 | 26 | // 这里从服务发现中 发现地址 27 | if len(consul.Find("item")) > 0{ 28 | itemChan, err = itemsaver.ItemSaver(consul.Find("item")[0]) 29 | if err != nil { 30 | panic(err) 31 | } 32 | } 33 | 34 | pool, err := createClientPool(consul.Find("car")) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | processor := worker.CreateProcessor(pool) 40 | e := engine.ConcurrentEngine{ 41 | Scheduler: &scheduler.QueuedScheduler{}, 42 | WorkerCount: 2, 43 | ItemChan: itemChan, 44 | RequestProcessor: processor, 45 | } 46 | 47 | e.Run(engine.Request{ 48 | Url: "http://newcar.xcar.com.cn/car/", 49 | Parser: engine.NewFuncParser( 50 | parser.ParseCarList, 51 | config.ParseCarList), 52 | }) 53 | } 54 | 55 | func init(){ 56 | gredis.Setup() 57 | } 58 | 59 | func createClientPool( 60 | hosts []string) (chan pb.ReptilesClient, error) { 61 | var clients []pb.ReptilesClient 62 | for _, h := range hosts { 63 | client, err := rpcsupport.NewClient(h) 64 | if err == nil { 65 | clients = append(clients, client) 66 | log.Printf("Connected to %s", h) 67 | } else { 68 | log.Printf( 69 | "Error connecting to %s: %v", 70 | h, err) 71 | } 72 | } 73 | 74 | if len(clients) == 0 { 75 | return nil, errors.New( 76 | "no connections available") 77 | } 78 | out := make(chan pb.ReptilesClient) 79 | go func() { 80 | for { 81 | for _, client := range clients { 82 | out <- client 83 | } 84 | } 85 | }() 86 | return out, nil 87 | } 88 | -------------------------------------------------------------------------------- /crawler_distributed/persist/client/itemsaver.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "log" 6 | pb "island/crawler_distributed/proto" 7 | "time" 8 | 9 | "island/crawler_distributed/rpcsupport" 10 | ) 11 | 12 | func ItemSaver( 13 | host string) (chan pb.Item, error) { 14 | c, err := rpcsupport.NewClient(host) 15 | if err != nil { 16 | return nil, err 17 | } 18 | out := make(chan pb.Item) 19 | go func() { 20 | itemCount := 0 21 | for { 22 | item := <-out 23 | log.Printf("Item Saver: got item "+ 24 | "#%d: %s", itemCount, item.Car) 25 | itemCount++ 26 | 27 | // Call RPC to save item 28 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 29 | defer cancel() 30 | _, err := c.SaveItem(ctx, &pb.SaveItemRequest{Item: &item}) 31 | 32 | if err != nil { 33 | log.Printf("Item Saver: error "+ 34 | "saving item %v: %v", 35 | item, err) 36 | } 37 | } 38 | }() 39 | 40 | return out, nil 41 | } 42 | -------------------------------------------------------------------------------- /crawler_distributed/persist/server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | ADD item /item 3 | ENTRYPOINT [ "/item"] 4 | -------------------------------------------------------------------------------- /crawler_distributed/persist/server/Makefile: -------------------------------------------------------------------------------- 1 | 2 | build: 3 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o item ./itemsaver.go 4 | docker: 5 | docker build . -t item:latest 6 | -------------------------------------------------------------------------------- /crawler_distributed/persist/server/client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "time" 7 | 8 | "island/crawler/engine" 9 | "island/crawler/model" 10 | "island/crawler_distributed/config" 11 | "island/crawler_distributed/rpcsupport" 12 | ) 13 | 14 | func TestItemSaver(t *testing.T) { 15 | const host = ":1234" 16 | 17 | // start ItemSaverServer 18 | go serveRpc(host, "test1") 19 | time.Sleep(time.Second) 20 | 21 | // start ItemSaverClient 22 | client, err := rpcsupport.NewClient(host) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | // Call save 28 | item := engine.Item{ 29 | Url: "http://album.zhenai.com/u/108906739", 30 | Type: "zhenai", 31 | Id: "108906739", 32 | Payload: model.Profile{ 33 | Age: 34, 34 | Height: 162, 35 | Weight: 57, 36 | Income: "3001-5000元", 37 | Gender: "女", 38 | Name: "安静的雪", 39 | Xinzuo: "牡羊座", 40 | Occupation: "人事/行政", 41 | Marriage: "离异", 42 | House: "已购房", 43 | Hokou: "山东菏泽", 44 | Education: "大学本科", 45 | Car: "未购车", 46 | }, 47 | } 48 | 49 | result := "" 50 | err = client.Call(config.ItemSaverRpc, 51 | item, &result) 52 | 53 | if err != nil || result != "ok" { 54 | t.Errorf("result: %s; err: %s", 55 | result, err) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /crawler_distributed/persist/server/itemsaver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/olivere/elastic/v7" 7 | "island/crawler/config" 8 | c "island/crawler_distributed/config" 9 | "island/crawler_distributed/rpcsupport" 10 | "log" 11 | "net/http" 12 | ) 13 | 14 | var port = flag.Int("port", 0, 15 | "the port for me to listen on") 16 | 17 | func main() { 18 | flag.Parse() 19 | if *port == 0 { 20 | fmt.Println("must specify a port") 21 | return 22 | } 23 | go func() { 24 | log.Fatal(serveRpc( 25 | fmt.Sprintf(":%d", *port), 26 | config.ElasticIndex)) 27 | }() 28 | 29 | http.HandleFunc("/ping", func(res http.ResponseWriter, req *http.Request){ 30 | _, err := res.Write([]byte("pong")); 31 | if err != nil{ 32 | log.Fatal("write err--->",err) 33 | } 34 | }) 35 | if err := http.ListenAndServe(":8080", nil); err != nil{ 36 | log.Fatal("open http err--->",err) 37 | } 38 | } 39 | 40 | func serveRpc(host, index string) error { 41 | client, err := elastic.NewClient( 42 | elastic.SetURL(c.ElasticHost), 43 | elastic.SetSniff(false)) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | return rpcsupport.ServeRpc(host, 49 | &rpcsupport.RPCService{ 50 | Client: client, 51 | Index: index, 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /crawler_distributed/proto/crawler.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: crawler.proto 3 | 4 | package go_grpc_srv_reptiles 5 | 6 | import ( 7 | context "context" 8 | fmt "fmt" 9 | proto "github.com/golang/protobuf/proto" 10 | grpc "google.golang.org/grpc" 11 | codes "google.golang.org/grpc/codes" 12 | status "google.golang.org/grpc/status" 13 | math "math" 14 | ) 15 | 16 | // Reference imports to suppress errors if they are not otherwise used. 17 | var _ = proto.Marshal 18 | var _ = fmt.Errorf 19 | var _ = math.Inf 20 | 21 | // This is a compile-time assertion to ensure that this generated file 22 | // is compatible with the proto package it is being compiled against. 23 | // A compilation error at this line likely means your copy of the 24 | // proto package needs to be updated. 25 | const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package 26 | 27 | type SaveItemRequest struct { 28 | Item *Item `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` 29 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 30 | XXX_unrecognized []byte `json:"-"` 31 | XXX_sizecache int32 `json:"-"` 32 | } 33 | 34 | func (m *SaveItemRequest) Reset() { *m = SaveItemRequest{} } 35 | func (m *SaveItemRequest) String() string { return proto.CompactTextString(m) } 36 | func (*SaveItemRequest) ProtoMessage() {} 37 | func (*SaveItemRequest) Descriptor() ([]byte, []int) { 38 | return fileDescriptor_84c7eabcfe7807d1, []int{0} 39 | } 40 | 41 | func (m *SaveItemRequest) XXX_Unmarshal(b []byte) error { 42 | return xxx_messageInfo_SaveItemRequest.Unmarshal(m, b) 43 | } 44 | func (m *SaveItemRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 45 | return xxx_messageInfo_SaveItemRequest.Marshal(b, m, deterministic) 46 | } 47 | func (m *SaveItemRequest) XXX_Merge(src proto.Message) { 48 | xxx_messageInfo_SaveItemRequest.Merge(m, src) 49 | } 50 | func (m *SaveItemRequest) XXX_Size() int { 51 | return xxx_messageInfo_SaveItemRequest.Size(m) 52 | } 53 | func (m *SaveItemRequest) XXX_DiscardUnknown() { 54 | xxx_messageInfo_SaveItemRequest.DiscardUnknown(m) 55 | } 56 | 57 | var xxx_messageInfo_SaveItemRequest proto.InternalMessageInfo 58 | 59 | func (m *SaveItemRequest) GetItem() *Item { 60 | if m != nil { 61 | return m.Item 62 | } 63 | return nil 64 | } 65 | 66 | type SaveItemResult struct { 67 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 68 | XXX_unrecognized []byte `json:"-"` 69 | XXX_sizecache int32 `json:"-"` 70 | } 71 | 72 | func (m *SaveItemResult) Reset() { *m = SaveItemResult{} } 73 | func (m *SaveItemResult) String() string { return proto.CompactTextString(m) } 74 | func (*SaveItemResult) ProtoMessage() {} 75 | func (*SaveItemResult) Descriptor() ([]byte, []int) { 76 | return fileDescriptor_84c7eabcfe7807d1, []int{1} 77 | } 78 | 79 | func (m *SaveItemResult) XXX_Unmarshal(b []byte) error { 80 | return xxx_messageInfo_SaveItemResult.Unmarshal(m, b) 81 | } 82 | func (m *SaveItemResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 83 | return xxx_messageInfo_SaveItemResult.Marshal(b, m, deterministic) 84 | } 85 | func (m *SaveItemResult) XXX_Merge(src proto.Message) { 86 | xxx_messageInfo_SaveItemResult.Merge(m, src) 87 | } 88 | func (m *SaveItemResult) XXX_Size() int { 89 | return xxx_messageInfo_SaveItemResult.Size(m) 90 | } 91 | func (m *SaveItemResult) XXX_DiscardUnknown() { 92 | xxx_messageInfo_SaveItemResult.DiscardUnknown(m) 93 | } 94 | 95 | var xxx_messageInfo_SaveItemResult proto.InternalMessageInfo 96 | 97 | type ProcessRequest struct { 98 | Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` 99 | SerializedParser *SerializedParser `protobuf:"bytes,2,opt,name=serializedParser,proto3" json:"serializedParser,omitempty"` 100 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 101 | XXX_unrecognized []byte `json:"-"` 102 | XXX_sizecache int32 `json:"-"` 103 | } 104 | 105 | func (m *ProcessRequest) Reset() { *m = ProcessRequest{} } 106 | func (m *ProcessRequest) String() string { return proto.CompactTextString(m) } 107 | func (*ProcessRequest) ProtoMessage() {} 108 | func (*ProcessRequest) Descriptor() ([]byte, []int) { 109 | return fileDescriptor_84c7eabcfe7807d1, []int{2} 110 | } 111 | 112 | func (m *ProcessRequest) XXX_Unmarshal(b []byte) error { 113 | return xxx_messageInfo_ProcessRequest.Unmarshal(m, b) 114 | } 115 | func (m *ProcessRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 116 | return xxx_messageInfo_ProcessRequest.Marshal(b, m, deterministic) 117 | } 118 | func (m *ProcessRequest) XXX_Merge(src proto.Message) { 119 | xxx_messageInfo_ProcessRequest.Merge(m, src) 120 | } 121 | func (m *ProcessRequest) XXX_Size() int { 122 | return xxx_messageInfo_ProcessRequest.Size(m) 123 | } 124 | func (m *ProcessRequest) XXX_DiscardUnknown() { 125 | xxx_messageInfo_ProcessRequest.DiscardUnknown(m) 126 | } 127 | 128 | var xxx_messageInfo_ProcessRequest proto.InternalMessageInfo 129 | 130 | func (m *ProcessRequest) GetUrl() string { 131 | if m != nil { 132 | return m.Url 133 | } 134 | return "" 135 | } 136 | 137 | func (m *ProcessRequest) GetSerializedParser() *SerializedParser { 138 | if m != nil { 139 | return m.SerializedParser 140 | } 141 | return nil 142 | } 143 | 144 | type SerializedParser struct { 145 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` 146 | Args string `protobuf:"bytes,2,opt,name=args,proto3" json:"args,omitempty"` 147 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 148 | XXX_unrecognized []byte `json:"-"` 149 | XXX_sizecache int32 `json:"-"` 150 | } 151 | 152 | func (m *SerializedParser) Reset() { *m = SerializedParser{} } 153 | func (m *SerializedParser) String() string { return proto.CompactTextString(m) } 154 | func (*SerializedParser) ProtoMessage() {} 155 | func (*SerializedParser) Descriptor() ([]byte, []int) { 156 | return fileDescriptor_84c7eabcfe7807d1, []int{3} 157 | } 158 | 159 | func (m *SerializedParser) XXX_Unmarshal(b []byte) error { 160 | return xxx_messageInfo_SerializedParser.Unmarshal(m, b) 161 | } 162 | func (m *SerializedParser) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 163 | return xxx_messageInfo_SerializedParser.Marshal(b, m, deterministic) 164 | } 165 | func (m *SerializedParser) XXX_Merge(src proto.Message) { 166 | xxx_messageInfo_SerializedParser.Merge(m, src) 167 | } 168 | func (m *SerializedParser) XXX_Size() int { 169 | return xxx_messageInfo_SerializedParser.Size(m) 170 | } 171 | func (m *SerializedParser) XXX_DiscardUnknown() { 172 | xxx_messageInfo_SerializedParser.DiscardUnknown(m) 173 | } 174 | 175 | var xxx_messageInfo_SerializedParser proto.InternalMessageInfo 176 | 177 | func (m *SerializedParser) GetName() string { 178 | if m != nil { 179 | return m.Name 180 | } 181 | return "" 182 | } 183 | 184 | func (m *SerializedParser) GetArgs() string { 185 | if m != nil { 186 | return m.Args 187 | } 188 | return "" 189 | } 190 | 191 | type ProcessResult struct { 192 | Item []*Item `protobuf:"bytes,1,rep,name=item,proto3" json:"item,omitempty"` 193 | Request []*ProcessRequest `protobuf:"bytes,2,rep,name=request,proto3" json:"request,omitempty"` 194 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 195 | XXX_unrecognized []byte `json:"-"` 196 | XXX_sizecache int32 `json:"-"` 197 | } 198 | 199 | func (m *ProcessResult) Reset() { *m = ProcessResult{} } 200 | func (m *ProcessResult) String() string { return proto.CompactTextString(m) } 201 | func (*ProcessResult) ProtoMessage() {} 202 | func (*ProcessResult) Descriptor() ([]byte, []int) { 203 | return fileDescriptor_84c7eabcfe7807d1, []int{4} 204 | } 205 | 206 | func (m *ProcessResult) XXX_Unmarshal(b []byte) error { 207 | return xxx_messageInfo_ProcessResult.Unmarshal(m, b) 208 | } 209 | func (m *ProcessResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 210 | return xxx_messageInfo_ProcessResult.Marshal(b, m, deterministic) 211 | } 212 | func (m *ProcessResult) XXX_Merge(src proto.Message) { 213 | xxx_messageInfo_ProcessResult.Merge(m, src) 214 | } 215 | func (m *ProcessResult) XXX_Size() int { 216 | return xxx_messageInfo_ProcessResult.Size(m) 217 | } 218 | func (m *ProcessResult) XXX_DiscardUnknown() { 219 | xxx_messageInfo_ProcessResult.DiscardUnknown(m) 220 | } 221 | 222 | var xxx_messageInfo_ProcessResult proto.InternalMessageInfo 223 | 224 | func (m *ProcessResult) GetItem() []*Item { 225 | if m != nil { 226 | return m.Item 227 | } 228 | return nil 229 | } 230 | 231 | func (m *ProcessResult) GetRequest() []*ProcessRequest { 232 | if m != nil { 233 | return m.Request 234 | } 235 | return nil 236 | } 237 | 238 | type Item struct { 239 | Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` 240 | Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` 241 | Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` 242 | Payload *Profile `protobuf:"bytes,4,opt,name=Payload,proto3" json:"Payload,omitempty"` 243 | Car *Car `protobuf:"bytes,5,opt,name=car,proto3" json:"car,omitempty"` 244 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 245 | XXX_unrecognized []byte `json:"-"` 246 | XXX_sizecache int32 `json:"-"` 247 | } 248 | 249 | func (m *Item) Reset() { *m = Item{} } 250 | func (m *Item) String() string { return proto.CompactTextString(m) } 251 | func (*Item) ProtoMessage() {} 252 | func (*Item) Descriptor() ([]byte, []int) { 253 | return fileDescriptor_84c7eabcfe7807d1, []int{5} 254 | } 255 | 256 | func (m *Item) XXX_Unmarshal(b []byte) error { 257 | return xxx_messageInfo_Item.Unmarshal(m, b) 258 | } 259 | func (m *Item) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 260 | return xxx_messageInfo_Item.Marshal(b, m, deterministic) 261 | } 262 | func (m *Item) XXX_Merge(src proto.Message) { 263 | xxx_messageInfo_Item.Merge(m, src) 264 | } 265 | func (m *Item) XXX_Size() int { 266 | return xxx_messageInfo_Item.Size(m) 267 | } 268 | func (m *Item) XXX_DiscardUnknown() { 269 | xxx_messageInfo_Item.DiscardUnknown(m) 270 | } 271 | 272 | var xxx_messageInfo_Item proto.InternalMessageInfo 273 | 274 | func (m *Item) GetUrl() string { 275 | if m != nil { 276 | return m.Url 277 | } 278 | return "" 279 | } 280 | 281 | func (m *Item) GetType() string { 282 | if m != nil { 283 | return m.Type 284 | } 285 | return "" 286 | } 287 | 288 | func (m *Item) GetId() string { 289 | if m != nil { 290 | return m.Id 291 | } 292 | return "" 293 | } 294 | 295 | func (m *Item) GetPayload() *Profile { 296 | if m != nil { 297 | return m.Payload 298 | } 299 | return nil 300 | } 301 | 302 | func (m *Item) GetCar() *Car { 303 | if m != nil { 304 | return m.Car 305 | } 306 | return nil 307 | } 308 | 309 | type Profile struct { 310 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` 311 | Gender string `protobuf:"bytes,2,opt,name=gender,proto3" json:"gender,omitempty"` 312 | Age int32 `protobuf:"varint,3,opt,name=age,proto3" json:"age,omitempty"` 313 | Height int32 `protobuf:"varint,4,opt,name=height,proto3" json:"height,omitempty"` 314 | Weight int32 `protobuf:"varint,5,opt,name=weight,proto3" json:"weight,omitempty"` 315 | Income string `protobuf:"bytes,6,opt,name=income,proto3" json:"income,omitempty"` 316 | Marriage string `protobuf:"bytes,7,opt,name=marriage,proto3" json:"marriage,omitempty"` 317 | Education string `protobuf:"bytes,8,opt,name=education,proto3" json:"education,omitempty"` 318 | Occupation string `protobuf:"bytes,9,opt,name=occupation,proto3" json:"occupation,omitempty"` 319 | Hokou string `protobuf:"bytes,10,opt,name=hokou,proto3" json:"hokou,omitempty"` 320 | Xinzuo string `protobuf:"bytes,11,opt,name=xinzuo,proto3" json:"xinzuo,omitempty"` 321 | House string `protobuf:"bytes,12,opt,name=house,proto3" json:"house,omitempty"` 322 | Car string `protobuf:"bytes,13,opt,name=car,proto3" json:"car,omitempty"` 323 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 324 | XXX_unrecognized []byte `json:"-"` 325 | XXX_sizecache int32 `json:"-"` 326 | } 327 | 328 | func (m *Profile) Reset() { *m = Profile{} } 329 | func (m *Profile) String() string { return proto.CompactTextString(m) } 330 | func (*Profile) ProtoMessage() {} 331 | func (*Profile) Descriptor() ([]byte, []int) { 332 | return fileDescriptor_84c7eabcfe7807d1, []int{6} 333 | } 334 | 335 | func (m *Profile) XXX_Unmarshal(b []byte) error { 336 | return xxx_messageInfo_Profile.Unmarshal(m, b) 337 | } 338 | func (m *Profile) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 339 | return xxx_messageInfo_Profile.Marshal(b, m, deterministic) 340 | } 341 | func (m *Profile) XXX_Merge(src proto.Message) { 342 | xxx_messageInfo_Profile.Merge(m, src) 343 | } 344 | func (m *Profile) XXX_Size() int { 345 | return xxx_messageInfo_Profile.Size(m) 346 | } 347 | func (m *Profile) XXX_DiscardUnknown() { 348 | xxx_messageInfo_Profile.DiscardUnknown(m) 349 | } 350 | 351 | var xxx_messageInfo_Profile proto.InternalMessageInfo 352 | 353 | func (m *Profile) GetName() string { 354 | if m != nil { 355 | return m.Name 356 | } 357 | return "" 358 | } 359 | 360 | func (m *Profile) GetGender() string { 361 | if m != nil { 362 | return m.Gender 363 | } 364 | return "" 365 | } 366 | 367 | func (m *Profile) GetAge() int32 { 368 | if m != nil { 369 | return m.Age 370 | } 371 | return 0 372 | } 373 | 374 | func (m *Profile) GetHeight() int32 { 375 | if m != nil { 376 | return m.Height 377 | } 378 | return 0 379 | } 380 | 381 | func (m *Profile) GetWeight() int32 { 382 | if m != nil { 383 | return m.Weight 384 | } 385 | return 0 386 | } 387 | 388 | func (m *Profile) GetIncome() string { 389 | if m != nil { 390 | return m.Income 391 | } 392 | return "" 393 | } 394 | 395 | func (m *Profile) GetMarriage() string { 396 | if m != nil { 397 | return m.Marriage 398 | } 399 | return "" 400 | } 401 | 402 | func (m *Profile) GetEducation() string { 403 | if m != nil { 404 | return m.Education 405 | } 406 | return "" 407 | } 408 | 409 | func (m *Profile) GetOccupation() string { 410 | if m != nil { 411 | return m.Occupation 412 | } 413 | return "" 414 | } 415 | 416 | func (m *Profile) GetHokou() string { 417 | if m != nil { 418 | return m.Hokou 419 | } 420 | return "" 421 | } 422 | 423 | func (m *Profile) GetXinzuo() string { 424 | if m != nil { 425 | return m.Xinzuo 426 | } 427 | return "" 428 | } 429 | 430 | func (m *Profile) GetHouse() string { 431 | if m != nil { 432 | return m.House 433 | } 434 | return "" 435 | } 436 | 437 | func (m *Profile) GetCar() string { 438 | if m != nil { 439 | return m.Car 440 | } 441 | return "" 442 | } 443 | 444 | type Car struct { 445 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` 446 | Price float32 `protobuf:"fixed32,2,opt,name=price,proto3" json:"price,omitempty"` 447 | ImageURL string `protobuf:"bytes,3,opt,name=imageURL,proto3" json:"imageURL,omitempty"` 448 | Size string `protobuf:"bytes,4,opt,name=size,proto3" json:"size,omitempty"` 449 | Fuel float32 `protobuf:"fixed32,5,opt,name=fuel,proto3" json:"fuel,omitempty"` 450 | Transmission string `protobuf:"bytes,6,opt,name=transmission,proto3" json:"transmission,omitempty"` 451 | Engine string `protobuf:"bytes,7,opt,name=engine,proto3" json:"engine,omitempty"` 452 | Displacement float32 `protobuf:"fixed32,8,opt,name=displacement,proto3" json:"displacement,omitempty"` 453 | MaxSpeed float32 `protobuf:"fixed32,9,opt,name=maxSpeed,proto3" json:"maxSpeed,omitempty"` 454 | Acceleration float32 `protobuf:"fixed32,10,opt,name=acceleration,proto3" json:"acceleration,omitempty"` 455 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 456 | XXX_unrecognized []byte `json:"-"` 457 | XXX_sizecache int32 `json:"-"` 458 | } 459 | 460 | func (m *Car) Reset() { *m = Car{} } 461 | func (m *Car) String() string { return proto.CompactTextString(m) } 462 | func (*Car) ProtoMessage() {} 463 | func (*Car) Descriptor() ([]byte, []int) { 464 | return fileDescriptor_84c7eabcfe7807d1, []int{7} 465 | } 466 | 467 | func (m *Car) XXX_Unmarshal(b []byte) error { 468 | return xxx_messageInfo_Car.Unmarshal(m, b) 469 | } 470 | func (m *Car) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 471 | return xxx_messageInfo_Car.Marshal(b, m, deterministic) 472 | } 473 | func (m *Car) XXX_Merge(src proto.Message) { 474 | xxx_messageInfo_Car.Merge(m, src) 475 | } 476 | func (m *Car) XXX_Size() int { 477 | return xxx_messageInfo_Car.Size(m) 478 | } 479 | func (m *Car) XXX_DiscardUnknown() { 480 | xxx_messageInfo_Car.DiscardUnknown(m) 481 | } 482 | 483 | var xxx_messageInfo_Car proto.InternalMessageInfo 484 | 485 | func (m *Car) GetName() string { 486 | if m != nil { 487 | return m.Name 488 | } 489 | return "" 490 | } 491 | 492 | func (m *Car) GetPrice() float32 { 493 | if m != nil { 494 | return m.Price 495 | } 496 | return 0 497 | } 498 | 499 | func (m *Car) GetImageURL() string { 500 | if m != nil { 501 | return m.ImageURL 502 | } 503 | return "" 504 | } 505 | 506 | func (m *Car) GetSize() string { 507 | if m != nil { 508 | return m.Size 509 | } 510 | return "" 511 | } 512 | 513 | func (m *Car) GetFuel() float32 { 514 | if m != nil { 515 | return m.Fuel 516 | } 517 | return 0 518 | } 519 | 520 | func (m *Car) GetTransmission() string { 521 | if m != nil { 522 | return m.Transmission 523 | } 524 | return "" 525 | } 526 | 527 | func (m *Car) GetEngine() string { 528 | if m != nil { 529 | return m.Engine 530 | } 531 | return "" 532 | } 533 | 534 | func (m *Car) GetDisplacement() float32 { 535 | if m != nil { 536 | return m.Displacement 537 | } 538 | return 0 539 | } 540 | 541 | func (m *Car) GetMaxSpeed() float32 { 542 | if m != nil { 543 | return m.MaxSpeed 544 | } 545 | return 0 546 | } 547 | 548 | func (m *Car) GetAcceleration() float32 { 549 | if m != nil { 550 | return m.Acceleration 551 | } 552 | return 0 553 | } 554 | 555 | func init() { 556 | proto.RegisterType((*SaveItemRequest)(nil), "go.grpc.srv.reptiles.SaveItemRequest") 557 | proto.RegisterType((*SaveItemResult)(nil), "go.grpc.srv.reptiles.SaveItemResult") 558 | proto.RegisterType((*ProcessRequest)(nil), "go.grpc.srv.reptiles.ProcessRequest") 559 | proto.RegisterType((*SerializedParser)(nil), "go.grpc.srv.reptiles.SerializedParser") 560 | proto.RegisterType((*ProcessResult)(nil), "go.grpc.srv.reptiles.ProcessResult") 561 | proto.RegisterType((*Item)(nil), "go.grpc.srv.reptiles.Item") 562 | proto.RegisterType((*Profile)(nil), "go.grpc.srv.reptiles.Profile") 563 | proto.RegisterType((*Car)(nil), "go.grpc.srv.reptiles.Car") 564 | } 565 | 566 | func init() { proto.RegisterFile("crawler.proto", fileDescriptor_84c7eabcfe7807d1) } 567 | 568 | var fileDescriptor_84c7eabcfe7807d1 = []byte{ 569 | // 609 bytes of a gzipped FileDescriptorProto 570 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcf, 0x6e, 0xd3, 0x4e, 571 | 0x10, 0xfe, 0xc5, 0x89, 0x9b, 0x78, 0xfa, 0xe7, 0x17, 0xad, 0x2a, 0x64, 0x22, 0x40, 0xc8, 0x14, 572 | 0x84, 0x84, 0xe4, 0x43, 0x39, 0x20, 0x71, 0x40, 0x42, 0x3d, 0x21, 0x71, 0xa8, 0x36, 0x02, 0x89, 573 | 0xe3, 0xb2, 0x9e, 0x3a, 0x2b, 0xfc, 0x8f, 0x5d, 0x3b, 0x69, 0x73, 0xe1, 0x21, 0x78, 0x03, 0xde, 574 | 0x03, 0xf1, 0x6a, 0x68, 0x67, 0xed, 0x34, 0x29, 0xae, 0xda, 0xdb, 0xcc, 0xb7, 0xdf, 0x37, 0x3b, 575 | 0xf3, 0xad, 0x3d, 0x70, 0x28, 0xb5, 0x58, 0x65, 0xa8, 0xe3, 0x4a, 0x97, 0x75, 0xc9, 0x8e, 0xd3, 576 | 0x32, 0x4e, 0x75, 0x25, 0x63, 0xa3, 0x97, 0xb1, 0xc6, 0xaa, 0x56, 0x19, 0x9a, 0xe8, 0x3d, 0xfc, 577 | 0x3f, 0x17, 0x4b, 0xfc, 0x50, 0x63, 0xce, 0xf1, 0x7b, 0x83, 0xa6, 0x66, 0x31, 0x8c, 0x54, 0x8d, 578 | 0x79, 0x38, 0x78, 0x3a, 0x78, 0xb9, 0x7f, 0x3a, 0x8b, 0xfb, 0x74, 0x31, 0x09, 0x88, 0x17, 0x4d, 579 | 0xe1, 0xe8, 0xba, 0x84, 0x69, 0xb2, 0x3a, 0x5a, 0xc2, 0xd1, 0xb9, 0x2e, 0x25, 0x1a, 0xd3, 0xd5, 580 | 0x9c, 0xc2, 0xb0, 0xd1, 0x19, 0x95, 0x0c, 0xb8, 0x0d, 0x19, 0x87, 0xa9, 0x41, 0xad, 0x44, 0xa6, 581 | 0xd6, 0x98, 0x9c, 0x0b, 0x6d, 0x50, 0x87, 0x1e, 0xdd, 0xf8, 0xa2, 0xff, 0xc6, 0xf9, 0x0d, 0x36, 582 | 0xff, 0x47, 0x1f, 0xbd, 0x85, 0xe9, 0x4d, 0x16, 0x63, 0x30, 0x2a, 0x44, 0x8e, 0xed, 0xd5, 0x14, 583 | 0x5b, 0x4c, 0xe8, 0xd4, 0xd0, 0x7d, 0x01, 0xa7, 0x38, 0xfa, 0x01, 0x87, 0x9b, 0x9e, 0xed, 0x10, 584 | 0x5b, 0x36, 0x0c, 0xef, 0x63, 0x03, 0x7b, 0x07, 0x63, 0xed, 0xa6, 0x0d, 0x3d, 0x92, 0x9c, 0xf4, 585 | 0x4b, 0x76, 0x9d, 0xe1, 0x9d, 0x28, 0xfa, 0x35, 0x80, 0x91, 0x2d, 0xd7, 0xe3, 0x15, 0x83, 0x51, 586 | 0x7d, 0x55, 0x61, 0xd7, 0xaf, 0x8d, 0xd9, 0x11, 0x78, 0x2a, 0x09, 0x87, 0x84, 0x78, 0x2a, 0x61, 587 | 0x6f, 0x60, 0x7c, 0x2e, 0xae, 0xb2, 0x52, 0x24, 0xe1, 0x88, 0x6c, 0x7c, 0x7c, 0xeb, 0xf5, 0x17, 588 | 0x2a, 0x43, 0xde, 0xb1, 0xd9, 0x2b, 0x18, 0x4a, 0xa1, 0x43, 0x9f, 0x44, 0x0f, 0xfb, 0x45, 0x67, 589 | 0x42, 0x73, 0xcb, 0x8a, 0xfe, 0x78, 0x30, 0x6e, 0x2b, 0xf4, 0x3a, 0xfb, 0x00, 0xf6, 0x52, 0x2c, 590 | 0x92, 0xf6, 0x2d, 0x03, 0xde, 0x66, 0x76, 0x26, 0x91, 0x22, 0xb5, 0xeb, 0x73, 0x1b, 0x5a, 0xe6, 591 | 0x02, 0x55, 0xba, 0xa8, 0xa9, 0x5d, 0x9f, 0xb7, 0x99, 0xc5, 0x57, 0x0e, 0xf7, 0x1d, 0xbe, 0xda, 592 | 0xe0, 0xaa, 0x90, 0x65, 0x8e, 0xe1, 0x9e, 0xab, 0xec, 0x32, 0x36, 0x83, 0x49, 0x2e, 0xb4, 0x56, 593 | 0xb6, 0xfc, 0x98, 0x4e, 0x36, 0x39, 0x7b, 0x04, 0x01, 0x26, 0x8d, 0x14, 0xb5, 0x2a, 0x8b, 0x70, 594 | 0x42, 0x87, 0xd7, 0x00, 0x7b, 0x02, 0x50, 0x4a, 0xd9, 0x54, 0xee, 0x38, 0xa0, 0xe3, 0x2d, 0x84, 595 | 0x1d, 0x83, 0xbf, 0x28, 0xbf, 0x95, 0x4d, 0x08, 0x74, 0xe4, 0x12, 0xdb, 0xc7, 0xa5, 0x2a, 0xd6, 596 | 0x4d, 0x19, 0xee, 0xbb, 0x3e, 0x5c, 0xe6, 0xd8, 0x8d, 0xc1, 0xf0, 0xa0, 0x63, 0x37, 0x06, 0xed, 597 | 0xdc, 0xd6, 0xdc, 0x43, 0xf7, 0x96, 0xd6, 0xc1, 0x9f, 0x1e, 0x0c, 0xcf, 0x44, 0xff, 0x77, 0x79, 598 | 0x0c, 0x7e, 0xa5, 0x95, 0x74, 0x0f, 0xed, 0x71, 0x97, 0xd8, 0x09, 0x55, 0x2e, 0x52, 0xfc, 0xc4, 599 | 0x3f, 0xb6, 0xef, 0xbd, 0xc9, 0x6d, 0x15, 0xa3, 0xd6, 0x48, 0x1e, 0x06, 0x9c, 0x62, 0x8b, 0x5d, 600 | 0x34, 0x98, 0x91, 0x7f, 0x1e, 0xa7, 0x98, 0x45, 0x70, 0x50, 0x6b, 0x51, 0x98, 0x5c, 0x19, 0x63, 601 | 0xa7, 0x75, 0x1e, 0xee, 0x60, 0x76, 0x32, 0x2c, 0x52, 0x55, 0x74, 0x3e, 0xb6, 0x99, 0xd5, 0x26, 602 | 0xca, 0x54, 0x99, 0x90, 0x98, 0x63, 0x51, 0x93, 0x91, 0x1e, 0xdf, 0xc1, 0xdc, 0x2b, 0x5c, 0xce, 603 | 0x2b, 0xc4, 0x84, 0x9c, 0xf4, 0xf8, 0x26, 0xb7, 0x7a, 0x21, 0x25, 0x66, 0xa8, 0x9d, 0xd3, 0xe0, 604 | 0xf4, 0xdb, 0xd8, 0xe9, 0xef, 0x01, 0x4c, 0xba, 0xaf, 0x8d, 0x7d, 0x81, 0x49, 0xb7, 0x50, 0xd8, 605 | 0xf3, 0x5b, 0x96, 0xc1, 0xee, 0xce, 0x9a, 0x9d, 0xdc, 0x45, 0xa3, 0xbd, 0xf4, 0x1f, 0xfb, 0x4c, 606 | 0x9f, 0xaf, 0xfd, 0xff, 0xd8, 0xbd, 0x7e, 0xcf, 0xd9, 0xb3, 0x3b, 0x58, 0xae, 0xee, 0xd7, 0x3d, 607 | 0xda, 0xb1, 0xaf, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xe2, 0x71, 0xeb, 0x02, 0x74, 0x05, 0x00, 608 | 0x00, 609 | } 610 | 611 | // Reference imports to suppress errors if they are not otherwise used. 612 | var _ context.Context 613 | var _ grpc.ClientConn 614 | 615 | // This is a compile-time assertion to ensure that this generated file 616 | // is compatible with the grpc package it is being compiled against. 617 | const _ = grpc.SupportPackageIsVersion4 618 | 619 | // ReptilesClient is the client API for Reptiles service. 620 | // 621 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. 622 | type ReptilesClient interface { 623 | SaveItem(ctx context.Context, in *SaveItemRequest, opts ...grpc.CallOption) (*SaveItemResult, error) 624 | Process(ctx context.Context, in *ProcessRequest, opts ...grpc.CallOption) (*ProcessResult, error) 625 | } 626 | 627 | type reptilesClient struct { 628 | cc *grpc.ClientConn 629 | } 630 | 631 | func NewReptilesClient(cc *grpc.ClientConn) ReptilesClient { 632 | return &reptilesClient{cc} 633 | } 634 | 635 | func (c *reptilesClient) SaveItem(ctx context.Context, in *SaveItemRequest, opts ...grpc.CallOption) (*SaveItemResult, error) { 636 | out := new(SaveItemResult) 637 | err := c.cc.Invoke(ctx, "/go.grpc.srv.reptiles.reptiles/SaveItem", in, out, opts...) 638 | if err != nil { 639 | return nil, err 640 | } 641 | return out, nil 642 | } 643 | 644 | func (c *reptilesClient) Process(ctx context.Context, in *ProcessRequest, opts ...grpc.CallOption) (*ProcessResult, error) { 645 | out := new(ProcessResult) 646 | err := c.cc.Invoke(ctx, "/go.grpc.srv.reptiles.reptiles/Process", in, out, opts...) 647 | if err != nil { 648 | return nil, err 649 | } 650 | return out, nil 651 | } 652 | 653 | // ReptilesServer is the server API for Reptiles service. 654 | type ReptilesServer interface { 655 | SaveItem(context.Context, *SaveItemRequest) (*SaveItemResult, error) 656 | Process(context.Context, *ProcessRequest) (*ProcessResult, error) 657 | } 658 | 659 | // UnimplementedReptilesServer can be embedded to have forward compatible implementations. 660 | type UnimplementedReptilesServer struct { 661 | } 662 | 663 | func (*UnimplementedReptilesServer) SaveItem(ctx context.Context, req *SaveItemRequest) (*SaveItemResult, error) { 664 | return nil, status.Errorf(codes.Unimplemented, "method SaveItem not implemented") 665 | } 666 | func (*UnimplementedReptilesServer) Process(ctx context.Context, req *ProcessRequest) (*ProcessResult, error) { 667 | return nil, status.Errorf(codes.Unimplemented, "method Process not implemented") 668 | } 669 | 670 | func RegisterReptilesServer(s *grpc.Server, srv ReptilesServer) { 671 | s.RegisterService(&_Reptiles_serviceDesc, srv) 672 | } 673 | 674 | func _Reptiles_SaveItem_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 675 | in := new(SaveItemRequest) 676 | if err := dec(in); err != nil { 677 | return nil, err 678 | } 679 | if interceptor == nil { 680 | return srv.(ReptilesServer).SaveItem(ctx, in) 681 | } 682 | info := &grpc.UnaryServerInfo{ 683 | Server: srv, 684 | FullMethod: "/go.grpc.srv.reptiles.reptiles/SaveItem", 685 | } 686 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 687 | return srv.(ReptilesServer).SaveItem(ctx, req.(*SaveItemRequest)) 688 | } 689 | return interceptor(ctx, in, info, handler) 690 | } 691 | 692 | func _Reptiles_Process_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 693 | in := new(ProcessRequest) 694 | if err := dec(in); err != nil { 695 | return nil, err 696 | } 697 | if interceptor == nil { 698 | return srv.(ReptilesServer).Process(ctx, in) 699 | } 700 | info := &grpc.UnaryServerInfo{ 701 | Server: srv, 702 | FullMethod: "/go.grpc.srv.reptiles.reptiles/Process", 703 | } 704 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 705 | return srv.(ReptilesServer).Process(ctx, req.(*ProcessRequest)) 706 | } 707 | return interceptor(ctx, in, info, handler) 708 | } 709 | 710 | var _Reptiles_serviceDesc = grpc.ServiceDesc{ 711 | ServiceName: "go.grpc.srv.reptiles.reptiles", 712 | HandlerType: (*ReptilesServer)(nil), 713 | Methods: []grpc.MethodDesc{ 714 | { 715 | MethodName: "SaveItem", 716 | Handler: _Reptiles_SaveItem_Handler, 717 | }, 718 | { 719 | MethodName: "Process", 720 | Handler: _Reptiles_Process_Handler, 721 | }, 722 | }, 723 | Streams: []grpc.StreamDesc{}, 724 | Metadata: "crawler.proto", 725 | } 726 | -------------------------------------------------------------------------------- /crawler_distributed/proto/crawler.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package go.grpc.srv.reptiles; 4 | 5 | service reptiles { 6 | rpc SaveItem(SaveItemRequest) returns (SaveItemResult) {} 7 | rpc Process(ProcessRequest) returns (ProcessResult) {} 8 | } 9 | 10 | message SaveItemRequest{ 11 | Item item = 1; 12 | } 13 | message SaveItemResult{} 14 | 15 | 16 | message ProcessRequest{ 17 | string url = 1; 18 | SerializedParser serializedParser = 2; 19 | } 20 | 21 | message SerializedParser { 22 | string name = 1; 23 | string args = 2; 24 | } 25 | 26 | message ProcessResult{ 27 | repeated Item item = 1; 28 | repeated ProcessRequest request = 2; 29 | } 30 | 31 | message Item { 32 | string url = 1; 33 | string type = 2; 34 | string id = 3; 35 | Profile Payload = 4; 36 | Car car = 5; 37 | } 38 | 39 | message Profile { 40 | string name = 1; 41 | string gender = 2; 42 | int32 age = 3; 43 | int32 height = 4; 44 | int32 weight = 5; 45 | string income = 6; 46 | string marriage = 7; 47 | string education = 8; 48 | string occupation = 9; 49 | string hokou = 10; 50 | string xinzuo = 11; 51 | string house = 12; 52 | string car = 13; 53 | } 54 | 55 | message Car{ 56 | string name = 1; 57 | float price = 2; 58 | string imageURL = 3; 59 | string size = 4; 60 | float fuel = 5; 61 | string transmission = 6; 62 | string engine = 7; 63 | float displacement = 8; 64 | float maxSpeed = 9; 65 | float acceleration = 10; 66 | } 67 | -------------------------------------------------------------------------------- /crawler_distributed/rpcsupport/handle.go: -------------------------------------------------------------------------------- 1 | package rpcsupport 2 | 3 | import ( 4 | "context" 5 | "github.com/olivere/elastic/v7" 6 | "island/crawler/engine" 7 | "island/crawler/persist" 8 | pb "island/crawler_distributed/proto" 9 | t "island/crawler_distributed/worker" 10 | "log" 11 | ) 12 | 13 | 14 | type RPCService struct { 15 | Client *elastic.Client 16 | Index string 17 | } 18 | 19 | 20 | func (s *RPCService) Process( 21 | ctx context.Context, req *pb.ProcessRequest)(*pb.ProcessResult,error){ 22 | engineReq, err := t.DeserializeRequest(req) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | engineResult, err := engine.Worker(engineReq) 28 | if err != nil { 29 | return nil, err 30 | } 31 | var result = t.SerializeResult(engineResult) 32 | 33 | return &result, nil 34 | } 35 | 36 | // SaveItem 把数据送到es中 37 | func (s *RPCService) SaveItem( 38 | ctx context.Context, item *pb.SaveItemRequest) (*pb.SaveItemResult, error) { 39 | err := persist.Save(s.Client, s.Index, item.Item) 40 | log.Printf("Item %v saved.", item.Item) 41 | if err != nil { 42 | log.Printf("Error saving item %v: %v", 43 | item, err) 44 | } 45 | return &pb.SaveItemResult{}, err 46 | } 47 | -------------------------------------------------------------------------------- /crawler_distributed/rpcsupport/rpc.go: -------------------------------------------------------------------------------- 1 | package rpcsupport 2 | 3 | import ( 4 | "google.golang.org/grpc" 5 | pb "island/crawler_distributed/proto" 6 | "log" 7 | "net" 8 | ) 9 | 10 | func ServeRpc( 11 | host string, service pb.ReptilesServer) error { 12 | 13 | s := grpc.NewServer() 14 | pb.RegisterReptilesServer(s, service) 15 | 16 | listener, err := net.Listen("tcp", host) 17 | if err != nil { 18 | log.Fatalf("failed to listen: %v", err) 19 | } 20 | log.Printf("Listening on %s", host) 21 | 22 | if err := s.Serve(listener);err != nil{ 23 | log.Fatalf("failed to listen: %v", err) 24 | } 25 | return nil 26 | } 27 | 28 | func NewClient(host string) (pb.ReptilesClient, error) { 29 | conn, err := grpc.Dial(host, grpc.WithInsecure()) 30 | if err != nil { 31 | return nil, err 32 | } 33 | client := pb.NewReptilesClient(conn) 34 | return client, nil 35 | } 36 | -------------------------------------------------------------------------------- /crawler_distributed/worker/client/worker.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | pb "island/crawler_distributed/proto" 6 | "time" 7 | 8 | "island/crawler/engine" 9 | "island/crawler_distributed/worker" 10 | ) 11 | 12 | func CreateProcessor( 13 | clientChan chan pb.ReptilesClient) engine.Processor { 14 | return func( 15 | req engine.Request) ( 16 | engine.ParseResult, error) { 17 | 18 | sReq := worker.SerializeRequest(req) 19 | c := <-clientChan 20 | // Call RPC to send work 21 | ctx, cancel := context.WithTimeout(context.Background(), 1 * time.Second) 22 | defer cancel() 23 | sResult, err := c.Process(ctx, &pb.ProcessRequest{ 24 | Url: sReq.Url, SerializedParser: sReq.SerializedParser}) 25 | if err != nil { 26 | return engine.ParseResult{}, err 27 | } 28 | return worker.DeserializeResult(*sResult), 29 | nil 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /crawler_distributed/worker/server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | ADD work /work 3 | ENTRYPOINT [ "/work"] 4 | -------------------------------------------------------------------------------- /crawler_distributed/worker/server/Makefile: -------------------------------------------------------------------------------- 1 | 2 | build: 3 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -o work ./worker.go 4 | docker: 5 | docker build . -t work:latest 6 | -------------------------------------------------------------------------------- /crawler_distributed/worker/server/client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "island/crawler/config" 9 | rpcnames "island/crawler_distributed/config" 10 | "island/crawler_distributed/rpcsupport" 11 | "island/crawler_distributed/worker" 12 | ) 13 | 14 | func TestCrawlService(t *testing.T) { 15 | const host = ":9000" 16 | go rpcsupport.ServeRpc( 17 | host, worker.CrawlService{}) 18 | time.Sleep(time.Second) 19 | 20 | client, err := rpcsupport.NewClient(host) 21 | if err != nil { 22 | panic(err) 23 | } 24 | 25 | // TODO: Use a fake fetcher to handle the url. 26 | // So we don't get data from zhenai.com 27 | req := worker.Request{ 28 | Url: "http://album.zhenai.com/u/108906739", 29 | Parser: worker.SerializedParser{ 30 | Name: config.ParseProfile, 31 | Args: "安静的雪", 32 | }, 33 | } 34 | var result worker.ParseResult 35 | err = client.Call( 36 | rpcnames.CrawlServiceRpc, req, &result) 37 | 38 | if err != nil { 39 | t.Error(err) 40 | } else { 41 | fmt.Println(result) 42 | } 43 | 44 | // TODO: Verify results 45 | } 46 | -------------------------------------------------------------------------------- /crawler_distributed/worker/server/worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "island/crawler_distributed/rpcsupport" 8 | 9 | "flag" 10 | 11 | "island/crawler/fetcher" 12 | ) 13 | 14 | var ( 15 | port = flag.Int("port", 0, 16 | "the port for me to listen on") 17 | httpPort = flag.Int("httpPort", 0, 18 | "the port for me to listen on") 19 | ) 20 | 21 | func main() { 22 | flag.Parse() 23 | fetcher.SetVerboseLogging() 24 | if *port == 0 { 25 | fmt.Println("must specify a port") 26 | return 27 | } 28 | go func() { 29 | log.Fatal(rpcsupport.ServeRpc( 30 | fmt.Sprintf(":%d", *port), 31 | &rpcsupport.RPCService{})) 32 | }() 33 | 34 | http.HandleFunc("/ping", func(res http.ResponseWriter, req *http.Request){ 35 | _, err := res.Write([]byte("pong")); 36 | if err != nil{ 37 | log.Fatal("write err--->",err) 38 | } 39 | }) 40 | if err := http.ListenAndServe(fmt.Sprintf(":%d", *httpPort), nil); err != nil{ 41 | log.Fatal("open http err--->",err) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /crawler_distributed/worker/types.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "errors" 5 | 6 | "log" 7 | 8 | "island/crawler/config" 9 | "island/crawler/engine" 10 | xcar "island/crawler/xcar/parser" 11 | zhenai "island/crawler/zhenai/parser" 12 | pb "island/crawler_distributed/proto" 13 | ) 14 | 15 | 16 | func SerializeRequest(r engine.Request) *pb.ProcessRequest { 17 | name, args := r.Parser.Serialize() 18 | return &pb.ProcessRequest{ 19 | Url: r.Url, 20 | SerializedParser: &pb.SerializedParser{ 21 | Name: name, 22 | Args: args, 23 | }, 24 | } 25 | } 26 | 27 | func SerializeResult( 28 | r engine.ParseResult) pb.ProcessResult { 29 | result := pb.ProcessResult{ 30 | Item: r.Items, 31 | } 32 | 33 | for _, req := range r.Requests { 34 | result.Request = append(result.Request, 35 | SerializeRequest(req)) 36 | } 37 | return result 38 | } 39 | 40 | func DeserializeRequest( 41 | r *pb.ProcessRequest) (engine.Request, error) { 42 | parser, err := deserializeParser(r.SerializedParser) 43 | if err != nil { 44 | return engine.Request{}, err 45 | } 46 | return engine.Request{ 47 | Url: r.Url, 48 | Parser: parser, 49 | }, nil 50 | } 51 | 52 | func DeserializeResult( 53 | r pb.ProcessResult) engine.ParseResult { 54 | result := engine.ParseResult{ 55 | Items: r.Item, 56 | } 57 | 58 | for _, req := range r.Request { 59 | engineReq, err := DeserializeRequest(req) 60 | if err != nil { 61 | log.Printf("error deserializing "+ 62 | "request: %v", err) 63 | continue 64 | } 65 | result.Requests = append(result.Requests, 66 | engineReq) 67 | } 68 | return result 69 | } 70 | 71 | func deserializeParser( 72 | p *pb.SerializedParser) (engine.Parser, error) { 73 | switch p.Name { 74 | case config.ParseCityList: 75 | return engine.NewFuncParser( 76 | zhenai.ParseCityList, 77 | config.ParseCityList), nil 78 | case config.ParseCity: 79 | return engine.NewFuncParser( 80 | zhenai.ParseCity, 81 | config.ParseCity), nil 82 | 83 | case config.ParseProfile: 84 | return zhenai.NewProfileParser( 85 | p.Args), nil 86 | case config.ParseCarDetail: 87 | return engine.NewFuncParser( 88 | xcar.ParseCarDetail, 89 | config.ParseCarDetail), nil 90 | case config.ParseCarModel: 91 | return engine.NewFuncParser( 92 | xcar.ParseCarModel, 93 | config.ParseCarModel), nil 94 | case config.ParseCarList: 95 | return engine.NewFuncParser( 96 | xcar.ParseCarList, 97 | config.ParseCarList), nil 98 | case config.NilParser: 99 | return engine.NilParser{}, nil 100 | default: 101 | return nil, errors.New( 102 | "unknown parser name") 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | consul: 6 | image: consul:latest 7 | container_name: c4 8 | ports: 9 | - "8500:8500" 10 | volumes: 11 | - ./crawler_distributed/data:/etc/consul.d 12 | command: "agent -server -bootstrap-expect 1 -data-dir /tmp/consul -node=s1 -bind=0.0.0.0 -rejoin -config-dir=/etc/consul.d/ -client 0.0.0.0" 13 | logging: 14 | options: 15 | max-size: 50m 16 | 17 | elastic-srv: 18 | image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2 19 | ports: 20 | - "9200:9200" 21 | - "9300:9300" 22 | environment: 23 | - "discovery.type=single-node" 24 | restart: always 25 | logging: 26 | options: 27 | max-size: 50m 28 | 29 | item-srv: 30 | image: item:latest 31 | command: "--port=1234" 32 | ports: 33 | - "1234:1234" 34 | - "8080:8080" 35 | restart: always 36 | logging: 37 | options: 38 | max-size: 50m 39 | 40 | work-srv: 41 | image: work:latest 42 | command: "--port=9001 --httpPort=8081" 43 | ports: 44 | - "9001:9001" 45 | - "8081:8081" 46 | restart: always 47 | logging: 48 | options: 49 | max-size: 50m 50 | 51 | work-two-srv: 52 | image: work:latest 53 | command: "--port=9002 --httpPort=8082" 54 | ports: 55 | - "9002:9002" 56 | - "8082:8082" 57 | restart: always 58 | logging: 59 | options: 60 | max-size: 50m 61 | 62 | work-three-srv: 63 | image: work:latest 64 | command: "--port=9003 --httpPort=8083" 65 | ports: 66 | - "9003:9003" 67 | - "8083:8083" 68 | restart: always 69 | logging: 70 | options: 71 | max-size: 50m 72 | 73 | client-srv: 74 | image: client:latest 75 | restart: always 76 | depends_on: 77 | - redis 78 | logging: 79 | options: 80 | max-size: 50m 81 | 82 | frontend-srv: 83 | image: frontend:latest 84 | volumes: 85 | - ./crawler/frontend:/frontend 86 | ports: 87 | - "8888:8888" 88 | restart: always 89 | logging: 90 | options: 91 | max-size: 50m 92 | 93 | redis: 94 | image: redis:alpine 95 | ports: 96 | - "6379:6379" 97 | volumes: 98 | - ./crawler_distributed/data/redis:/data 99 | command: "redis-server" 100 | logging: 101 | options: 102 | max-size: 50m 103 | 104 | 105 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module island 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/gin-contrib/sse v0.1.0 // indirect 7 | github.com/gin-gonic/gin v1.4.0 8 | github.com/golang/protobuf v1.3.2 9 | github.com/gomodule/redigo v2.0.0+incompatible 10 | github.com/hashicorp/consul/api v1.3.0 11 | github.com/json-iterator/go v1.1.8 // indirect 12 | github.com/mattn/go-isatty v0.0.10 // indirect 13 | github.com/olivere/elastic/v7 v7.0.8 14 | github.com/prometheus/common v0.2.0 15 | github.com/ugorji/go v1.1.7 // indirect 16 | go.uber.org/zap v1.12.0 17 | golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect 18 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 19 | golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 // indirect 20 | golang.org/x/text v0.3.0 21 | golang.org/x/tools v0.0.0-20200128002243-345141a36859 // indirect 22 | google.golang.org/grpc v1.23.0 23 | gopkg.in/yaml.v2 v2.2.4 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 4 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 5 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 7 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 8 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 9 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 10 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 11 | github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 12 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 13 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= 14 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 15 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 16 | github.com/aws/aws-sdk-go v1.19.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 17 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 18 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 19 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 20 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 21 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 23 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 25 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 26 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 27 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 28 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 29 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 30 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 31 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 32 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 33 | github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g= 34 | github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 35 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 36 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 37 | github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ= 38 | github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= 39 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 40 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 41 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 42 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 43 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 44 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 45 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 46 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 47 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 48 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 49 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 50 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 51 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 52 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 53 | github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= 54 | github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 55 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 56 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 57 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 58 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 59 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 60 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 61 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 62 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 63 | github.com/hashicorp/consul v1.6.2 h1:OjnFfc1vHPLtwrYAMC9HlJ4WTSmgBTq2erWuBERm3hY= 64 | github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= 65 | github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= 66 | github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= 67 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 68 | github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= 69 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 70 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 71 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 72 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 73 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 74 | github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= 75 | github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= 76 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 77 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 78 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 79 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 80 | github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= 81 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 82 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 83 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 84 | github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= 85 | github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 86 | github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= 87 | github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= 88 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 89 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 90 | github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= 91 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 92 | github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= 93 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 94 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 95 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 96 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 97 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 98 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 99 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 100 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 101 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 102 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 103 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 104 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= 105 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 106 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 107 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 108 | github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= 109 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 110 | github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= 111 | github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= 112 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 113 | github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 114 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 115 | github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= 116 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 117 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 118 | github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= 119 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 120 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 121 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 122 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 123 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 124 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 125 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 126 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 127 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 128 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 129 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 130 | github.com/olivere/elastic v6.2.23+incompatible h1:oRGUA/8fKcnkDcqLuwGb5YCzgbgEBo+Y9gamsWqZ0qU= 131 | github.com/olivere/elastic v6.2.23+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= 132 | github.com/olivere/elastic/v7 v7.0.8 h1:tp9BHGFilpoH7O7fQOwWiXJQFJkl9PZJvUwO74OvfKc= 133 | github.com/olivere/elastic/v7 v7.0.8/go.mod h1:UcXCjbh5xfX9uMB1VCcIYgGJBItbd4uRBdYRsBnnXHo= 134 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 135 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 136 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 137 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 138 | github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= 139 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 140 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 141 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 142 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 143 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 144 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 145 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 146 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 147 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 148 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 149 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 150 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 151 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 152 | github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= 153 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 154 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 155 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 156 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 157 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 158 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 159 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 160 | github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= 161 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 162 | github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= 163 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 164 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 165 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 166 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 167 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 168 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 169 | github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= 170 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 171 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 172 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 173 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 174 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 175 | go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 176 | go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= 177 | go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 178 | go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= 179 | go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= 180 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= 181 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 182 | go.uber.org/zap v1.12.0 h1:dySoUQPFBGj6xwjmBzageVL8jGi8uxc6bEmJQjA06bw= 183 | go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= 184 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 185 | golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 186 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 187 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 188 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 189 | golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= 190 | golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 191 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 192 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 193 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 194 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 195 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 196 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= 197 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 198 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 199 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 200 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 201 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 202 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 203 | golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 204 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 205 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 206 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 207 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 208 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 209 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 210 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 211 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 212 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 213 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 214 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 215 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 216 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 217 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 218 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 219 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 220 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 221 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 222 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 223 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 224 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 225 | golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 226 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 227 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 228 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 229 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= 230 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 231 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 232 | golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 233 | golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c h1:S/FtSvpNLtFBgjTqcKsRpsa6aVsI6iztaz1bQd9BJwE= 234 | golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 235 | golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= 236 | golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 237 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 238 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 239 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 240 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 241 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 242 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 243 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 244 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 245 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 246 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 247 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= 248 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 249 | golang.org/x/tools v0.0.0-20191101200257-8dbcdeb83d3f h1:+QO45yvqhfD79HVNFPAgvstYLFye8zA+rd0mHFsGV9s= 250 | golang.org/x/tools v0.0.0-20191101200257-8dbcdeb83d3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 251 | golang.org/x/tools v0.0.0-20200128002243-345141a36859 h1:xIszjAtlVeHg9hhv6Zhntvwqowji1k2rrgoOhj/aaKw= 252 | golang.org/x/tools v0.0.0-20200128002243-345141a36859/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 253 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 254 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 255 | google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= 256 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 257 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 258 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 259 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= 260 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 261 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= 262 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 263 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 264 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 265 | google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= 266 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 267 | google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= 268 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 269 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 270 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 271 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 272 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 273 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 274 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 275 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 276 | gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= 277 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 278 | gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= 279 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 280 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 281 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 282 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 283 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 284 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 285 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 286 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 287 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 288 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 289 | honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= 290 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 291 | --------------------------------------------------------------------------------