├── .github └── FUNDING.yml ├── Client.go ├── LICENSE ├── README.md ├── build-archs.sh ├── cmd ├── leakix-ns │ └── README.md └── leakix │ └── main.go ├── example └── main.go ├── go.mod └── go.sum /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: gboddin 2 | -------------------------------------------------------------------------------- /Client.go: -------------------------------------------------------------------------------- 1 | package LeakIXClient 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/LeakIX/l9format" 8 | "github.com/gorilla/websocket" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | url2 "net/url" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | var LeakIXProxy = &l9format.ServicePluginBase{} 18 | var LeakIXHttpTranport = &http.Transport{ 19 | DialContext: LeakIXProxy.DialContext, 20 | ResponseHeaderTimeout: 5 * time.Second, 21 | ExpectContinueTimeout: 5 * time.Second, 22 | } 23 | var HttpClient = &http.Client{ 24 | Transport: LeakIXHttpTranport, 25 | Timeout: 5 * time.Second, 26 | } 27 | 28 | type SearchResultsClient struct { 29 | Scope string 30 | Query string 31 | SearchResults []l9format.L9Event 32 | Position int 33 | Page int 34 | ApiKey string 35 | Endpoint string 36 | LastError error 37 | } 38 | 39 | const defaultEndpoint = "https://leakix.net" 40 | 41 | func (sc *SearchResultsClient) GetEndpoint() string { 42 | if len(sc.Endpoint) > 8 { 43 | return sc.Endpoint 44 | } 45 | return defaultEndpoint 46 | } 47 | 48 | func (sc *SearchResultsClient) Next() bool { 49 | var results []l9format.L9Event 50 | if len(sc.SearchResults) > sc.Position { 51 | sc.Position++ 52 | return true 53 | } 54 | // Try to load next page 55 | results, sc.LastError = sc.GetSearchResults(sc.Scope, sc.Query, sc.Page) 56 | for _, result := range results { 57 | sc.SearchResults = append(sc.SearchResults, result) 58 | } 59 | sc.Page++ 60 | if len(sc.SearchResults) > sc.Position { 61 | sc.Position++ 62 | return true 63 | } 64 | return false 65 | } 66 | 67 | func (sc *SearchResultsClient) SearchResult() l9format.L9Event { 68 | return sc.SearchResults[sc.Position-1] 69 | } 70 | 71 | func (sc *SearchResultsClient) GetSearchResults(scope string, query string, page int) ([]l9format.L9Event, error) { 72 | url := fmt.Sprintf( 73 | "%s/search?scope=%s&q=%s&page=%d", sc.GetEndpoint(), url2.QueryEscape(scope), url2.QueryEscape(query), page) 74 | var searchResults []l9format.L9Event 75 | req, _ := http.NewRequest("GET", url, nil) 76 | req.Header.Set("Accept", "application/json") 77 | req.Header.Set("api-key", sc.ApiKey) 78 | resp, err := HttpClient.Do(req) 79 | if err != nil { 80 | return searchResults, err 81 | } 82 | defer resp.Body.Close() 83 | body, err := ioutil.ReadAll(resp.Body) 84 | if err != nil { 85 | return searchResults, err 86 | } 87 | if resp.StatusCode == 429 { 88 | // parse wait header and try again 89 | duration, err := time.ParseDuration(resp.Header.Get("x-limited-for")) 90 | if err != nil { 91 | panic(err) 92 | } 93 | time.Sleep(duration) 94 | return sc.GetSearchResults(scope, query, page) 95 | } 96 | err = json.Unmarshal(body, &searchResults) 97 | if err != nil { 98 | return searchResults, err 99 | } 100 | return searchResults, nil 101 | } 102 | 103 | func (sc *SearchResultsClient) GetChannel(scope string) (chan l9format.L9Event, error) { 104 | channel := make(chan l9format.L9Event) 105 | endpointUrl, err := url2.Parse(sc.GetEndpoint()) 106 | if err != nil { 107 | return nil, errors.New("invalid endpoint") 108 | } 109 | endpointUrl.Scheme = strings.Replace(endpointUrl.Scheme, "http", "ws", -1) 110 | log.Println(endpointUrl.String()) 111 | wsConnection, _, err := websocket.DefaultDialer.Dial(endpointUrl.String()+"/ws/"+scope, map[string][]string{ 112 | "Origin": {endpointUrl.Host + ":" + endpointUrl.Port()}, 113 | "api-key": {sc.ApiKey}, 114 | }) 115 | if err != nil { 116 | return nil, err 117 | } 118 | go func() { 119 | searchResult := l9format.L9Event{} 120 | for { 121 | err := wsConnection.ReadJSON(&searchResult) 122 | if err != nil { 123 | log.Println("Error parsing websocket results. Is your scope correct?") 124 | log.Fatal(err) 125 | } 126 | channel <- searchResult 127 | } 128 | }() 129 | return channel, nil 130 | } 131 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, LeakIX 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LeakIXClient 2 | 3 | This is a Go CLI & library making queries to LeakIX easier. 4 | 5 | ## leakix - Command line usage 6 | 7 | ```sh 8 | $ leakix -h 9 | Usage of leakix: 10 | 11 | -j JSON mode, (excludes -t) 12 | -l int 13 | Limit results output (default 100) 14 | -q string 15 | Search mode, specify search query (default "*") 16 | -r Realtime mode, (excludes -q) 17 | -s string 18 | Specify scope (default "leak") 19 | -t string 20 | Specify output template (default "{{ .Ip }}:{{ .Port }}") 21 | 22 | $ # Example query on the index 23 | $ leakix -l 2 -q "protocol:web AND plugin:GitConfigPlugin" -t "{{ .Ip }}:{{ .Port }} : {{ .Data }}" 24 | 178.62.217.44:80 : Found git deployment configuration 25 | [core] 26 | repositoryformatversion = 0 27 | filemode = false 28 | bare = false 29 | logallrefupdates = true 30 | [remote "origin"] 31 | url = https://gitlab.com/lyranalytics/lyra-website.git 32 | fetch = +refs/heads/*:refs/remotes/origin/* 33 | [branch "abdulrahman"] 34 | remote = origin 35 | merge = refs/heads/abdulrahman 36 | 37 | 2604:a880:800:a1::10f:1001:80 : Found git deployment configuration 38 | [core] 39 | repositoryformatversion = 0 40 | filemode = true 41 | bare = false 42 | logallrefupdates = true 43 | [remote "origin"] 44 | url = https://github.com/mautic/mautic.git 45 | fetch = +refs/heads/*:refs/remotes/origin/* 46 | [branch "staging"] 47 | remote = origin 48 | merge = refs/heads/staging 49 | 50 | $ # Stream results in realtime from the engine, no filtering 51 | $ ./leakix -r -s services -l 0 52 | 14.167.7.149:81 53 | 54.249.38.136:9200 54 | 23.65.39.190:80 55 | [2a01:4f8:10a:1b5a::2]:80 56 | 23.225.38.43:3306 57 | 210.16.68.51:80 58 | ...keeps streaming... 59 | ``` 60 | 61 | ## Library usage 62 | 63 | ```golang 64 | package main 65 | import ( 66 | "fmt" 67 | "github.com/LeakIX/LeakIXClient" 68 | ) 69 | 70 | func DoSearch(){ 71 | // Create a searcher 72 | LeakIXSearch := LeakIXClient.SearchResultsClient{ 73 | Scope: "leak", 74 | Query: "protocol:kafka AND \"telecom_italia_data\"", 75 | } 76 | // Iterate, the lib will query further pages if needed 77 | for LeakIXSearch.Next() { 78 | // Use the result 79 | leak := LeakIXSearch.SearchResult() 80 | fmt.Printf("%s:%s - Country:%s\n", leak.Ip, leak.Port, leak.GeoLocation.CountryName) 81 | } 82 | } 83 | 84 | 85 | func LiveStream() { 86 | // Get a channel from the websocket 87 | serviceChannel, err := LeakIXClient.GetChannel("services") 88 | if err != nil { 89 | log.Println("Websocket connection error:") 90 | log.Fatal(err) 91 | } 92 | for { 93 | // Print everything received on the channel 94 | service := <- serviceChannel 95 | log.Println(service.Ip) 96 | } 97 | } 98 | ``` 99 | -------------------------------------------------------------------------------- /build-archs.sh: -------------------------------------------------------------------------------- 1 | 2 | CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-32 ./cmd/leakix & 3 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-64 ./cmd/leakix & 4 | CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-freebsd-64 ./cmd/leakix & 5 | CGO_ENABLED=0 GOOS=freebsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-freebsd-32 ./cmd/leakix & 6 | 7 | CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-arm7 ./cmd/leakix & 8 | CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-linux-arm6 ./cmd/leakix & 9 | 10 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-osx ./cmd/leakix & 11 | 12 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-win64.exe ./cmd/leakix & 13 | CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-win32.exe ./cmd/leakix & 14 | 15 | CGO_ENABLED=0 GOOS=netbsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-netbsd-64 ./cmd/leakix & 16 | CGO_ENABLED=0 GOOS=netbsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-netbsd-32 ./cmd/leakix & 17 | 18 | CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-openbsd-64 ./cmd/leakix & 19 | CGO_ENABLED=0 GOOS=openbsd GOARCH=386 go build -ldflags="-s -w -extldflags '-static'" -o bin/leakix-openbsd-32 ./cmd/leakix & -------------------------------------------------------------------------------- /cmd/leakix-ns/README.md: -------------------------------------------------------------------------------- 1 | ## deprecated 2 | 3 | Kept as example -------------------------------------------------------------------------------- /cmd/leakix/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "github.com/LeakIX/LeakIXClient" 8 | "github.com/LeakIX/l9format" 9 | "log" 10 | "os" 11 | "text/template" 12 | ) 13 | 14 | // It's all sequential, Couldn't care sorry : 15 | var outputJson bool 16 | var scope string 17 | var query string 18 | var liveStream bool 19 | var outputTemplate string 20 | var apiKey string 21 | var endPoint string 22 | var limit int 23 | var tmpl *template.Template 24 | var err error 25 | 26 | func main() { 27 | //Config our app 28 | err = nil 29 | flag.StringVar(&scope, "s", "leak", "Specify scope") 30 | flag.StringVar(&query, "q", "*", "Search mode, specify search query") 31 | flag.BoolVar(&liveStream, "r", false, "Realtime mode, (excludes -q)") 32 | flag.BoolVar(&outputJson, "j", false, "JSON mode, (excludes -t)") 33 | flag.StringVar(&outputTemplate, "t", "{{ .Ip }}:{{ .Port }}", "Specify output template") 34 | flag.StringVar(&apiKey, "k", "", "Specify API key") 35 | flag.StringVar(&endPoint, "e", "https://leakix.net", "Leakix endpoint to use") 36 | 37 | flag.IntVar(&limit, "l", 100, "Limit results output") 38 | flag.Usage = func() { 39 | fmt.Printf("Usage of leakix: \n") 40 | fmt.Printf(" ./leakix -q '*' -s leaks -l 200\n\n") 41 | fmt.Printf(" ./leakix -r -s services -l 0\n\n") 42 | flag.PrintDefaults() 43 | } 44 | flag.Parse() 45 | tmpl, err = template.New("output").Parse(outputTemplate + "\n") 46 | if err != nil { 47 | log.Println("Template error :") 48 | log.Fatal(err) 49 | } 50 | 51 | // Run the right thing 52 | if liveStream { 53 | LiveStream() 54 | } else { 55 | Search() 56 | } 57 | 58 | } 59 | 60 | func Search() { 61 | searcher := LeakIXClient.SearchResultsClient{ 62 | Scope: scope, 63 | Query: query, 64 | ApiKey: apiKey, 65 | Endpoint: endPoint, 66 | } 67 | count := 0 68 | for searcher.Next() { 69 | count++ 70 | OutputSearchResult(searcher.SearchResult()) 71 | if count >= limit || count >= 10000 { 72 | os.Exit(0) 73 | } 74 | } 75 | if searcher.LastError != nil { 76 | log.Println("finished with errors: " + searcher.LastError.Error()) 77 | } 78 | } 79 | 80 | func LiveStream() { 81 | count := 0 82 | searcher := LeakIXClient.SearchResultsClient{ 83 | ApiKey: apiKey, 84 | Endpoint: endPoint, 85 | } 86 | serviceChannel, err := searcher.GetChannel(scope) 87 | if err != nil { 88 | log.Println("Websocket connection error:") 89 | log.Fatal(err) 90 | } 91 | for { 92 | service := <-serviceChannel 93 | count++ 94 | OutputSearchResult(service) 95 | if count >= limit && limit != 0 { 96 | os.Exit(0) 97 | } 98 | } 99 | } 100 | 101 | func OutputSearchResult(searchResult l9format.L9Event) { 102 | if outputJson { 103 | jsonBody, err := json.Marshal(searchResult) 104 | if err != nil { 105 | log.Println("JSON error :") 106 | log.Fatal(err) 107 | } 108 | fmt.Println(string(jsonBody)) 109 | } else { 110 | err := tmpl.Execute(os.Stdout, searchResult) 111 | if err != nil { 112 | log.Println("Template error :") 113 | log.Fatal(err) 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | import ( 3 | "fmt" 4 | "github.com/LeakIX/LeakIXClient" 5 | ) 6 | 7 | func main(){ 8 | // Create a searcher 9 | LeakIXSearch := LeakIXClient.SearchResultsClient{ 10 | Scope: "leak", 11 | Query: "+protocol:elasticsearch +\"telecom_italia_data\"", 12 | } 13 | // Iterate, the lib will query further pages if needed 14 | for LeakIXSearch.Next() { 15 | // Use the result 16 | leak := LeakIXSearch.SearchResult() 17 | fmt.Printf("%s:%s - Country:%s\n", leak.Ip, leak.Port, leak.GeoIp.CountryName) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/LeakIX/LeakIXClient 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/LeakIX/l9format v1.3.1 7 | github.com/PuerkitoBio/goquery v1.8.0 // indirect 8 | github.com/gorilla/websocket v1.4.2 9 | golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/LeakIX/l9format v1.3.1 h1:IjkVAwzijtQGVar9py93IDGkW5pNiByPgK2xzupxwig= 2 | github.com/LeakIX/l9format v1.3.1/go.mod h1:ra1Z8Zxia6ucdVubZmtuKDOoSVHLxkpT1Sklqbzg+JU= 3 | github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= 4 | github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= 5 | github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= 6 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 7 | github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= 8 | github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= 9 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 10 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 11 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 12 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 13 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 14 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 15 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 16 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 17 | golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= 18 | golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 19 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 20 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 21 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 22 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 23 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 24 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 25 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 26 | --------------------------------------------------------------------------------