├── .gitignore ├── Makefile ├── README.md ├── agent.go ├── cmd.go ├── conf_default.yml ├── file.go ├── go.mod ├── go.sum ├── main.go ├── print.go └── zoomeye ├── result.go ├── zoomeye.go └── zoomeye_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | bin/ 3 | data/ 4 | 5 | *.json 6 | *.txt -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ver?=dev 2 | NAME=ZoomEye-go_$(ver) 3 | BINDIR=bin 4 | GOBUILD=CGO_ENABLED=0 go build -ldflags '-w -s' 5 | 6 | all: linux macos win64 win32 7 | 8 | linux: 9 | GOARCH=amd64 GOOS=linux $(GOBUILD) -o $(BINDIR)/$(NAME)_$@ 10 | 11 | macos: 12 | GOARCH=amd64 GOOS=darwin $(GOBUILD) -o $(BINDIR)/$(NAME)_$@ 13 | 14 | win64: 15 | GOARCH=amd64 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)_$@.exe 16 | 17 | win32: 18 | GOARCH=386 GOOS=windows $(GOBUILD) -o $(BINDIR)/$(NAME)_$@.exe 19 | 20 | release: linux macos win64 win32 21 | chmod +x $(BINDIR)/$(NAME)_* 22 | gzip $(BINDIR)/$(NAME)_linux 23 | gzip $(BINDIR)/$(NAME)_macos 24 | zip -m -j $(BINDIR)/$(NAME)_win32.zip $(BINDIR)/$(NAME)_win32.exe 25 | zip -m -j $(BINDIR)/$(NAME)_win64.zip $(BINDIR)/$(NAME)_win64.exe 26 | 27 | clean: 28 | rm $(BINDIR)/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ZoomEye-go 2 | 3 | `ZoomEye` 是一款网络空间搜索引擎，用户可以使用浏览器方式 [搜索网络设备](https://www.zoomeye.org)。 4 | 5 | `ZoomEye-go` 是一款基于 `ZoomEye API` 开发的 Golang 库，提供了 `ZoomEye` 命令行模式，同时也可以作为 `SDK` 集成到其他工具中。该库可以让技术人员更便捷地**搜索**、**筛选**、**导出** `ZoomEye` 的数据。 6 | 7 | ### 下载安装 8 | 9 | - 在 [Releases](https://github.com/gyyyy/ZoomEye-go/releases) 中获得已经编译好的二进制文件 10 | 11 | - 直接通过Github下载源代码，或运行 `go get` 进行下载安装： 12 | 13 | go get -u github.com/gyyyy/ZoomEye-go 14 | 15 | 进入项目目录，执行 `make` 命令完成编译，编译好的二进制文件存放于 `bin` 目录下： 16 | 17 | make [all/linux/macos/win64/win32] 18 | 19 | ### 使用命令行模式 20 | 21 | #### 基础配置 22 | 23 | 在 `ZoomEye-go` 二进制文件同目录下创建 `conf.yml` 文件，对配置变量进行自定义设置： 24 | 25 | ```yaml 26 | # API-Key和JWT存放路径 27 | ZOOMEYE_CONFIG_PATH: "~/.config/zoomeye/setting" 28 | 29 | # 搜索结果数据缓存路径 30 | ZOOMEYE_CACHE_PATH: "~/.config/zoomeye/cache" 31 | 32 | # 搜索和过滤结果数据保存路径 33 | ZOOMEYE_DATA_PATH: "data" 34 | 35 | # 本地数据超时时间，默认为5天 36 | EXPIRED_TIME: 432000 37 | ``` 38 | 39 | 若不创建或修改配置文件，`ZoomEye-go` 相关文件路径和其他参数默认值都将与 [`conf_default.yml`](conf_default.yml) 描述一致。 40 | 41 | #### 初始化用户凭证 42 | 43 | 首次使用 `ZoomEye-go` 时，需要通过 `init` 命令对用户凭证进行初始化，该凭证将自动作为之后其他 `ZoomEye API` 接口调用时的身份标识。`ZoomEye-go` 实现了 `ZoomEye API` 支持的两种认证方式： 44 | 45 | 1. API-Key (推荐) 46 | 47 | ./ZoomEye-go init -apikey [API-KEY] 48 | 49 | 1. JWT 50 | 51 | ./ZoomEye-go init -username [USERNAME] -password [PASSWORD] 52 | 53 | 使用示例： 54 | 55 | ```bash 56 | ./ZoomEye-go init -apikey "XXXXXXXX-XXXX-XXXXX-XXXX-XXXXXXXXXXX" 57 | succeed to initialize 58 | 59 | [ZoomEye Resources Info] 60 | 61 | Role: developer 62 | Quota: 10000 63 | 64 | ``` 65 | 66 | 推荐使用 `API-Key` 认证方式，用户可以登录 `ZoomEye` 在 [个人信息](https://www.zoomeye.org/profile) 中获取，**注意不要将其泄露给其他人**。`JWT` 认证方式获取的凭证具有时效性，失效后需要重新初始化才能正常使用，本地存储的旧的凭证数据会被覆盖。 67 | 68 | 可以通过 `init -h` 获取帮助。 69 | 70 | #### 查询用户资源信息 71 | 72 | 通过 `info` 命令可以查询 `ZoomEye` 当前用户个人信息以及数据配额，初始化成功后也会自动查询： 73 | 74 | ```bash 75 | ./ZoomEye-go info 76 | succeed to query 77 | 78 | [ZoomEye Resources Info] 79 | 80 | Role: developer 81 | Quota: 10000 82 | 83 | ``` 84 | 85 | #### 搜索 86 | 87 | 搜索是 `ZoomEye-go` 最核心的功能，通过 `search` 命令指定搜索关键词进行使用，支持的参数说明如下： 88 | 89 | ```text 90 | -num [NUM] 设置显示/搜索的数据条数，默认为 20（建议设置20的倍数，因为ZoomEye一次接口查询为20条） 91 | -type [host/web] 设置搜索资源类型，默认为 host（如：-type "web"） 92 | -force 强制调用 ZoomEye API 查询，忽略本地数据和缓存 93 | -count 查询该 dork 在 ZoomEye 数据库中的总量 94 | -facet [FIELD,...] 查询该 dork 在 ZoomEye 数据库中全量数据的分布情况，以逗号分隔（如：-facet "app,service,os"） 95 | -stat [FIELD,...] 统计本次搜索结果数据中指定字段的分布情况，以逗号分隔（如：-stat "app,service,os"） 96 | -figure [pie/hist] 输出统计数据的饼状图/柱状图（仅在指定了 -facet 或 -stat 参数下有效） 97 | -filter [FIELD,...] 对本次搜索结果数据中指定字段进行筛选，以逗号分隔（如：-filter "app,ip,title"） 98 | -save 保存本次搜索结果数据，若使用 filter 参数指定了筛选条件，筛选结果也会保存 99 | ``` 100 | 101 | 根据搜索资源类型的不同（由参数 `-type` 确定），其他部分参数值范围存在差异，并且可能根据 `ZoomEye` 官方更新而改变： 102 | 103 | - `-facet` 与 `-stat` ，它们取值一样，只是统计的数据集范围不同 104 | - 当 `-type` 为 `host` 时，可以使用 `app,device,service,os,port,country,city` 105 | - 当 `-type` 为 `web` 时，可以使用 `webapp,component,framework,frontend,server,waf,os,country,city` 106 | - `-filter` 107 | - 当 `-type` 为 `host` 时，可以使用 `app,version,device,ip,port,hostname,city,country,asn,banner,time,*` 108 | - 当 `-type` 为 `web` 时，可以使用 `app,headers,keywords,title,ip,site,city,country,time,*` 109 | 110 | 使用示例： 111 | 112 | ```bash 113 | ./ZoomEye-go search "telnet" -count 114 | succeed to search (in 272.080753ms) 115 | 116 | [ZoomEye Total] 117 | 118 | Count: 57003299 119 | 120 | 121 | ./ZoomEye-go search "telnet" -num 1 122 | succeed to search (in 370.930383ms) 123 | 124 | [Host Search Result] 125 | 126 | +-----------------------+----------------------+----------------------+------------------------------------------+----------------------+ 127 | | Host | Application | Service | Banner | Country | 128 | +-----------------------+----------------------+----------------------+------------------------------------------+----------------------+ 129 | | 159.203.16.45:10005 | Pocket CMD telnetd | telnet | \xff\xfb\x01\xff\xfb\x03\xff\xfc\'\xf... | Canada | 130 | +-----------------------+----------------------+----------------------+------------------------------------------+----------------------+ 131 | | Total: 1 | 132 | +-----------------------+----------------------+----------------------+------------------------------------------+----------------------+ 133 | 134 | 135 | ./ZoomEye-go search "weblogic" -facet "country" -figure "hist" 136 | succeed to search (in 177.088662ms) 137 | 138 | [ZoomEye Facets - HIST] 139 | 140 | Type: country 141 | 142 | United States [232751] ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 143 | Japan [ 45285] ▉▉▉▉▉▉▉ 144 | China [ 37926] ▉▉▉▉▉▉ 145 | Bahrain [ 28816] ▉▉▉▉▌ 146 | Germany [ 28001] ▉▉▉▉▍ 147 | South Africa [ 27929] ▉▉▉▉▍ 148 | Sweden [ 25679] ▉▉▉▉ 149 | Brazil [ 25655] ▉▉▉▉ 150 | India [ 25036] ▉▉▉▉ 151 | Ireland [ 24407] ▉▉▉▊ 152 | 153 | ``` 154 | 155 | 可以通过 `search -h` 获取帮助。 156 | 157 | #### 缓存机制 158 | 159 | `ZoomEye-go` 参考官方 `ZoomEye-python` 的设计，在命令行模式下提供了相似的缓存机制，数据默认存储在 `~/.config/zoomeye/cache` 目录，尽可能节约用户配额。搜索过的数据将默认在本地缓存 5 天，在缓存数据有效期内，重复执行同条件搜索不会消耗配额。可以设置 `-force` 参数强制调用 `ZoomEye API` 进行搜索，结果会覆盖当前缓存数据。 160 | 161 | 通过 `clear` 命令可以清空所有缓存数据和用户数据。 162 | 163 | #### 加载分析本地数据 164 | 165 | `ZoomEye-go` 也可以通过 `load` 命令加载本地数据文件，并将它解析成搜索结果数据类型，支持与 `search` 命令类似的 `-count` 、 `-facet` 、 `-stat` 、 `-figure` 和 `-filter` 参数对数据进行统计分析。不同的是，`-save` 参数仅会保存 `-filter` 的执行结果。 166 | 167 | 可以通过 `load -h` 获取帮助。 168 | 169 | #### 设备历史数据搜索 170 | 171 | `ZoomEye-go`使用`history`命令根据指定的IP查询设备历史数据，支持的参数说明如下： 172 | 173 | ```text 174 | -filter [FIELD,...] 对本次搜索结果数据中指定字段进行筛选，以逗号分隔（如：-filter "time,app,service"） 175 | -num [NUM] 设置显示的数据条数 176 | -force 强制调用 ZoomEye API 查询，忽略本地缓存 177 | ``` 178 | 179 | 其中，`-filter`参数支持的取值范围有：`time,port,service,app,raw,*` 180 | 181 | 使用示例： 182 | 183 | ```bash 184 | ./ZoomEye-go history "1.2.3.4" -filter "time=^2016-,app,service" 185 | succeed to query (in 533.785779ms) 186 | 187 | [History Info] 188 | 189 | 1.2.3.4 190 | 191 | Hostname: [unknown] 192 | Country: United States 193 | City: Mukilteo 194 | Organization: [unknown] 195 | Last Updated: 2016-11-22T12:08:31 196 | 197 | Open Ports: 1 198 | Historical Probes: 1 199 | 200 | 201 | [History Result] 202 | 203 | +---------------------+---------------------------+---------------------------+ 204 | | Time | Service | App | 205 | +---------------------+---------------------------+---------------------------+ 206 | | 2016-11-22T12:08:31 | ssh | OpenSSH | 207 | +---------------------+---------------------------+---------------------------+ 208 | | Total: 1 | 209 | +---------------------+---------------------------+---------------------------+ 210 | 211 | ``` 212 | 213 | ### 使用SDK API 214 | 215 | 使用示例： 216 | 217 | ```go 218 | package main 219 | 220 | import ( 221 | "github.com/gyyyy/ZoomEye-go/zoomeye" 222 | ) 223 | 224 | func main() { 225 | // 初始化用户凭证 226 | zoom := zoomeye.New() 227 | jwt, _ := zoom.Login("username@zoomeye.org", "password") 228 | // 或使用 API-Key 进行初始化，不需要再调用 Login() 方法 229 | // zoom := zoomeye.NewWithKey("XXXXXXXX-XXXX-XXXXX-XXXX-XXXXXXXXXXX") 230 | 231 | // 查询用户资源信息 232 | info, _ := zoom.ResourcesInfo() 233 | 234 | // 搜索 235 | result, _ := zoom.DorkSearch("port:80 nginx", 1, "host", "app,service,os") 236 | // 多页搜索，5页（100条）以上会进行并发搜索，减少搜索耗时 237 | // results, _ := zoom.MultiPageSearch("wordpress country:cn", 5, "web", "webapp,server,os") 238 | // 多页搜索（结果合并） 239 | // result, _ := zoom.MultiToOneSearch("wordpress country:cn", 5, "web", "webapp,server,os") 240 | 241 | // 对搜索结果进行统计 242 | stat := result.Statistics("app,service,os") 243 | 244 | // 对搜索结果进行筛选 245 | filt := result.Filter("app,ip,title") 246 | 247 | // 设备历史搜索（需要高级用户或VIP用户权限，结果包含多少条记录就会扣多少额度，非土豪慎用） 248 | history, _ := zoom.HistoryIP("1.2.3.4") 249 | // 对搜索结果进行筛选 250 | histFilt := history.Filter("time=^2016", "app") 251 | } 252 | ``` 253 | 254 | ### TODO 255 | 256 | - 实现交互式命令行模式 257 | 258 | --- 259 | 260 | ### 404StarLink 2.0 - Galaxy 261 | 262 | ![404StarLink Logo](https://github.com/knownsec/404StarLink-Project/raw/master/logo.png) 263 | 264 | `ZoomEye-go` 是 404Team [星链计划2.0](https://github.com/knownsec/404StarLink2.0-Galaxy) 中的一环，如果对 `ZoomEye-go` 有任何疑问又或是想要找小伙伴交流，可以参考星链计划的加群方式。 265 | 266 | - [https://github.com/knownsec/404StarLink2.0-Galaxy#community](https://github.com/knownsec/404StarLink2.0-Galaxy#community) 267 | 268 | --- 269 | 270 | 如果发现Bug请提Issues，欢迎PR。 -------------------------------------------------------------------------------- /agent.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "net" 7 | "net/url" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | "github.com/gyyyy/ZoomEye-go/zoomeye" 14 | "gopkg.in/yaml.v2" 15 | ) 16 | 17 | // NoAuthKeyErr represents error of no any Auth Keys 18 | type NoAuthKeyErr struct { 19 | msg string 20 | } 21 | 22 | func (e *NoAuthKeyErr) Error() string { 23 | return e.msg 24 | } 25 | 26 | func noAuthKey(err error) *NoAuthKeyErr { 27 | return &NoAuthKeyErr{ 28 | msg: err.Error(), 29 | } 30 | } 31 | 32 | type config struct { 33 | ConfigPath string `yaml:"ZOOMEYE_CONFIG_PATH"` 34 | CachePath string `yaml:"ZOOMEYE_CACHE_PATH"` 35 | DataPath string `yaml:"ZOOMEYE_DATA_PATH"` 36 | ExpiredSec uint `yaml:"EXPIRED_TIME"` 37 | } 38 | 39 | func (c *config) check() { 40 | if c.ConfigPath == "" { 41 | c.ConfigPath = abs("~/.config/zoomeye/setting") 42 | } 43 | if c.CachePath == "" { 44 | c.CachePath = abs("~/.config/zoomeye/cache") 45 | } 46 | if c.DataPath == "" { 47 | c.DataPath = "data" 48 | } 49 | if c.ExpiredSec == 0 { 50 | c.ExpiredSec = 432000 51 | } 52 | if checkFolder(c.ConfigPath) == nil && checkFolder(c.CachePath) == nil && checkFolder(c.DataPath) == nil { 53 | if b, err := yaml.Marshal(c); err == nil { 54 | writeFile(filepath.Join(c.ConfigPath, "conf.yml"), b) 55 | } 56 | } 57 | } 58 | 59 | func newConfig() *config { 60 | conf := &config{ 61 | ConfigPath: abs("~/.config/zoomeye/setting"), 62 | } 63 | defer conf.check() 64 | var ( 65 | path = "conf.yml" 66 | b, err = readFile(path) 67 | ) 68 | if err != nil { 69 | if !os.IsNotExist(err) { 70 | return conf 71 | } 72 | if b, err = readFile(filepath.Join(conf.ConfigPath, path)); err != nil && !os.IsNotExist(err) { 73 | return conf 74 | } 75 | } 76 | if b != nil && len(b) > 0 { 77 | yaml.Unmarshal(b, conf) 78 | } 79 | return conf 80 | } 81 | 82 | func filename(resource, dork string, n int, enc bool) string { 83 | name := fmt.Sprintf("%s_%s_%d", resource, dork, n) 84 | if enc { 85 | name = fmt.Sprintf("%x", md5.Sum([]byte(name))) 86 | } 87 | return name + ".json" 88 | } 89 | 90 | // ZoomEyeAgent represents agent of ZoomEye 91 | type ZoomEyeAgent struct { 92 | zoom *zoomeye.ZoomEye 93 | conf *config 94 | } 95 | 96 | func (a *ZoomEyeAgent) isExpiredData(t time.Time) bool { 97 | return time.Now().Sub(t) > (time.Duration(a.conf.ExpiredSec) * time.Second) 98 | } 99 | 100 | func (a *ZoomEyeAgent) hasCached(path string) bool { 101 | info, err := os.Stat(path) 102 | if err != nil { 103 | return false 104 | } 105 | if a.isExpiredData(info.ModTime()) { 106 | os.Remove(path) 107 | return false 108 | } 109 | return true 110 | } 111 | 112 | func (a *ZoomEyeAgent) fromCache(name string, result interface{}) bool { 113 | path := filepath.Join(a.conf.CachePath, name) 114 | return a.hasCached(path) && readObject(result, path) == nil 115 | } 116 | 117 | func (a *ZoomEyeAgent) cache(name string, result interface{}) error { 118 | return writeObject(filepath.Join(a.conf.CachePath, name), result) 119 | } 120 | 121 | // InitByKey initializes ZoomEye by API-Key 122 | func (a *ZoomEyeAgent) InitByKey(apiKey string) (*zoomeye.ResourcesInfoResult, error) { 123 | var ( 124 | zoom = zoomeye.NewWithKey(apiKey, "") 125 | result, err = zoom.ResourcesInfo() 126 | ) 127 | if err != nil { 128 | return nil, err 129 | } 130 | if err = writeFile(filepath.Join(a.conf.ConfigPath, "apikey"), []byte(apiKey)); err != nil { 131 | return nil, err 132 | } 133 | a.zoom = zoom 134 | return result, nil 135 | } 136 | 137 | // InitByUser initializes ZoomEye by username/password 138 | func (a *ZoomEyeAgent) InitByUser(username, password string) (*zoomeye.ResourcesInfoResult, error) { 139 | var ( 140 | zoom = zoomeye.New() 141 | tok, err = zoom.Login(username, password) 142 | ) 143 | if err != nil { 144 | return nil, err 145 | } 146 | result, err := zoom.ResourcesInfo() 147 | if err != nil { 148 | return nil, err 149 | } 150 | if err = writeFile(filepath.Join(a.conf.ConfigPath, "jwt"), []byte(tok)); err != nil { 151 | return nil, err 152 | } 153 | a.zoom = zoom 154 | return result, nil 155 | } 156 | 157 | func (a *ZoomEyeAgent) loadAuthKey(name string) (string, error) { 158 | var ( 159 | path = filepath.Join(a.conf.ConfigPath, name) 160 | info, err = os.Stat(path) 161 | ) 162 | if err != nil { 163 | return "", noAuthKey(err) 164 | } 165 | if !strings.HasSuffix(fmt.Sprintf("%o", info.Mode()), "600") { 166 | os.Chmod(path, 0o600) 167 | } 168 | b, err := readFile(path) 169 | if err != nil { 170 | return "", noAuthKey(err) 171 | } 172 | return string(b), nil 173 | } 174 | 175 | // InitLocal initializes ZoomEye from local Key files 176 | func (a *ZoomEyeAgent) InitLocal() (*zoomeye.ResourcesInfoResult, error) { 177 | var ( 178 | result *zoomeye.ResourcesInfoResult 179 | apiKey, accessToken string 180 | err error 181 | ) 182 | if apiKey, err = a.loadAuthKey("apikey"); err != nil { 183 | if accessToken, err = a.loadAuthKey("jwt"); err != nil { 184 | return nil, err 185 | } 186 | } 187 | zoom := zoomeye.NewWithKey(apiKey, accessToken) 188 | if result, err = zoom.ResourcesInfo(); err != nil { 189 | return nil, err 190 | } 191 | a.zoom = zoom 192 | return result, nil 193 | } 194 | 195 | // Info gets resources information 196 | func (a *ZoomEyeAgent) Info() (*zoomeye.ResourcesInfoResult, error) { 197 | if a.zoom == nil { 198 | return a.InitLocal() 199 | } 200 | return a.zoom.ResourcesInfo() 201 | } 202 | 203 | func (a *ZoomEyeAgent) fromLocal(name string) (*zoomeye.SearchResult, bool) { 204 | path := filepath.Join(a.conf.DataPath, name) 205 | if info, err := os.Stat(path); err != nil || a.isExpiredData(info.ModTime()) { 206 | return nil, false 207 | } 208 | result := &zoomeye.SearchResult{} 209 | if readObject(result, path) != nil { 210 | return nil, false 211 | } 212 | return result, true 213 | } 214 | 215 | func (a *ZoomEyeAgent) forceSearch(dork string, maxPage int, resource string) (*zoomeye.SearchResult, error) { 216 | results, err := a.zoom.MultiPageSearch(dork, maxPage, resource, "") 217 | if err != nil { 218 | return nil, err 219 | } 220 | result := &zoomeye.SearchResult{ 221 | Type: resource, 222 | } 223 | for i, n := 0, len(results); i < maxPage && n > 0; i++ { 224 | page := i + 1 225 | if res, ok := results[page]; ok { 226 | a.cache(filename(resource, dork, page, true), res) 227 | result.Extend(res) 228 | n-- 229 | } 230 | } 231 | return result, nil 232 | } 233 | 234 | // Search gets search results from local, cache or API 235 | func (a *ZoomEyeAgent) Search(dork string, num int, resource string, force bool) (*zoomeye.SearchResult, error) { 236 | if a.zoom == nil { 237 | if _, err := a.InitLocal(); err != nil { 238 | return nil, err 239 | } 240 | } 241 | if num <= 0 { 242 | num = 20 243 | } 244 | maxPage := num / 20 245 | if num%20 > 0 { 246 | maxPage++ 247 | } 248 | if resource = strings.ToLower(resource); resource != "web" { 249 | resource = "host" 250 | } 251 | if force { 252 | return a.forceSearch(dork, maxPage, resource) 253 | } 254 | result, ok := a.fromLocal(filename(resource, url.QueryEscape(dork), num, false)) 255 | if ok { 256 | result.Type = resource 257 | return result, nil 258 | } 259 | result = &zoomeye.SearchResult{ 260 | Type: resource, 261 | } 262 | for i := 0; i < maxPage; i++ { 263 | var ( 264 | res = &zoomeye.SearchResult{} 265 | page = i + 1 266 | name = filename(resource, dork, page, true) 267 | ) 268 | if !a.fromCache(name, res) { 269 | var err error 270 | if res, err = a.zoom.DorkSearch(dork, page, resource, ""); err != nil { 271 | return nil, err 272 | } 273 | a.cache(name, res) 274 | } 275 | result.Extend(res) 276 | } 277 | if num < len(result.Matches) { 278 | result.Matches = result.Matches[:num] 279 | } 280 | return result, nil 281 | } 282 | 283 | // Load reads local data, and unmarshals to search results 284 | func (a *ZoomEyeAgent) Load(path string) (*zoomeye.SearchResult, error) { 285 | if _, err := os.Stat(path); err != nil { 286 | return nil, err 287 | } 288 | result := &zoomeye.SearchResult{ 289 | Type: "host", 290 | } 291 | if err := readObject(result, path); err != nil { 292 | return nil, err 293 | } 294 | if len(result.Matches) > 0 && result.Matches[0].Find("site") != nil { 295 | result.Type = "web" 296 | } 297 | return result, nil 298 | } 299 | 300 | // History gets query results of device history by IP 301 | func (a *ZoomEyeAgent) History(ip string, force bool) (*zoomeye.HistoryResult, error) { 302 | if net.ParseIP(ip) == nil { 303 | return nil, fmt.Errorf("invalid ip address") 304 | } 305 | info, err := a.Info() 306 | if err != nil { 307 | return nil, err 308 | } 309 | switch strings.ToLower(info.Plan) { 310 | case "user", "developer": 311 | return nil, fmt.Errorf("this function is only open to advanced users and VIP users.") 312 | } 313 | var ( 314 | result *zoomeye.HistoryResult 315 | name = fmt.Sprintf("%x", md5.Sum([]byte("history_"+ip))) + ".json" 316 | ok bool 317 | ) 318 | if !force { 319 | result = &zoomeye.HistoryResult{} 320 | ok = a.fromCache(name, result) 321 | } 322 | if !ok { 323 | if result, err = a.zoom.HistoryIP(ip); err != nil { 324 | return nil, err 325 | } 326 | if result.Count == 0 || len(result.Data) == 0 { 327 | return result, nil 328 | } 329 | a.cache(name, result) 330 | } 331 | for i := 0; i < len(result.Data); { 332 | if _, ok := result.Data[i]["component"]; ok { 333 | result.Data = append(result.Data[:i], result.Data[i+1:]...) 334 | } else { 335 | i++ 336 | } 337 | } 338 | result.Count = uint64(len(result.Data)) 339 | return result, nil 340 | } 341 | 342 | // Clear removes all cache or setting data 343 | func (a *ZoomEyeAgent) Clear(cache, setting bool) { 344 | if cache { 345 | os.RemoveAll(a.conf.CachePath) 346 | } 347 | if setting { 348 | os.RemoveAll(a.conf.ConfigPath) 349 | os.MkdirAll(a.conf.ConfigPath, os.ModePerm) 350 | if b, err := yaml.Marshal(a.conf); err == nil { 351 | writeFile(filepath.Join(a.conf.ConfigPath, "conf.yml"), b) 352 | } 353 | } 354 | } 355 | 356 | // SaveFilterData writes the filter data to local file 357 | func (a *ZoomEyeAgent) SaveFilterData(path string, data []map[string]interface{}) error { 358 | if data == nil || len(data) == 0 { 359 | return fmt.Errorf("no any filter datas") 360 | } 361 | return writeObject(path, data) 362 | } 363 | 364 | // Save writes the search results (and filter data) to local file 365 | func (a *ZoomEyeAgent) Save(name string, result *zoomeye.SearchResult) (string, error) { 366 | path := filepath.Join(a.conf.DataPath, name+".json") 367 | if err := writeObject(path, result); err != nil { 368 | return "", err 369 | } 370 | path, _ = filepath.Abs(path) 371 | return path, nil 372 | } 373 | 374 | // NewAgent creates instance of ZoomEyeAgent 375 | func NewAgent() *ZoomEyeAgent { 376 | return &ZoomEyeAgent{ 377 | conf: newConfig(), 378 | } 379 | } 380 | -------------------------------------------------------------------------------- /cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "reflect" 10 | "strconv" 11 | "strings" 12 | "time" 13 | "unsafe" 14 | 15 | "github.com/gyyyy/ZoomEye-go/zoomeye" 16 | ) 17 | 18 | func parseFlags(cmd string, flgs interface{}, examples ...string) []string { 19 | if flgs != nil { 20 | var ( 21 | v = reflect.ValueOf(flgs) 22 | elem = v.Elem() 23 | ptr = v.Pointer() 24 | ) 25 | for i := 0; i < elem.NumField(); i++ { 26 | var ( 27 | f = elem.Type().Field(i) 28 | fname = f.Tag.Get("name") 29 | fvalue = f.Tag.Get("value") 30 | fusage = f.Tag.Get("usage") 31 | fptr = unsafe.Pointer(ptr + f.Offset) 32 | ) 33 | if fname == "" { 34 | fname = strings.ToLower(f.Name) 35 | } 36 | switch f.Type.Kind() { 37 | case reflect.String: 38 | flag.StringVar((*string)(fptr), fname, fvalue, fusage) 39 | case reflect.Int: 40 | val, _ := strconv.Atoi(fvalue) 41 | flag.IntVar((*int)(fptr), fname, val, fusage) 42 | case reflect.Bool: 43 | val, _ := strconv.ParseBool(fvalue) 44 | flag.BoolVar((*bool)(fptr), fname, val, fusage) 45 | } 46 | } 47 | } 48 | flag.Usage = func() { 49 | fmt.Fprintf(flag.CommandLine.Output(), "\nUsage of %s (%s):\n", filepath.Base(os.Args[0]), cmd) 50 | flag.PrintDefaults() 51 | if len(examples) > 0 { 52 | example := "Example:\n" 53 | for _, v := range examples { 54 | example += fmt.Sprintf(" ./ZoomEye-go %s %s\n", cmd, v) 55 | } 56 | fmt.Fprintf(flag.CommandLine.Output(), "\n%s\n", example) 57 | } 58 | } 59 | var args []string 60 | for i := 1; i < len(os.Args); { 61 | if p := os.Args[i]; !strings.HasPrefix(p, "-") { 62 | args = append(args, p) 63 | os.Args = append(os.Args[0:i], os.Args[i+1:]...) 64 | } else { 65 | break 66 | } 67 | } 68 | flag.Parse() 69 | return args 70 | } 71 | 72 | func checkError(err error) { 73 | switch err.(type) { 74 | case *NoAuthKeyErr: 75 | warnf("not found any Auth Keys, please run first") 76 | case *zoomeye.ErrorResult: 77 | errorf("failed to authenticate: %v", err) 78 | case nil: 79 | default: 80 | errorf("something is wrong: %v", err) 81 | } 82 | } 83 | 84 | type resultAnalyzer struct { 85 | count bool 86 | facet string 87 | stat string 88 | figure string 89 | filter string 90 | save bool 91 | } 92 | 93 | func newResultAnalyzer() *resultAnalyzer { 94 | analyzer := &resultAnalyzer{} 95 | flag.BoolVar(&analyzer.count, "count", false, "The total number of results in ZoomEye database") 96 | flag.StringVar(&analyzer.facet, "facet", "", "Perform statistics on ZoomEye database") 97 | flag.StringVar(&analyzer.stat, "stat", "", "Perform statistics on search results") 98 | flag.StringVar(&analyzer.figure, "figure", "", "Output Pie or bar chart only be used under -facet and -stat") 99 | flag.StringVar(&analyzer.filter, "filter", "", "Output more clearer search results by set filter field") 100 | flag.BoolVar(&analyzer.save, "save", false, "Save data in JSON format") 101 | return analyzer 102 | } 103 | 104 | func (a *resultAnalyzer) do(result *zoomeye.SearchResult, saveCallback func([]map[string]interface{})) { 105 | if a.count { 106 | infof("ZoomEye Total", "Count: %d", result.Total) 107 | } 108 | if a.figure != "" { 109 | if a.figure = strings.ToLower(a.figure); a.figure != "pie" { 110 | a.figure = "hist" 111 | } 112 | } 113 | if a.facet != "" { 114 | showFacet(result, strings.Split(a.facet, ","), a.figure) 115 | } 116 | if a.stat != "" { 117 | showStat(result, strings.Split(a.stat, ","), a.figure) 118 | } 119 | var filtered []map[string]interface{} 120 | if a.filter != "" { 121 | filtered = showFilter(result, strings.Split(a.filter, ",")) 122 | } 123 | if !a.count && a.facet == "" && a.stat == "" && a.filter == "" { 124 | showData(result) 125 | } 126 | if a.save && saveCallback != nil { 127 | saveCallback(filtered) 128 | } 129 | } 130 | 131 | func cmdInit(agent *ZoomEyeAgent) { 132 | var flgs struct { 133 | apiKey string `usage:"ZoomEye API-Key"` 134 | username string `usage:"ZoomEye account username"` 135 | password string `usage:"ZoomEye account password"` 136 | } 137 | parseFlags("init", &flgs, `-apikey "XXXXXXXX-XXXX-XXXXX-XXXX-XXXXXXXXXXX"`, 138 | `-username "username@zoomeye.org" -password "password"`) 139 | var ( 140 | result *zoomeye.ResourcesInfoResult 141 | err error 142 | ) 143 | if flgs.apiKey != "" { 144 | if result, err = agent.InitByKey(flgs.apiKey); err != nil { 145 | errorf("failed to initialize: %v", err) 146 | return 147 | } 148 | } else if flgs.username != "" && flgs.password != "" { 149 | if result, err = agent.InitByUser(flgs.username, flgs.password); err != nil { 150 | errorf("failed to initialize: %v", err) 151 | return 152 | } 153 | } else if result, err = agent.InitLocal(); err != nil { 154 | warnf("required parameter missing, please run for help") 155 | return 156 | } 157 | successf("succeed to initialize") 158 | infof("ZoomEye Resources Info", "Role: %s\nQuota: %d", result.Plan, result.Resources.Search) 159 | } 160 | 161 | func cmdInfo(agent *ZoomEyeAgent) { 162 | result, err := agent.Info() 163 | if err != nil { 164 | checkError(err) 165 | return 166 | } 167 | successf("succeed to query") 168 | infof("ZoomEye Resources Info", "Role: %s\nQuota: %d", result.Plan, result.Resources.Search) 169 | } 170 | 171 | func cmdSearch(agent *ZoomEyeAgent) { 172 | var ( 173 | analyzer = newResultAnalyzer() 174 | flgs struct { 175 | num int `value:"20" usage:"The number of search results that should be returned, multiple of 20"` 176 | resource string `name:"type" usage:"Specify the type of resource to search"` 177 | force bool `usage:"Ignore local and cache data"` 178 | } 179 | args = parseFlags("search", &flgs, `"weblogic" -facet "app" -count`) 180 | ) 181 | if len(args) == 0 { 182 | warnf("search keyword missing, please run for help") 183 | return 184 | } 185 | var ( 186 | dork = args[0] 187 | start = time.Now() 188 | result, err = agent.Search(dork, flgs.num, flgs.resource, flgs.force) 189 | since = time.Since(start) 190 | ) 191 | if err != nil { 192 | checkError(err) 193 | return 194 | } 195 | successf("succeed to search (in %v)", since) 196 | analyzer.do(result, func(filtered []map[string]interface{}) { 197 | name := fmt.Sprintf("%s_%s_%d", flgs.resource, url.QueryEscape(dork), flgs.num) 198 | if path, err := agent.Save(name, result); err != nil { 199 | errorf("failed to save: %v", err) 200 | } else { 201 | successf("succeed to save (%s)", path) 202 | agent.SaveFilterData(filepath.Join(agent.conf.DataPath, name+"_filtered.json"), filtered) 203 | } 204 | }) 205 | } 206 | 207 | func cmdLoad(agent *ZoomEyeAgent) { 208 | var ( 209 | analyzer = newResultAnalyzer() 210 | args = parseFlags("load", nil, `"data/host_weblogic_20.json" -facet "app" -count`) 211 | ) 212 | if len(args) == 0 { 213 | warnf("path of local data file missing, please run for help") 214 | return 215 | } 216 | var ( 217 | file = args[0] 218 | result, err = agent.Load(file) 219 | ) 220 | if err != nil { 221 | errorf("invalid local data: %v", err) 222 | return 223 | } 224 | successf("succeed to load") 225 | analyzer.do(result, func(filtered []map[string]interface{}) { 226 | var ( 227 | ext = filepath.Ext(file) 228 | path = strings.TrimSuffix(file, ext) + "_filtered" + ext 229 | ) 230 | if err := agent.SaveFilterData(path, filtered); err != nil { 231 | errorf("failed to save: %v", err) 232 | } else { 233 | path, _ = filepath.Abs(path) 234 | successf("succeed to save (%s)", path) 235 | } 236 | }) 237 | } 238 | 239 | func cmdHistory(agent *ZoomEyeAgent) { 240 | var ( 241 | flgs struct { 242 | filter string `usage:"Output more clearer query results by set filter field"` 243 | num int `value:"20" usage:"The number of results that should be returned"` 244 | force bool `usage:"Ignore cache data"` 245 | } 246 | args = parseFlags("history", &flgs, `"0.0.0.0" -filter "time=^2020-03,port,service" -num 1`) 247 | ) 248 | if len(args) == 0 { 249 | warnf("ip missing, please run for help") 250 | return 251 | } 252 | var ( 253 | start = time.Now() 254 | result, err = agent.History(args[0], flgs.force) 255 | since = time.Since(start) 256 | ) 257 | if err != nil { 258 | checkError(err) 259 | return 260 | } 261 | successf("succeed to query (in %v)", since) 262 | showHistory(result, strings.Split(flgs.filter, ","), flgs.num) 263 | } 264 | 265 | func cmdClear(agent *ZoomEyeAgent) { 266 | var flgs struct { 267 | cache bool 268 | setting bool 269 | } 270 | parseFlags("clear", &flgs, `-cache`, `-cache -setting`) 271 | agent.Clear(flgs.cache, flgs.setting) 272 | successf("succeed to clear data") 273 | } 274 | -------------------------------------------------------------------------------- /conf_default.yml: -------------------------------------------------------------------------------- 1 | # api key and json web token file path 2 | ZOOMEYE_CONFIG_PATH: "~/.config/zoomeye/setting" 3 | 4 | # search data cache path 5 | ZOOMEYE_CACHE_PATH: "~/.config/zoomeye/cache" 6 | 7 | # search and filter data save path 8 | ZOOMEYE_DATA_PATH: "data" 9 | 10 | # data expired time, default five day 11 | EXPIRED_TIME: 432000 -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | "os/user" 10 | "path/filepath" 11 | "runtime" 12 | "strings" 13 | ) 14 | 15 | func winHome() string { 16 | var ( 17 | drive = os.Getenv("HOMEDRIVE") 18 | path = os.Getenv("HOMEPATH") 19 | ) 20 | if drive == "" || path == "" { 21 | return os.Getenv("USERPROFILE") 22 | } 23 | return drive + path 24 | } 25 | 26 | func unixHome() string { 27 | if home := os.Getenv("HOME"); home != "" { 28 | return home 29 | } 30 | var ( 31 | out bytes.Buffer 32 | cmd = exec.Command("sh", "-c", "eval echo ~$USER") 33 | ) 34 | cmd.Stdout = &out 35 | if err := cmd.Run(); err != nil { 36 | return "" 37 | } 38 | return strings.TrimSpace(out.String()) 39 | } 40 | 41 | func home() string { 42 | if user, err := user.Current(); nil == err { 43 | return user.HomeDir 44 | } 45 | if runtime.GOOS == "windows" { 46 | return winHome() 47 | } 48 | return unixHome() 49 | } 50 | 51 | func abs(dir string) string { 52 | if strings.HasPrefix(dir, "~/") { 53 | if base := home(); base != "" { 54 | return filepath.Join(base, strings.TrimPrefix(dir, "~/")) 55 | } 56 | } 57 | return dir 58 | } 59 | 60 | func checkFolder(dir string) error { 61 | if info, err := os.Stat(dir); err != nil { 62 | if !os.IsNotExist(err) { 63 | return err 64 | } 65 | } else if info != nil && info.IsDir() { 66 | return nil 67 | } 68 | return os.MkdirAll(dir, os.ModePerm) 69 | } 70 | 71 | func readFile(path string) ([]byte, error) { 72 | return ioutil.ReadFile(path) 73 | } 74 | 75 | func readObject(dst interface{}, path string) error { 76 | b, err := readFile(path) 77 | if err != nil { 78 | return err 79 | } 80 | return json.Unmarshal(b, dst) 81 | } 82 | 83 | func writeFile(path string, data []byte) error { 84 | return ioutil.WriteFile(path, data, 0o600) 85 | } 86 | 87 | func writeObject(path string, src interface{}) error { 88 | b, err := json.Marshal(src) 89 | if err != nil { 90 | return err 91 | } 92 | return writeFile(path, b) 93 | } 94 | 95 | func appendToFile(path string, data []byte) error { 96 | f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600) 97 | if err != nil { 98 | return err 99 | } 100 | defer f.Close() 101 | if data != nil && len(data) > 0 { 102 | _, err = f.Write(append(data, '\n')) 103 | } 104 | return err 105 | } 106 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gyyyy/ZoomEye-go 2 | 3 | go 1.15 4 | 5 | require gopkg.in/yaml.v2 v2.4.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 2 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 3 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 4 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 5 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | const ver = "v1.6" 11 | 12 | func banner() { 13 | fmt.Println( 14 | colorf("\n ,----,\n"+ 15 | " .' .`| ____ ,---,.\n"+ 16 | " .' .' ; ,' , `. ,' .' |\n"+ 17 | " ,---, ' .' ,---. ,---. ,-+-,.' _ |,---.' | ", colorLightYellow), 18 | colorf(ver, colorLightWhite), 19 | colorf("\n | : ./ ' ,'\\ ' ,'\\ ,-+-. ; , ||| | .'\n"+ 20 | " ; | .' / / / | / / | ,--.'|' | ||: : |-, .--, ,---.\n"+ 21 | " `---' / ; . ; ,. :. ; ,. :| | ,', | |,: | ;/| /_ ./| / \\\n"+ 22 | " / ; / ' | |: :' | |: :| | / | |--' | : .' , ' , ' : / / |\n"+ 23 | " ; / /--,' | .; :' | .; :| : | | , | | |-,/___/ \\: |. ' / |\n"+ 24 | " / / / .`|| : || : || : | |/ ' : ;/| . \\ ' |' ; /|\n"+ 25 | "./__; : \\ \\ / \\ \\ / | | |`-' | | \\ \\ ; :' | / |\n"+ 26 | "| : .' `----' `----' | ;/ | : .' \\ \\ ;| : |\n"+ 27 | "; | .' '---' | | ,' : \\ \\\\ \\ /\n"+ 28 | "`---' ", colorLightYellow), 29 | colorf("", colorLightWhite), 30 | colorf(" `----' \\ ' ; `----'\n"+ 31 | " `--`\n", colorLightYellow), 32 | ) 33 | } 34 | 35 | func help() { 36 | fmt.Printf("Usage of %s:\n"+ 37 | " version\n Show current version\n"+ 38 | " init\n Initialize ZoomEye by username/password or API-Key\n"+ 39 | " info\n Query resources information\n"+ 40 | " search\n Search results from local, cache or API\n"+ 41 | " load\n Load results from local data file\n"+ 42 | " history\n Query device history\n"+ 43 | " clear\n Removes all cache and setting data\n"+ 44 | " help\n Usage of ZoomEye-go\n", 45 | filepath.Base(os.Args[0])) 46 | } 47 | 48 | func main() { 49 | var ( 50 | agent = NewAgent() 51 | cmd string 52 | ) 53 | if len(os.Args) > 1 { 54 | cmd = os.Args[1] 55 | os.Args = append(os.Args[0:1], os.Args[2:]...) 56 | } 57 | switch strings.ToLower(cmd) { 58 | case "init": 59 | cmdInit(agent) 60 | case "info": 61 | cmdInfo(agent) 62 | case "search": 63 | cmdSearch(agent) 64 | case "load": 65 | cmdLoad(agent) 66 | case "history": 67 | cmdHistory(agent) 68 | case "clear": 69 | cmdClear(agent) 70 | case "version", "-version", "--version", "ver", "-ver", "--ver", "-v", "--v": 71 | banner() 72 | case "help", "-help", "--help", "-h", "--h", "?": 73 | help() 74 | case "": 75 | warnf("Cli-User-Interact mode is coming soon, please run for help") 76 | default: 77 | warnf("unsupported command please run for help") 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /print.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "sort" 7 | "strings" 8 | 9 | "github.com/gyyyy/ZoomEye-go/zoomeye" 10 | ) 11 | 12 | const ( 13 | colorReset = "\033[0m" 14 | colorBlack = "\033[0;30m" 15 | colorRed = "\033[0;31m" 16 | colorGreen = "\033[0;32m" 17 | colorYellow = "\033[0;33m" 18 | colorBlue = "\033[0;34m" 19 | colorPurple = "\033[0;35m" 20 | colorCyan = "\033[0;36m" 21 | colorWhite = "\033[0;37m" 22 | colorLightBlack = "\033[1;30m" 23 | colorLightRed = "\033[1;31m" 24 | colorLightGreen = "\033[1;32m" 25 | colorLightYellow = "\033[1;33m" 26 | colorLightBlue = "\033[1;34m" 27 | colorLightPurple = "\033[1;35m" 28 | colorLightCyan = "\033[1;36m" 29 | colorLightWhite = "\033[1;37m" 30 | colorDarkBlack = "\033[2;30m" 31 | colorDarkRed = "\033[2;31m" 32 | colorDarkGreen = "\033[2;32m" 33 | colorDarkYellow = "\033[2;33m" 34 | colorDarkBlue = "\033[2;34m" 35 | colorDarkPurple = "\033[2;35m" 36 | colorDarkCyan = "\033[2;36m" 37 | colorDarkWhite = "\033[2;37m" 38 | ) 39 | 40 | var ( 41 | ctrlChars = map[rune]string{ 42 | '\t': "\\t", 43 | '\n': "\\n", 44 | '\v': "\\v", 45 | '\f': "\\f", 46 | '\r': "\\r", 47 | '\a': "\\a", 48 | '\b': "\\b", 49 | } 50 | pieColors = []string{ 51 | "\033[1;34m", "\033[1;35m", "\033[1;36m", "\033[1;31m", "\033[1;33m", 52 | "\033[0;94m", "\033[0;95m", "\033[0;96m", "\033[0;91m", "\033[0;93m", 53 | } 54 | histChars = []string{ 55 | " ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█", 56 | } 57 | ) 58 | 59 | func colorf(s, color string) string { 60 | if color == "" { 61 | return s 62 | } 63 | return color + s + colorReset 64 | } 65 | 66 | func print(s, color string) { 67 | fmt.Println(colorf(s, color)) 68 | } 69 | 70 | func errorf(format string, a ...interface{}) { 71 | print(fmt.Sprintf(format, a...), colorRed) 72 | } 73 | 74 | func successf(format string, a ...interface{}) { 75 | print(fmt.Sprintf(format, a...), colorGreen) 76 | } 77 | 78 | func warnf(format string, a ...interface{}) { 79 | print(fmt.Sprintf(format, a...), colorYellow) 80 | } 81 | 82 | func infof(title, format string, a ...interface{}) { 83 | if title != "" { 84 | format = "\n" + colorf("["+title+"]", colorLightCyan) + "\n\n" + 85 | colorf(" "+strings.ReplaceAll(format, "\n", "\n "), colorLightWhite) + "\n" 86 | print(fmt.Sprintf(format, a...), "") 87 | } else { 88 | print(fmt.Sprintf(format, a...), colorWhite) 89 | } 90 | } 91 | 92 | func toStr(o interface{}) string { 93 | if o == nil { 94 | return "" 95 | } 96 | switch o := o.(type) { 97 | case string: 98 | return o 99 | case []string: 100 | return strings.Join(o, ",") 101 | case []interface{}: 102 | s := make([]string, len(o)) 103 | for i, v := range o { 104 | s[i] = fmt.Sprintf("%v", v) 105 | } 106 | return strings.Join(s, ",") 107 | default: 108 | return fmt.Sprintf("%v", o) 109 | } 110 | } 111 | 112 | func omitStr(o interface{}, maxWidth int) string { 113 | var builder strings.Builder 114 | for _, r := range toStr(o) { 115 | if r > 31 && r < 127 { 116 | builder.WriteRune(r) 117 | } else if v, ok := ctrlChars[r]; ok { 118 | builder.WriteString(v) 119 | } else { 120 | builder.WriteString(fmt.Sprintf("\\x%02x", r)) 121 | } 122 | } 123 | s := builder.String() 124 | if n := len(s); n > maxWidth { 125 | if maxWidth > 3 { 126 | s = s[:maxWidth-3] + "..." 127 | } else { 128 | s = strings.Repeat(".", maxWidth) 129 | } 130 | } 131 | return strings.ReplaceAll(s, "%", "%%") 132 | } 133 | 134 | func tablef(title string, head [][2]interface{}, body map[string][][]interface{}, count bool) { 135 | var ( 136 | builder strings.Builder 137 | n = len(head) 138 | names = make([]interface{}, 0, n) 139 | widths = make([]int, 0, n) 140 | hfmt = colorf("|", colorLightBlack) 141 | bfmt = colorf("|", colorLightBlack) 142 | isGroup bool 143 | ) 144 | for i, v := range head { 145 | var ( 146 | name = v[0].(string) 147 | width = v[1].(int) 148 | ) 149 | if i == 0 && name != "-" { 150 | isGroup = true 151 | } 152 | if i > 0 || name != "-" { 153 | hfmt += fmt.Sprintf(colorf(" %%-%dv ", colorLightGreen)+colorf("|", colorLightBlack), width) 154 | bfmt += fmt.Sprintf(colorf(" %%-%dv ", colorLightWhite)+colorf("|", colorLightBlack), width) 155 | names = append(names, name) 156 | widths = append(widths, width) 157 | } 158 | } 159 | line := "+" 160 | for _, v := range widths { 161 | line += strings.Repeat("-", v+2) + "+" 162 | } 163 | line = colorf(line, colorLightBlack) 164 | var total int 165 | builder.WriteString(line + "\n") 166 | builder.WriteString(fmt.Sprintf(hfmt, names...) + "\n") 167 | if len(body) > 0 { 168 | for k, group := range body { 169 | if isGroup { 170 | k = omitStr(k, widths[0]) 171 | } 172 | builder.WriteString(line + "\n") 173 | for i, v := range group { 174 | if len(v) < len(head)-1 { 175 | continue 176 | } 177 | v = v[:len(head)-1] 178 | for i := range v { 179 | j := i 180 | if isGroup { 181 | j++ 182 | } 183 | v[i] = omitStr(v[i], widths[j]) 184 | } 185 | if total++; k == "" && !isGroup { 186 | builder.WriteString(fmt.Sprintf(bfmt, v...) + "\n") 187 | continue 188 | } 189 | if i > 0 { 190 | k = "" 191 | } 192 | params := append([]interface{}{k}, v...) 193 | builder.WriteString(fmt.Sprintf(bfmt, params...) + "\n") 194 | } 195 | } 196 | } else { 197 | builder.WriteString(line + "\n") 198 | } 199 | if count { 200 | builder.WriteString(line + "\n") 201 | builder.WriteString(fmt.Sprintf( 202 | fmt.Sprintf( 203 | colorf("|", colorLightBlack)+ 204 | colorf(" %%-%ds ", colorLightPurple)+ 205 | colorf("|", colorLightBlack)+"\n", 206 | len(line)-15, 207 | ), 208 | fmt.Sprintf("Total: %d", total), 209 | )) 210 | } 211 | builder.WriteString(line) 212 | infof(title, builder.String()) 213 | } 214 | 215 | func htablef(title string, body []map[string]interface{}, widths [3]int, count bool) { 216 | var ( 217 | builder strings.Builder 218 | hfmt = fmt.Sprintf(colorf("|", colorLightBlack)+ 219 | colorf(" %%-%dv ", colorLightGreen)+colorf("|", colorLightBlack)+ 220 | colorf(" %%-%dv ", colorLightGreen)+colorf("|", colorLightBlack)+ 221 | colorf(" %%-%dv ", colorLightGreen)+colorf("|", colorLightBlack), 222 | widths[0], widths[1], widths[2]) 223 | bfmt = fmt.Sprintf(colorf("|", colorLightBlack)+ 224 | colorf(" %%-%dv ", colorLightWhite)+colorf("|", colorLightBlack)+ 225 | colorf(" %%-%dv ", colorLightWhite)+colorf("|", colorLightBlack)+ 226 | colorf(" %%-%dv ", colorLightWhite)+colorf("|", colorLightBlack), 227 | widths[0], widths[1], widths[2]) 228 | line = "+" 229 | ) 230 | for _, v := range widths { 231 | line += strings.Repeat("-", v+2) + "+" 232 | } 233 | line = colorf(line, colorLightBlack) 234 | var total int 235 | builder.WriteString(line + "\n") 236 | builder.WriteString(fmt.Sprintf(hfmt, "Name", "Key", "Value") + "\n") 237 | if len(body) > 0 { 238 | for _, item := range body { 239 | total++ 240 | builder.WriteString(line + "\n") 241 | name := omitStr(item["name"], widths[0]) 242 | for i, v := range item["items"].([]map[string]interface{}) { 243 | if i > 0 { 244 | name = "" 245 | } 246 | var ( 247 | key = omitStr(v["key"], widths[1]) 248 | val = omitStr(v["value"], widths[2]) 249 | ) 250 | builder.WriteString(fmt.Sprintf(bfmt, name, key, val) + "\n") 251 | } 252 | } 253 | } else { 254 | builder.WriteString(line + "\n") 255 | } 256 | if count { 257 | builder.WriteString(line + "\n") 258 | builder.WriteString(fmt.Sprintf( 259 | fmt.Sprintf( 260 | colorf("|", colorLightBlack)+ 261 | colorf(" %%-%ds ", colorLightPurple)+ 262 | colorf("|", colorLightBlack)+"\n", 263 | len(line)-15, 264 | ), 265 | fmt.Sprintf("Total: %d", total), 266 | )) 267 | } 268 | builder.WriteString(line) 269 | infof(title, builder.String()) 270 | } 271 | 272 | func atanChar(data [][]interface{}, at float64, colors []string) string { 273 | if len(data) == 0 { 274 | return " " 275 | } 276 | n := at - data[0][2].(float64) 277 | if n <= 0 { 278 | return colors[0] + "* " + colorReset 279 | } 280 | return atanChar(data[1:], n, colors[1:]) 281 | } 282 | 283 | func pief(title string, body map[string][][]interface{}) { 284 | var builder strings.Builder 285 | if len(body) > 0 { 286 | first := true 287 | for k, v := range body { 288 | if !first { 289 | builder.WriteString("\n\n\n") 290 | } else { 291 | first = false 292 | } 293 | if len(v) > 10 { 294 | v = v[:10] 295 | } 296 | for i, y := 0, -7; y < 7; y++ { 297 | var c string 298 | for x := -7; x < 7; x++ { 299 | if x*x+y*y < 49 { 300 | c += atanChar(v, math.Atan2(float64(y), float64(x))/math.Pi/2+0.5, pieColors) 301 | } else { 302 | c += " " 303 | } 304 | } 305 | if i > 0 { 306 | if i == 1 { 307 | builder.WriteString(c + " " + colorf("Type: "+k, colorLightGreen) + "\n") 308 | } else if n := i - 3; n >= 0 && n < len(v) { 309 | name := omitStr(v[n][0], 35) 310 | builder.WriteString(c + " " + 311 | colorf(fmt.Sprintf("%5.2f%%%% - %s", v[n][2].(float64)*100, name), pieColors[n]) + "\n") 312 | } else if builder.WriteString(c); i < 13 { 313 | builder.WriteString("\n") 314 | } 315 | } 316 | i++ 317 | } 318 | } 319 | } 320 | infof(title, builder.String()) 321 | } 322 | 323 | func histf(title string, body map[string][][]interface{}) { 324 | var builder strings.Builder 325 | if len(body) > 0 { 326 | first := true 327 | for k, v := range body { 328 | if !first { 329 | builder.WriteString("\n\n\n") 330 | } else { 331 | first = false 332 | } 333 | builder.WriteString(colorf("Type: "+k, colorLightGreen) + "\n\n") 334 | var ( 335 | maxNameLen int 336 | maxCountLen int 337 | maxCount uint64 338 | ) 339 | for _, o := range v { 340 | if n := len(o[0].(string)); n > maxNameLen { 341 | maxNameLen = n 342 | } 343 | if n := len(fmt.Sprintf("%d", o[1])); n > maxCountLen { 344 | maxCountLen = n 345 | } 346 | if n := o[1].(uint64); n > maxCount { 347 | maxCount = n 348 | } 349 | } 350 | if maxNameLen > 35 { 351 | maxNameLen = 35 352 | } 353 | format := fmt.Sprintf("%%%ds [%%%dd] %%s", maxNameLen, maxCountLen) 354 | for i, o := range v { 355 | var ( 356 | n = int(math.Round(float64(o[1].(uint64)) / float64(maxCount) * 36 * 8)) 357 | bar = strings.Repeat(histChars[7], n/8) 358 | ) 359 | if n%8 > 0 { 360 | bar += histChars[n%8] 361 | } 362 | builder.WriteString(colorf(fmt.Sprintf(format, omitStr(o[0], 35), o[1], bar), colorLightWhite)) 363 | if i < len(v)-1 { 364 | builder.WriteString("\n") 365 | } 366 | } 367 | } 368 | } 369 | infof(title, builder.String()) 370 | } 371 | 372 | func withUnknown(o interface{}) string { 373 | if s := toStr(o); s != "" && !strings.EqualFold(strings.TrimSpace(s), "unknown") { 374 | return s 375 | } 376 | return "[unknown]" 377 | } 378 | 379 | func withVersion(o interface{}) string { 380 | if o == nil { 381 | return "" 382 | } 383 | if o, ok := o.([]interface{}); ok { 384 | var s string 385 | for i, v := range o { 386 | if v, ok := v.(map[string]interface{}); ok { 387 | if i > 0 { 388 | s += "," 389 | } 390 | s += v["name"].(string) 391 | if ver := v["version"]; ver != nil && ver != "" { 392 | s += "(" + ver.(string) + ")" 393 | } 394 | } 395 | } 396 | return s 397 | } 398 | return "" 399 | } 400 | 401 | func showFacet(result *zoomeye.SearchResult, facets []string, figure string) { 402 | var ( 403 | head = [][2]interface{}{ 404 | {"Type", 10}, 405 | {"Name", 35}, 406 | {"Count", 20}, 407 | } 408 | body = make(map[string][][]interface{}) 409 | ) 410 | for _, f := range facets { 411 | f = strings.ToLower(strings.TrimSpace(f)) 412 | s := f 413 | if result.Type == "host" && f == "app" { 414 | s = "product" 415 | } 416 | if facet, ok := result.Facets[s]; ok { 417 | group := make([][]interface{}, 0, len(facet)) 418 | for _, v := range facet { 419 | group = append(group, []interface{}{ 420 | withUnknown(v.Name), 421 | v.Count, 422 | float64(v.Count) / float64(result.Total), 423 | }) 424 | } 425 | body[f] = group 426 | } 427 | } 428 | switch figure { 429 | case "": 430 | tablef("ZoomEye Facets", head, body, false) 431 | case "pie": 432 | pief("ZoomEye Facets - PIE", body) 433 | case "hist": 434 | histf("ZoomEye Facets - HIST", body) 435 | } 436 | } 437 | 438 | func showStat(result *zoomeye.SearchResult, keys []string, figure string) { 439 | var ( 440 | head = [][2]interface{}{ 441 | {"Type", 10}, 442 | {"Name", 35}, 443 | {"Count", 20}, 444 | } 445 | body = make(map[string][][]interface{}) 446 | ) 447 | for s, stat := range result.Statistics(keys...) { 448 | group := make([][]interface{}, 0, len(stat)) 449 | for k, v := range stat { 450 | group = append(group, []interface{}{ 451 | k, 452 | v, 453 | float64(v) / float64(len(result.Matches)), 454 | }) 455 | } 456 | sort.Slice(group, func(i, j int) bool { 457 | return group[j][1].(uint64) < group[i][1].(uint64) 458 | }) 459 | body[s] = group 460 | } 461 | switch figure { 462 | case "": 463 | tablef("Result Statistics", head, body, false) 464 | case "pie": 465 | pief("Result Statistics - PIE", body) 466 | case "hist": 467 | histf("Result Statistics - HIST", body) 468 | } 469 | } 470 | 471 | func showFilter(result *zoomeye.SearchResult, keys []string) []map[string]interface{} { 472 | var ( 473 | filtered = result.Filter(keys...) 474 | body = make([]map[string]interface{}, len(filtered)) 475 | ) 476 | for i, filt := range filtered { 477 | var ( 478 | index = filt["_index"].(string) 479 | items = make([]map[string]interface{}, 0, len(filt)-1) 480 | ) 481 | for k, v := range filt { 482 | if k != "_index" { 483 | items = append(items, map[string]interface{}{ 484 | "key": strings.ToTitle(k), 485 | "value": v, 486 | }) 487 | } 488 | } 489 | sort.Slice(items, func(i, j int) bool { 490 | return items[i]["key"].(string) < items[j]["key"].(string) 491 | }) 492 | body[i] = map[string]interface{}{ 493 | "name": index, 494 | "items": items, 495 | } 496 | } 497 | htablef("Result Filtered", body, [3]int{30, 15, 75}, true) 498 | return filtered 499 | } 500 | 501 | func showData(result *zoomeye.SearchResult) { 502 | switch result.Type { 503 | case "host": 504 | var ( 505 | head = [][2]interface{}{ 506 | {"-", 0}, 507 | {"Host", 21}, 508 | {"Application", 20}, 509 | {"Service", 20}, 510 | {"Banner", 40}, 511 | {"Country", 20}, 512 | } 513 | body = make([][]interface{}, len(result.Matches)) 514 | ) 515 | for i, v := range result.Matches { 516 | body[i] = []interface{}{ 517 | v.FindString("ip") + ":" + v.FindString("portinfo.port"), 518 | v.FindString("portinfo.app"), 519 | v.FindString("portinfo.service"), 520 | v.FindString("portinfo.banner"), 521 | v.FindString("geoinfo.country.names.en"), 522 | } 523 | } 524 | tablef("Host Search Result", head, map[string][][]interface{}{"": body}, true) 525 | case "web": 526 | body := make([]map[string]interface{}, len(result.Matches)) 527 | for i, v := range result.Matches { 528 | body[i] = map[string]interface{}{ 529 | "name": v.FindString("site"), 530 | "items": []map[string]interface{}{ 531 | { 532 | "key": "IP", 533 | "value": v.Find("ip"), 534 | }, 535 | { 536 | "key": "Domains", 537 | "value": v.Find("domains"), 538 | }, 539 | { 540 | "key": "Country", 541 | "value": v.FindString("geoinfo.country.names.en"), 542 | }, 543 | { 544 | "key": "Title", 545 | "value": v.FindString("title"), 546 | }, 547 | { 548 | "key": "Application", 549 | "value": withVersion(v.Find("webapp")), 550 | }, 551 | { 552 | "key": "Framework", 553 | "value": withVersion(v.Find("framework")), 554 | }, 555 | { 556 | "key": "Server", 557 | "value": withVersion(v.Find("server")), 558 | }, 559 | { 560 | "key": "System", 561 | "value": withVersion(v.Find("system")), 562 | }, 563 | { 564 | "key": "Database", 565 | "value": withVersion(v.Find("db")), 566 | }, 567 | { 568 | "key": "WAF", 569 | "value": withVersion(v.Find("waf")), 570 | }, 571 | }, 572 | } 573 | } 574 | htablef("Web Search Result", body, [3]int{30, 15, 75}, true) 575 | } 576 | } 577 | 578 | func showHistory(result *zoomeye.HistoryResult, keys []string, num int) { 579 | var ( 580 | filtered = result.Filter(keys...) 581 | n = len(filtered) 582 | ) 583 | if n == 0 { 584 | infof("[History Info]", "no any historical data") 585 | return 586 | } 587 | if num > 0 && num < n { 588 | filtered = filtered[:n] 589 | } 590 | var ( 591 | first = filtered[0] 592 | info = fmt.Sprintf("%s\n\n"+ 593 | "Hostname: %s\n"+ 594 | "Country: %s\n"+ 595 | "City: %s\n"+ 596 | "Organization: %s\n"+ 597 | "Last Updated: %s\n\n", 598 | first["ip"], withUnknown(first["host"]), withUnknown(first["country"]), 599 | withUnknown(first["city"]), withUnknown(first["org"]), first["last_update"]) 600 | ) 601 | var ( 602 | head = [][2]interface{}{ 603 | {"-", 0}, 604 | {"Time", 19}, 605 | {"Port", 5}, 606 | {"Service", 25}, 607 | {"App", 25}, 608 | {"Raw", 45}, 609 | } 610 | body = make([][]interface{}, len(filtered)) 611 | ) 612 | for i := 2; i < len(head); { 613 | if _, ok := first[strings.ToLower(head[i][0].(string))]; !ok { 614 | head = append(head[:i], head[i+1:]...) 615 | } else { 616 | i++ 617 | } 618 | } 619 | if len(head) == 6 { 620 | head[3][0] = "Port/Service" 621 | head = append(head[:2], head[3:]...) 622 | } 623 | ports := make(map[string]struct{}) 624 | for i, f := range filtered { 625 | if p := toStr(f["open_port"]); p != "" { 626 | ports[p] = struct{}{} 627 | } 628 | row := make([]interface{}, 0, 5) 629 | for i := 1; i < len(head); i++ { 630 | var ( 631 | sp = strings.SplitN(head[i][0].(string), "/", 2) 632 | v = make([]string, 0, 2) 633 | ) 634 | for _, k := range sp { 635 | v = append(v, toStr(f[strings.ToLower(k)])) 636 | } 637 | row = append(row, strings.Join(v, "/")) 638 | } 639 | if row[0] == "" { 640 | row[0] = f["last_update"] 641 | } 642 | body[i] = row 643 | } 644 | info += fmt.Sprintf("Open Ports: %d\nHistorical Probes: %d", len(ports), len(filtered)) 645 | infof("History Info", info) 646 | tablef("History Result", head, map[string][][]interface{}{"": body}, true) 647 | } 648 | -------------------------------------------------------------------------------- /zoomeye/result.go: -------------------------------------------------------------------------------- 1 | package zoomeye 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | "regexp" 8 | "strings" 9 | ) 10 | 11 | var ( 12 | statisticsFields = map[string]map[string]string{ 13 | "host": { 14 | "app": "portinfo.app", 15 | "device": "portinfo.device", 16 | "service": "portinfo.service", 17 | "os": "portinfo.os", 18 | "port": "portinfo.port", 19 | "country": "geoinfo.country.names.en", 20 | "city": "geoinfo.city.names.en", 21 | }, 22 | "web": { 23 | "webapp": "webapp", 24 | "component": "component", 25 | "framework": "framework", 26 | "frontend": "frontend", 27 | "server": "server", 28 | "waf": "waf", 29 | "os": "system", 30 | "country": "geoinfo.country.names.en", 31 | "city": "geoinfo.city.names.en", 32 | }, 33 | } 34 | filterFields = map[string]map[string]string{ 35 | "host": { 36 | "_index": "ip", 37 | "app": "portinfo.app", 38 | "version": "portinfo.version", 39 | "device": "portinfo.device", 40 | "ip": "ip", 41 | "port": "portinfo.port", 42 | "hostname": "portinfo.hostname", 43 | "city": "geoinfo.city.names.en", 44 | "country": "geoinfo.country.names.en", 45 | "asn": "geoinfo.asn", 46 | "banner": "portinfo.banner", 47 | "time": "timestamp", 48 | }, 49 | "web": { 50 | "_index": "site", 51 | "app": "webapp", 52 | "headers": "headers", 53 | "keywords": "keywords", 54 | "title": "title", 55 | "ip": "ip", 56 | "site": "site", 57 | "city": "geoinfo.city.names.en", 58 | "country": "geoinfo.country.names.en", 59 | "time": "timestamp", 60 | }, 61 | } 62 | historyFilterFields = map[string]string{ 63 | "time": "timestamp", 64 | "port": "portinfo.port", 65 | "service": "portinfo.service", 66 | "app": "portinfo.product", 67 | "raw": "raw_data", 68 | } 69 | ) 70 | 71 | type findableMap map[string]interface{} 72 | 73 | func (m findableMap) Find(expr string) interface{} { 74 | var ( 75 | set = strings.Split(expr, ".") 76 | n = len(set) 77 | curr = m 78 | val interface{} 79 | ok bool 80 | ) 81 | for i, v := range set { 82 | if val, ok = curr[v]; !ok { 83 | return nil 84 | } 85 | rc := reflect.ValueOf(val) 86 | if rc.Kind() != reflect.Map { 87 | if i < n-1 { 88 | return nil 89 | } 90 | return val 91 | } 92 | if vm, ok := val.(map[string]interface{}); ok { 93 | curr = vm 94 | } else { 95 | curr = make(map[string]interface{}) 96 | for _, k := range rc.MapKeys() { 97 | curr[k.String()] = rc.MapIndex(k).Interface() 98 | } 99 | } 100 | } 101 | return val 102 | } 103 | 104 | func (m findableMap) FindString(expr string) string { 105 | switch v := m.Find(expr).(type) { 106 | case nil: 107 | return "" 108 | case string: 109 | return v 110 | case []string: 111 | return strings.Join(v, ",") 112 | case []interface{}: 113 | s := make([]string, len(v)) 114 | for i, o := range v { 115 | s[i] = fmt.Sprintf("%v", o) 116 | } 117 | return strings.Join(s, ",") 118 | default: 119 | return fmt.Sprintf("%v", v) 120 | } 121 | } 122 | 123 | // ErrorResult represents result of error 124 | type ErrorResult struct { 125 | Err string `json:"error"` 126 | Message string `json:"message"` 127 | URL string `json:"url"` 128 | } 129 | 130 | func (r *ErrorResult) Error() string { 131 | return r.Message 132 | } 133 | 134 | // Result represents each type of result 135 | type Result interface { 136 | setRawData([]byte) 137 | Raw() []byte 138 | String() string 139 | } 140 | 141 | func toString(r Result) string { 142 | b, err := json.MarshalIndent(r, "", " ") 143 | if err != nil { 144 | return "" 145 | } 146 | return string(b) 147 | } 148 | 149 | type baseResult struct { 150 | rawData []byte 151 | } 152 | 153 | func (r *baseResult) setRawData(raw []byte) { 154 | r.rawData = raw 155 | } 156 | 157 | func (r *baseResult) Raw() []byte { 158 | return r.rawData 159 | } 160 | 161 | // LoginResult represents result of login 162 | type LoginResult struct { 163 | baseResult 164 | AccessToken string `json:"access_token"` 165 | } 166 | 167 | func (r *LoginResult) String() string { 168 | return toString(r) 169 | } 170 | 171 | // ResourcesInfoResult represents result of resources information 172 | type ResourcesInfoResult struct { 173 | baseResult 174 | Plan string `json:"plan"` 175 | Resources *struct { 176 | Interval string `json:"interval"` 177 | Search int `json:"search"` 178 | Stats int `json:"stats"` 179 | } `json:"resources"` 180 | } 181 | 182 | func (r *ResourcesInfoResult) String() string { 183 | return toString(r) 184 | } 185 | 186 | // SearchResult represents result of search 187 | type SearchResult struct { 188 | baseResult 189 | Type string `json:"-"` 190 | Available uint64 `json:"available"` 191 | Total uint64 `json:"total"` 192 | Matches []findableMap `json:"matches"` 193 | Facets map[string][]*struct { 194 | Name interface{} `json:"name"` 195 | Count uint64 `json:"count"` 196 | } `json:"facets"` 197 | } 198 | 199 | // Sites finds ip and site in web search results 200 | func (r *SearchResult) Sites() []map[string]string { 201 | m := make([]map[string]string, 0, len(r.Matches)) 202 | if r.Type == "web" { 203 | for _, v := range r.Matches { 204 | if ip := v.FindString("ip"); ip != "" { 205 | m = append(m, map[string]string{ 206 | "ip": ip, 207 | "site": v.FindString("site"), 208 | }) 209 | } 210 | } 211 | } 212 | return m 213 | } 214 | 215 | // Hosts finds ip and port in host search results 216 | func (r *SearchResult) Hosts() []map[string]string { 217 | m := make([]map[string]string, 0, len(r.Matches)) 218 | if r.Type == "host" { 219 | for _, v := range r.Matches { 220 | if ip := v.FindString("ip"); ip != "" { 221 | m = append(m, map[string]string{ 222 | "ip": ip, 223 | "port": v.FindString("portinfo.port"), 224 | }) 225 | } 226 | } 227 | } 228 | return m 229 | } 230 | 231 | // Statistics counts data by specified fields from search results 232 | func (r *SearchResult) Statistics(keys ...string) map[string]map[string]uint64 { 233 | var ( 234 | counts = make(map[string]map[string]uint64) 235 | fields, ok = statisticsFields[r.Type] 236 | ) 237 | if !ok { 238 | return counts 239 | } 240 | for _, k := range keys { 241 | k = strings.ToLower(strings.TrimSpace(k)) 242 | field, ok := fields[k] 243 | if !ok { 244 | continue 245 | } 246 | if _, ok := counts[k]; !ok { 247 | counts[k] = make(map[string]uint64) 248 | } 249 | for _, v := range r.Matches { 250 | switch s := v.Find(field).(type) { 251 | case string: 252 | if s != "" { 253 | counts[k][s]++ 254 | continue 255 | } 256 | case []interface{}: 257 | if len(s) > 0 { 258 | for _, sv := range s { 259 | if sv, ok := sv.(map[string]interface{}); ok { 260 | name, ok := sv["name"] 261 | if !ok || name == nil { 262 | name = "[unknown]" 263 | } 264 | counts[k][fmt.Sprintf("%v", name)]++ 265 | } else { 266 | counts[k]["[unknown]"]++ 267 | } 268 | } 269 | continue 270 | } 271 | } 272 | counts[k]["[unknown]"]++ 273 | } 274 | } 275 | return counts 276 | } 277 | 278 | // Filter extracts data by specified fields from search results 279 | func (r *SearchResult) Filter(keys ...string) []map[string]interface{} { 280 | var ( 281 | filtered = make([]map[string]interface{}, 0, len(r.Matches)) 282 | n = len(keys) 283 | ) 284 | if n == 0 { 285 | return filtered 286 | } 287 | fields, ok := filterFields[r.Type] 288 | if !ok { 289 | return filtered 290 | } 291 | if n == 1 && (keys[0] == "" || keys[0] == "*") { 292 | keys = make([]string, 0, len(fields)) 293 | for k := range fields { 294 | keys = append(keys, k) 295 | } 296 | } else { 297 | keys = append(keys, "_index") 298 | } 299 | for _, v := range r.Matches { 300 | var ( 301 | item = make(map[string]interface{}) 302 | count int 303 | notMatch bool 304 | ) 305 | for _, k := range keys { 306 | var expr string 307 | if kv := strings.SplitN(k, "=", 2); len(kv) == 2 { 308 | k = kv[0] 309 | if expr = strings.TrimSpace(kv[1]); !strings.HasPrefix(expr, "(?i)") { 310 | expr = "(?i)" + expr 311 | } 312 | } 313 | k = strings.ToLower(strings.TrimSpace(k)) 314 | field, ok := fields[k] 315 | if !ok { 316 | continue 317 | } 318 | if _, ok = item[k]; ok { 319 | continue 320 | } 321 | find := v.Find(field) 322 | if k != "_index" && find != nil { 323 | if fv := fmt.Sprintf("%v", find); expr == "" { 324 | count++ 325 | } else if reg, err := regexp.Compile(expr); err == nil && reg.MatchString(fv) { 326 | count++ 327 | } else { 328 | notMatch = true 329 | break 330 | } 331 | } 332 | item[k] = find 333 | } 334 | if !notMatch && count > 0 { 335 | filtered = append(filtered, item) 336 | } 337 | } 338 | return filtered 339 | } 340 | 341 | // Extend merges more than one search results 342 | func (r *SearchResult) Extend(res *SearchResult) { 343 | if res == nil { 344 | return 345 | } 346 | if res.Type != "" { 347 | if r.Type == "" { 348 | r.Type = res.Type 349 | } else if r.Type != res.Type { 350 | return 351 | } 352 | } 353 | if n := len(r.rawData); n == 0 { 354 | r.rawData = res.rawData 355 | } else if r.rawData[0] == '[' && r.rawData[n-1] == ']' { 356 | r.rawData = append(r.rawData[:n-1], make([]byte, len(res.rawData)+3)...) 357 | copy(r.rawData[n-1:], []byte(", ")) 358 | copy(r.rawData[n+1:], res.rawData) 359 | r.rawData[len(r.rawData)-1] = ']' 360 | } else { 361 | raw := r.rawData 362 | r.rawData = make([]byte, len(res.rawData)+n+4) 363 | r.rawData[0] = '[' 364 | copy(r.rawData[1:], raw) 365 | copy(r.rawData[n+1:], []byte(", ")) 366 | copy(r.rawData[n+3:], res.rawData) 367 | r.rawData[len(r.rawData)-1] = ']' 368 | } 369 | if res.Total > 0 { 370 | r.Available = res.Available 371 | r.Total = res.Total 372 | r.Matches = append(r.Matches, res.Matches...) 373 | r.Facets = res.Facets 374 | } 375 | } 376 | 377 | func (r *SearchResult) String() string { 378 | return toString(r) 379 | } 380 | 381 | // HistoryResult represents result of history 382 | type HistoryResult struct { 383 | baseResult 384 | Count uint64 `json:"count"` 385 | Data []findableMap `json:"data"` 386 | } 387 | 388 | // Filter extracts data by specified fields from query results 389 | func (r *HistoryResult) Filter(keys ...string) []map[string]interface{} { 390 | if n := len(keys); n == 0 || (n == 1 && (keys[0] == "" || keys[0] == "*")) { 391 | keys = make([]string, 0, len(historyFilterFields)) 392 | for k := range historyFilterFields { 393 | keys = append(keys, k) 394 | } 395 | } 396 | filtered := make([]map[string]interface{}, 0, len(r.Data)) 397 | for _, v := range r.Data { 398 | var ( 399 | item = make(map[string]interface{}) 400 | count int 401 | notMatch bool 402 | ) 403 | for _, k := range keys { 404 | var expr string 405 | if kv := strings.SplitN(k, "=", 2); len(kv) == 2 { 406 | k = kv[0] 407 | if expr = strings.TrimSpace(kv[1]); !strings.HasPrefix(expr, "(?i)") { 408 | expr = "(?i)" + expr 409 | } 410 | } 411 | k = strings.ToLower(strings.TrimSpace(k)) 412 | field, ok := historyFilterFields[k] 413 | if !ok { 414 | continue 415 | } 416 | find := v.Find(field) 417 | if find != nil { 418 | if fv := fmt.Sprintf("%v", find); expr == "" { 419 | count++ 420 | } else if reg, err := regexp.Compile(expr); err == nil && reg.MatchString(fv) { 421 | count++ 422 | } else { 423 | notMatch = true 424 | break 425 | } 426 | } 427 | item[k] = find 428 | } 429 | if !notMatch && count > 0 { 430 | item["ip"] = v.Find("ip") 431 | item["open_port"] = v.Find("portinfo.port") 432 | item["host"] = v.Find("portinfo.hostname") 433 | item["country"] = v.Find("geoinfo.country.names.en") 434 | item["city"] = v.Find("geoinfo.city.names.en") 435 | item["org"] = v.Find("geoinfo.organization") 436 | item["last_update"] = v.Find("timestamp") 437 | filtered = append(filtered, item) 438 | } 439 | } 440 | return filtered 441 | } 442 | 443 | func (r *HistoryResult) String() string { 444 | return toString(r) 445 | } 446 | -------------------------------------------------------------------------------- /zoomeye/zoomeye.go: -------------------------------------------------------------------------------- 1 | package zoomeye 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/tls" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "net/url" 13 | "strings" 14 | "sync" 15 | "sync/atomic" 16 | "time" 17 | ) 18 | 19 | const ( 20 | loginAPI = "https://api.zoomeye.org/user/login" 21 | userinfoAPI = "https://api.zoomeye.org/resources-info" 22 | searchAPI = "https://api.zoomeye.org/%s/search" 23 | historyAPI = "https://api.zoomeye.org/both/search?history=true&ip=%s" 24 | ) 25 | 26 | var httpCli = &http.Client{ 27 | Timeout: 30 * time.Second, 28 | Transport: &http.Transport{ 29 | TLSClientConfig: &tls.Config{ 30 | InsecureSkipVerify: true, 31 | }, 32 | }, 33 | } 34 | 35 | var defaultFacets = map[string]string{ 36 | "host": "app,device,service,os,port,country,city", 37 | "web": "webapp,component,framework,frontend,server,waf,os,country,city", 38 | } 39 | 40 | // ZoomEye represents SDK for using 41 | type ZoomEye struct { 42 | apiKey string 43 | accessToken string 44 | } 45 | 46 | func (z *ZoomEye) request(method, u string, body io.Reader, result Result) error { 47 | req, err := http.NewRequest(method, u, body) 48 | if z.apiKey != "" { 49 | req.Header.Set("API-KEY", z.apiKey) 50 | } 51 | if z.accessToken != "" { 52 | req.Header.Set("Authorization", "JWT "+z.accessToken) 53 | } 54 | resp, err := httpCli.Do(req) 55 | if err != nil { 56 | return err 57 | } 58 | b, err := ioutil.ReadAll(resp.Body) 59 | if resp.Body.Close(); err != nil { 60 | return err 61 | } 62 | if resp.StatusCode == 200 { 63 | if err = json.Unmarshal(b, result); err != nil { 64 | return err 65 | } 66 | result.setRawData(b) 67 | return nil 68 | } 69 | if resp.StatusCode == 403 && bytes.Contains(b, []byte("specified resource")) { 70 | return nil 71 | } 72 | e := &ErrorResult{} 73 | if err = json.Unmarshal(b, &e); err != nil { 74 | return err 75 | } 76 | return e 77 | } 78 | 79 | func (z *ZoomEye) get(u string, params map[string]interface{}, result Result) error { 80 | if params != nil { 81 | uu, err := url.Parse(u) 82 | if err != nil { 83 | return err 84 | } 85 | query := uu.Query() 86 | for k, v := range params { 87 | query.Add(k, fmt.Sprintf("%v", v)) 88 | } 89 | uu.RawQuery = query.Encode() 90 | u = uu.String() 91 | } 92 | return z.request(http.MethodGet, u, nil, result) 93 | } 94 | 95 | func (z *ZoomEye) post(u string, headers map[string]string, data map[string]interface{}, result Result) error { 96 | var body io.Reader 97 | if data != nil { 98 | b, err := json.Marshal(data) 99 | if err != nil { 100 | return err 101 | } 102 | body = bytes.NewBuffer(b) 103 | } 104 | return z.request(http.MethodPost, u, body, result) 105 | } 106 | 107 | // Login uses username/password for authentication 108 | func (z *ZoomEye) Login(username, password string) (string, error) { 109 | var ( 110 | data = map[string]interface{}{ 111 | "username": username, 112 | "password": password, 113 | } 114 | result = &LoginResult{} 115 | ) 116 | if err := z.post(loginAPI, nil, data, result); err != nil { 117 | return "", err 118 | } 119 | z.accessToken = result.AccessToken 120 | return z.accessToken, nil 121 | } 122 | 123 | // ResourcesInfo gets account resource information 124 | func (z *ZoomEye) ResourcesInfo() (*ResourcesInfoResult, error) { 125 | var ( 126 | result = &ResourcesInfoResult{} 127 | err = z.get(userinfoAPI, nil, result) 128 | ) 129 | if err != nil { 130 | return nil, err 131 | } 132 | return result, nil 133 | } 134 | 135 | // DorkSearch searches the data of the specified page according to dork 136 | func (z *ZoomEye) DorkSearch(dork string, page int, resource string, facet string) (*SearchResult, error) { 137 | if page <= 0 { 138 | page = 1 139 | } 140 | if resource = strings.ToLower(resource); resource != "web" { 141 | resource = "host" 142 | } 143 | if facet == "" { 144 | facet = defaultFacets[resource] 145 | } 146 | var ( 147 | params = map[string]interface{}{ 148 | "query": dork, 149 | "page": page, 150 | "facets": facet, 151 | } 152 | result = &SearchResult{ 153 | Type: resource, 154 | } 155 | err = z.get(fmt.Sprintf(searchAPI, resource), params, result) 156 | ) 157 | if err != nil { 158 | return nil, err 159 | } 160 | if len(result.Matches) == 0 { 161 | return nil, fmt.Errorf("no any results for the dork") 162 | } 163 | return result, nil 164 | } 165 | 166 | func (z *ZoomEye) conMPSearch(dork string, maxPage int, resource string, facet string) (map[int]*SearchResult, error) { 167 | var ( 168 | results = make(map[int]*SearchResult) 169 | ch = make(chan map[string]interface{}, maxPage-1) 170 | ctx, cancel = context.WithCancel(context.Background()) 171 | ) 172 | defer close(ch) 173 | defer cancel() 174 | var ( 175 | wg sync.WaitGroup 176 | groupSize = 20 177 | ) 178 | if maxPage < 21 { 179 | groupSize = maxPage - 1 180 | } 181 | var currPage int32 = 1 182 | for i := 0; i < groupSize; i++ { 183 | wg.Add(1) 184 | go func() { 185 | defer wg.Done() 186 | for { 187 | select { 188 | case <-ctx.Done(): 189 | return 190 | default: 191 | page := int(atomic.AddInt32(&currPage, 1)) 192 | if page > maxPage { 193 | return 194 | } 195 | m := map[string]interface{}{ 196 | "page": page, 197 | } 198 | if res, err := z.DorkSearch(dork, page, resource, facet); err != nil { 199 | m["result"] = err 200 | } else { 201 | m["result"] = res 202 | } 203 | ch <- m 204 | } 205 | } 206 | }() 207 | } 208 | var ( 209 | err error 210 | waiting bool 211 | done = make(chan struct{}, 1) 212 | ) 213 | RCPT_LOOP: 214 | for i := 0; i < groupSize; i++ { 215 | select { 216 | case <-ctx.Done(): 217 | if !waiting { 218 | waiting = true 219 | go func() { 220 | wg.Wait() 221 | close(done) 222 | }() 223 | } 224 | case <-done: 225 | break RCPT_LOOP 226 | case c := <-ch: 227 | switch res := c["result"].(type) { 228 | case error: 229 | cancel() 230 | err = res 231 | case *SearchResult: 232 | results[c["page"].(int)] = res 233 | } 234 | } 235 | } 236 | if wg.Wait(); err != nil && len(results) == 0 { 237 | return nil, err 238 | } 239 | return results, nil 240 | } 241 | 242 | // MultiPageSearch searches multiple pages of data according to dork 243 | func (z *ZoomEye) MultiPageSearch(dork string, maxPage int, resource string, facet string) (map[int]*SearchResult, error) { 244 | if maxPage <= 0 { 245 | maxPage = 1 246 | } 247 | info, err := z.ResourcesInfo() 248 | if err != nil { 249 | return nil, err 250 | } 251 | allowPage := info.Resources.Search / 20 252 | if info.Resources.Search%20 > 0 { 253 | allowPage++ 254 | } 255 | results := make(map[int]*SearchResult) 256 | if allowPage > 0 { 257 | res, err := z.DorkSearch(dork, 1, resource, facet) 258 | if err != nil { 259 | return nil, err 260 | } 261 | results[1] = res 262 | n := int(res.Total / 20) 263 | if res.Total%20 > 0 { 264 | n++ 265 | } 266 | if n < allowPage { 267 | allowPage = n 268 | } 269 | } 270 | if maxPage > allowPage { 271 | maxPage = allowPage 272 | } 273 | if maxPage > 5 { 274 | corResults, err := z.conMPSearch(dork, maxPage, resource, facet) 275 | if err != nil { 276 | if len(results) > 0 { 277 | return results, nil 278 | } 279 | return nil, err 280 | } 281 | for k, v := range results { 282 | if _, ok := corResults[k]; !ok { 283 | corResults[k] = v 284 | } 285 | } 286 | return corResults, nil 287 | } 288 | for i := 1; i < maxPage; i++ { 289 | var ( 290 | page = i + 1 291 | res *SearchResult 292 | ) 293 | if res, err = z.DorkSearch(dork, page, resource, facet); err != nil { 294 | break 295 | } 296 | results[page] = res 297 | } 298 | if err != nil && len(results) == 0 { 299 | return nil, err 300 | } 301 | return results, nil 302 | } 303 | 304 | // MultiToOneSearch searches multiple pages of data according to dork, and merges all results 305 | func (z *ZoomEye) MultiToOneSearch(dork string, maxPage int, resource string, facet string) (*SearchResult, error) { 306 | results, err := z.MultiPageSearch(dork, maxPage, resource, facet) 307 | if err != nil { 308 | return nil, err 309 | } 310 | result := &SearchResult{ 311 | Type: resource, 312 | } 313 | for i, n := 0, len(results); i < maxPage && n > 0; i++ { 314 | if res, ok := results[i+1]; ok { 315 | result.Extend(res) 316 | n-- 317 | } 318 | } 319 | return result, nil 320 | } 321 | 322 | // HistoryIP queries IP history information 323 | func (z *ZoomEye) HistoryIP(ip string) (*HistoryResult, error) { 324 | var ( 325 | result = &HistoryResult{} 326 | err = z.get(fmt.Sprintf(historyAPI, ip), nil, result) 327 | ) 328 | if err != nil { 329 | return nil, err 330 | } 331 | return result, nil 332 | } 333 | 334 | // NewWithKey creates instance of ZoomEye with API-Key and AccessToken 335 | func NewWithKey(apiKey, accessToken string) *ZoomEye { 336 | return &ZoomEye{ 337 | apiKey: apiKey, 338 | accessToken: accessToken, 339 | } 340 | } 341 | 342 | // New creates instance of ZoomEye 343 | func New() *ZoomEye { 344 | return &ZoomEye{} 345 | } 346 | -------------------------------------------------------------------------------- /zoomeye/zoomeye_test.go: -------------------------------------------------------------------------------- 1 | package zoomeye 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | const ( 8 | tUsername = "username@zoomeye.org" 9 | tPassword = "password" 10 | tAPIKey = "XXXXXXXX-XXXX-XXXXX-XXXX-XXXXXXXXXXX" 11 | ) 12 | 13 | var defaultZoom = NewWithKey(tAPIKey, "") 14 | 15 | func TestLogin(t *testing.T) { 16 | zoom := New() 17 | tok, err := zoom.Login(tUsername, tPassword) 18 | if err != nil || (tok != zoom.accessToken) { 19 | t.Fail() 20 | } 21 | if _, err = New().Login("test", "123456"); err == nil { 22 | t.Fail() 23 | } 24 | } 25 | 26 | func TestResourcesInfo(t *testing.T) { 27 | result, err := defaultZoom.ResourcesInfo() 28 | if err != nil || (result.Plan == "") { 29 | t.Fail() 30 | } 31 | if _, err = NewWithKey("00000000-0000-00000-0000-00000000000", "").ResourcesInfo(); err == nil { 32 | t.Fail() 33 | } 34 | } 35 | 36 | func TestDorkSearch(t *testing.T) { 37 | result, err := defaultZoom.DorkSearch("solr", 0, "", "") 38 | if err != nil { 39 | t.FailNow() 40 | } 41 | t.Log(result.Hosts()) 42 | if _, err = defaultZoom.DorkSearch("country:cn", 0, "", ""); err != nil { 43 | t.Fail() 44 | } 45 | if _, err = defaultZoom.DorkSearch("solr country:cn", 0, "", "os,country"); err != nil { 46 | t.Fail() 47 | } 48 | if result, err = defaultZoom.DorkSearch("solr country:cn", 0, "web", ""); err != nil { 49 | t.FailNow() 50 | } 51 | t.Log(result.Sites()) 52 | if n := len(result.Matches); n > 0 { 53 | t.Log(result.Matches[n-1].FindString("geoinfo.country.names.zh-CN")) 54 | } 55 | } 56 | 57 | func TestMultiPageSearch(t *testing.T) { 58 | var ( 59 | maxPage = 2 60 | results, err = defaultZoom.MultiPageSearch("dedecms country:cn", maxPage, "web", "") 61 | ) 62 | if err != nil || (len(results) == 0) { 63 | t.FailNow() 64 | } 65 | t.Log(results[1].Total, results[1].Type, len(results)) 66 | } 67 | 68 | func TestMultiToOneSearch(t *testing.T) { 69 | var ( 70 | maxPage = 2 71 | result, err = defaultZoom.MultiToOneSearch("dedecms country:cn", maxPage, "web", "") 72 | ) 73 | if err != nil || (len(result.Matches) != maxPage*20) { 74 | t.FailNow() 75 | } 76 | t.Log(result.Total, result.Type, len(result.Matches)) 77 | } 78 | 79 | func TestFilter(t *testing.T) { 80 | result, err := defaultZoom.DorkSearch("port:21", 0, "host", "") 81 | if err != nil { 82 | t.Fail() 83 | } else { 84 | t.Log(result.Filter("app")) 85 | } 86 | if result, err = defaultZoom.DorkSearch("dedecms", 0, "web", ""); err != nil { 87 | t.FailNow() 88 | } 89 | t.Log(result.Filter("site", "ip", "country")) 90 | } 91 | 92 | func TestHistoryIP(t *testing.T) { 93 | result, err := defaultZoom.HistoryIP("1.2.3.4") 94 | if err != nil { 95 | t.FailNow() 96 | } 97 | t.Log(result) 98 | t.Log(result.Filter("time=^2016", "app")) 99 | } 100 | --------------------------------------------------------------------------------