├── .gitignore ├── LICENSE ├── README.MD ├── common.go ├── go.mod ├── parse.go ├── parse_test.go ├── struct.go └── technologies.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/**/usage.statistics.xml 10 | .idea/**/dictionaries 11 | .idea/**/shelf 12 | 13 | # Generated files 14 | .idea/**/contentModel.xml 15 | 16 | # Sensitive or high-churn files 17 | .idea/**/dataSources/ 18 | .idea/**/dataSources.ids 19 | .idea/**/dataSources.local.xml 20 | .idea/**/sqlDataSources.xml 21 | .idea/**/dynamic.xml 22 | .idea/**/uiDesigner.xml 23 | .idea/**/dbnavigator.xml 24 | 25 | # Gradle 26 | .idea/**/gradle.xml 27 | .idea/**/libraries 28 | 29 | # Gradle and Maven with auto-import 30 | # When using Gradle or Maven with auto-import, you should exclude module files, 31 | # since they will be recreated, and may cause churn. Uncomment if using 32 | # auto-import. 33 | # .idea/artifacts 34 | # .idea/compiler.xml 35 | # .idea/jarRepositories.xml 36 | # .idea/modules.xml 37 | # .idea/*.iml 38 | # .idea/modules 39 | # *.iml 40 | # *.ipr 41 | 42 | # CMake 43 | cmake-build-*/ 44 | 45 | # Mongo Explorer plugin 46 | .idea/**/mongoSettings.xml 47 | 48 | # File-based project format 49 | *.iws 50 | 51 | # IntelliJ 52 | out/ 53 | 54 | # mpeltonen/sbt-idea plugin 55 | .idea_modules/ 56 | 57 | # JIRA plugin 58 | atlassian-ide-plugin.xml 59 | 60 | # Cursive Clojure plugin 61 | .idea/replstate.xml 62 | 63 | # Crashlytics plugin (for Android Studio and IntelliJ) 64 | com_crashlytics_export_strings.xml 65 | crashlytics.properties 66 | crashlytics-build.properties 67 | fabric.properties 68 | 69 | # Editor-based Rest Client 70 | .idea/httpRequests 71 | 72 | # Android studio 3.1+ serialized cache file 73 | .idea/caches/build_file_checksums.ser 74 | 75 | ### Example user template template 76 | ### Example user template 77 | 78 | # IntelliJ project files 79 | .idea 80 | *.iml 81 | out 82 | gen 83 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # WappalyzerParse 2 | ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/iiiusky/WappalyzerParse) 3 | ![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/iiiusky/WappalyzerParse) 4 | ![GitHub](https://img.shields.io/github/license/iiiusky/WappalyzerParse) 5 | ![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/iiiusky/WappalyzerParse) 6 | ![GitHub top language](https://img.shields.io/github/languages/top/iiiusky/WappalyzerParse) 7 | 8 | ## 描述 9 | > 该项目是我在小工具中使用到了[Wappalyzer](https://github.com/AliasIO/wappalyzer)的指纹库,并将其抽取出来作为一个简单的指纹识别库. 10 | > 当前支持直接解析源`technologies.json`文件,并且输出结构体或者json字符串信息,方便后续的使用. 11 | 12 | ## 使用方式 13 | ``` 14 | go get github.com/iiiusky/WappalyzerParse 15 | ``` 16 | 17 | ### 直接获取指纹库结构体 18 | 19 | ```go 20 | w := &WappalyzerParse{} 21 | w.InitFingerPrintData() 22 | fmt.Println(w.Fingerprints) 23 | ``` 24 | 25 | ### 输出json字符串 26 | ```go 27 | w := &WappalyzerParse{} 28 | w.InitFingerPrintData() 29 | str := w.FormatJson() 30 | fmt.Println(str) 31 | ``` 32 | 33 | ### 自定义文件路径 34 | ```go 35 | w := &WappalyzerParse{ 36 | Filename: "xxx.json", 37 | } 38 | w.InitFingerPrintData() 39 | fmt.Println(w.Fingerprints) 40 | ``` 41 | 42 | ### 开启错误信息输出 43 | 44 | ```go 45 | w := &WappalyzerParse{ 46 | IsDebug: true, 47 | } 48 | w.InitFingerPrintData() 49 | fmt.Println(w.Fingerprints) 50 | ``` 51 | 52 | ## todo 53 | - 支持解析 54 | - 兼容后续的官方指纹库信息 55 | -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 iiusky sky@03sec.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package WappalyzerParse 18 | 19 | import ( 20 | "encoding/json" 21 | "reflect" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | ) 26 | 27 | // 指纹类别信息抽取 28 | func categoriesExtractor(data interface{}) map[string]FingerprintCategories { 29 | var cats map[string]FingerprintCategories 30 | 31 | marshal, _ := json.Marshal(data) 32 | _ = json.Unmarshal(marshal, &cats) 33 | 34 | return cats 35 | } 36 | 37 | // 技术指纹信息抽取 38 | func technologiesExtractor(data interface{}) map[string]interface{} { 39 | var fingerprints map[string]interface{} 40 | 41 | marshal, _ := json.Marshal(data) 42 | _ = json.Unmarshal(marshal, &fingerprints) 43 | 44 | return fingerprints 45 | } 46 | 47 | // 结果输出 48 | func outputData(fingerprints map[string]interface{}, cats map[string]FingerprintCategories) []Fingerprint { 49 | var results []Fingerprint 50 | 51 | for name, valueInterface := range fingerprints { 52 | // 初始化默认值，不然json序列化出来的的空数据为null。 53 | _name := name 54 | _slug := slugify(name) 55 | _url := "" 56 | _description := "" 57 | _categories := make([]FingerprintCategories, 0) 58 | _headers := make(map[string]string) 59 | _dns := make(map[string][]string) 60 | _cookies := make(map[string]string) 61 | _dom := []DomInfo{} 62 | _html := []string{} 63 | _css := []string{} 64 | _certIssuer := "" 65 | _robots := "" 66 | _meta := make(map[string]string) 67 | _scripts := []string{} 68 | _js := make(map[string]string) 69 | _implies := []string{} 70 | _excludes := []string{} 71 | _icon := "default.svg" 72 | _website := "" 73 | _cpe := "" 74 | 75 | valueMap := valueInterface.(map[string]interface{}) 76 | for mapKey, mapValueInterface := range valueMap { 77 | // 根据不同的key进行不同的处理 78 | switch mapKey { 79 | case "url": 80 | _url = mapValueInterface.(string) 81 | break 82 | case "description": 83 | _description = mapValueInterface.(string) 84 | break 85 | case "cats": 86 | for _, interfaceValue := range mapValueInterface.([]interface{}) { 87 | _categories = append(_categories, cats[strconv.Itoa(int(interfaceValue.(float64)))]) 88 | } 89 | break 90 | case "headers": 91 | for interfaceKey, interfaceValue := range mapValueInterface.(map[string]interface{}) { 92 | _headers[interfaceKey] = interfaceValue.(string) 93 | } 94 | break 95 | case "dns": 96 | for interfaceKey, interfaceValue := range mapValueInterface.(map[string]interface{}) { 97 | if reflect.TypeOf(interfaceValue).Kind() == reflect.String { 98 | _dns[interfaceKey] = append(_dns[interfaceKey], interfaceValue.(string)) 99 | } else { 100 | for _, _interfaceValue := range interfaceValue.([]interface{}) { 101 | _dns[interfaceKey] = append(_dns[interfaceKey], _interfaceValue.(string)) 102 | } 103 | } 104 | } 105 | break 106 | case "cookies": 107 | for interfaceKey, interfaceValue := range mapValueInterface.(map[string]interface{}) { 108 | _cookies[interfaceKey] = interfaceValue.(string) 109 | } 110 | break 111 | case "dom": 112 | for key, domInfo := range mapValueInterface.(map[string]interface{}) { 113 | for attr, info := range domInfo.(map[string]interface{}) { 114 | for label, rule := range info.(map[string]interface{}) { 115 | _dom = append(_dom, DomInfo{ 116 | Key: key, 117 | Attr: attr, 118 | Label: label, 119 | Rule: rule.(string), 120 | }) 121 | } 122 | } 123 | } 124 | break 125 | case "html": 126 | if reflect.TypeOf(mapValueInterface).Kind() == reflect.String { 127 | _html = append(_html, mapValueInterface.(string)) 128 | } else { 129 | for _, interfaceValue := range mapValueInterface.([]interface{}) { 130 | _html = append(_html, interfaceValue.(string)) 131 | } 132 | } 133 | break 134 | case "css": 135 | if reflect.TypeOf(mapValueInterface).Kind() == reflect.String { 136 | _css = append(_css, mapValueInterface.(string)) 137 | } else { 138 | for _, interfaceValue := range mapValueInterface.([]interface{}) { 139 | _css = append(_css, interfaceValue.(string)) 140 | } 141 | } 142 | break 143 | case "cert_issuer": 144 | _certIssuer = mapValueInterface.(string) 145 | break 146 | case "robots": 147 | // todo: 不知道robots的结构是什么样的,暂时空置. 148 | break 149 | case "meta": 150 | for interfaceKey, interfaceValue := range mapValueInterface.(map[string]interface{}) { 151 | _meta[interfaceKey] = interfaceValue.(string) 152 | } 153 | break 154 | case "scripts": 155 | if reflect.TypeOf(mapValueInterface).Kind() == reflect.String { 156 | _scripts = append(_scripts, mapValueInterface.(string)) 157 | } else { 158 | for _, interfaceValue := range mapValueInterface.([]interface{}) { 159 | _scripts = append(_scripts, interfaceValue.(string)) 160 | } 161 | } 162 | break 163 | case "js": 164 | for interfaceKey, interfaceValue := range mapValueInterface.(map[string]interface{}) { 165 | _js[interfaceKey] = interfaceValue.(string) 166 | } 167 | break 168 | case "implies": 169 | if reflect.TypeOf(mapValueInterface).Kind() == reflect.String { 170 | _implies = append(_implies, mapValueInterface.(string)) 171 | } else { 172 | for _, interfaceValue := range mapValueInterface.([]interface{}) { 173 | _implies = append(_implies, interfaceValue.(string)) 174 | } 175 | } 176 | break 177 | case "excludes": 178 | if reflect.TypeOf(mapValueInterface).Kind() == reflect.String { 179 | _excludes = append(_excludes, mapValueInterface.(string)) 180 | } else { 181 | for _, interfaceValue := range mapValueInterface.([]interface{}) { 182 | _excludes = append(_excludes, interfaceValue.(string)) 183 | } 184 | } 185 | break 186 | case "icon": 187 | _icon = mapValueInterface.(string) 188 | break 189 | case "website": 190 | _website = mapValueInterface.(string) 191 | break 192 | case "cpe": 193 | _cpe = mapValueInterface.(string) 194 | break 195 | } 196 | 197 | } 198 | 199 | // 将赋值后的key填充至fingerprint,并将其追加至results列表中 200 | results = append(results, Fingerprint{ 201 | Name: _name, 202 | Slug: _slug, 203 | URL: _url, 204 | Description: _description, 205 | Categories: _categories, 206 | Headers: _headers, 207 | DNS: _dns, 208 | Cookies: _cookies, 209 | Dom: _dom, 210 | HTML: _html, 211 | Css: _css, 212 | CertIssuer: _certIssuer, 213 | Robots: _robots, 214 | Meta: _meta, 215 | Scripts: _scripts, 216 | Js: _js, 217 | Implies: _implies, 218 | Excludes: _excludes, 219 | Icon: _icon, 220 | Website: _website, 221 | CPE: _cpe, 222 | }) 223 | } 224 | 225 | return results 226 | } 227 | 228 | func slugify(slug string) string { 229 | slug = strings.ToLower(slug) 230 | slug = regexp.MustCompile(`[^a-z0-9-]`).ReplaceAllString(slug, "-") 231 | slug = regexp.MustCompile(`--+`).ReplaceAllString(slug, "-") 232 | slug = regexp.MustCompile(`(?:^-|-$)`).ReplaceAllString(slug, "") 233 | 234 | return slug 235 | } 236 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/iiiusky/WappalyzerParse 2 | 3 | go 1.14 4 | -------------------------------------------------------------------------------- /parse.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 iiusky sky@03sec.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package WappalyzerParse 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "fmt" 23 | "io/ioutil" 24 | ) 25 | 26 | type WappalyzerParse struct { 27 | Filename string // 指纹库路径,如果不指定,则使用默认内置路径 28 | IsDebug bool // 开启后将直接打印出错误信息 29 | Fingerprints []Fingerprint // 指纹库结果 30 | } 31 | 32 | // 获取wappalyzer指纹库的结果,返回包装好的结构体类型 33 | func (w *WappalyzerParse) InitFingerPrintData() { 34 | var dataJson map[string]interface{} 35 | var dataByte []byte 36 | var err error 37 | 38 | if w.Filename == "" { 39 | w.Filename = "technologies.json" 40 | } 41 | 42 | dataByte, err = ioutil.ReadFile(w.Filename) 43 | if err != nil { 44 | if w.IsDebug { 45 | fmt.Printf("open file err: %v \n", err) 46 | } 47 | return 48 | } 49 | 50 | err = json.Unmarshal(dataByte, &dataJson) 51 | if err != nil { 52 | if w.IsDebug { 53 | fmt.Printf("json unmarshal err: %v \n", err) 54 | } 55 | return 56 | } 57 | 58 | categories := categoriesExtractor(dataJson["categories"]) 59 | technologies := technologiesExtractor(dataJson["technologies"]) 60 | 61 | w.Fingerprints = outputData(technologies, categories) 62 | } 63 | 64 | // 输出格式转为json,涉及到json序列化后html标签实体化问题,通过自定义json.encoder解决 65 | func (w *WappalyzerParse) FormatJson() string { 66 | bf := bytes.NewBuffer([]byte{}) 67 | 68 | jsonEncoder := json.NewEncoder(bf) 69 | jsonEncoder.SetEscapeHTML(false) 70 | jsonEncoder.SetIndent("", " ") 71 | _ = jsonEncoder.Encode(w.Fingerprints) 72 | 73 | return bf.String() 74 | } 75 | -------------------------------------------------------------------------------- /parse_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 iiusky sky@03sec.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package WappalyzerParse 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestWappalyzerParse_GetFingerPrintData(t *testing.T) { 24 | type fields struct { 25 | Filename string 26 | IsDebug bool 27 | FormatJson bool 28 | } 29 | tests := []struct { 30 | name string 31 | fields fields 32 | want int 33 | }{ 34 | { 35 | "Test Open Debug Mode", 36 | fields{ 37 | Filename: "xxxx.json", 38 | IsDebug: true, 39 | FormatJson: false, 40 | }, 41 | -1, 42 | }, 43 | { 44 | "Test Success", 45 | fields{ 46 | }, 47 | 1, 48 | }, 49 | { 50 | "Test Json", 51 | fields{ 52 | FormatJson: true, 53 | }, 54 | 1, 55 | }, 56 | { 57 | "Test External File", 58 | fields{ 59 | Filename: "technologies.json", 60 | }, 61 | 1, 62 | }, 63 | } 64 | 65 | for _, tt := range tests { 66 | t.Run(tt.name, func(t *testing.T) { 67 | w := &WappalyzerParse{ 68 | Filename: tt.fields.Filename, 69 | IsDebug: tt.fields.IsDebug, 70 | } 71 | w.InitFingerPrintData() 72 | 73 | if got := len(w.Fingerprints); got <= tt.want { 74 | t.Errorf("Fingerprints length = %v, want <= %v", got, tt.want) 75 | } 76 | }) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /struct.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 iiusky sky@03sec.com 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package WappalyzerParse 18 | 19 | // 指纹结构体 20 | type Fingerprint struct { 21 | Name string `json:"name"` 22 | Slug string `json:"slug"` 23 | URL string `json:"url"` 24 | Description string `json:"description"` 25 | Categories []FingerprintCategories `json:"categories"` 26 | Headers map[string]string `json:"headers"` 27 | DNS map[string][]string `json:"dns"` 28 | Cookies map[string]string `json:"cookies"` 29 | Dom []DomInfo `json:"dom"` 30 | HTML []string `json:"html"` 31 | Css []string `json:"css"` 32 | CertIssuer string `json:"cert_issuer"` 33 | Robots string `json:"robots"` 34 | Meta map[string]string `json:"meta"` 35 | Scripts []string `json:"scripts"` 36 | Js map[string]string `json:"js"` 37 | Implies []string `json:"implies"` 38 | Excludes []string `json:"excludes"` 39 | Icon string `json:"icon"` 40 | Website string `json:"website"` 41 | CPE string `json:"cpe"` 42 | } 43 | 44 | // 指纹信息中类别结构体 45 | type FingerprintCategories struct { 46 | Name string `json:"name"` 47 | Priority int `json:"priority"` 48 | } 49 | 50 | // dom信息结构体 51 | type DomInfo struct { 52 | Key string `json:"key"` 53 | Attr string `json:"attr"` 54 | Label string `json:"label"` 55 | Rule string `json:"rule"` 56 | } 57 | --------------------------------------------------------------------------------