├── web ├── styles │ └── style.css ├── scripts │ └── script.js └── templates │ └── index.html ├── .gitignore ├── .github └── workflows │ ├── go.yml │ └── release.yml ├── go.mod ├── .vscode └── launch.json ├── mlweb.go ├── README.md ├── mlget-test-config └── samples.yaml ├── history.go ├── hashes.go ├── upload.go ├── mlget.go ├── config.go ├── mlget_test.go └── download.go /web/styles/style.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | mlget 2 | mlget.yml 3 | go.sum 4 | .mlget.yml 5 | mlget.bak.yml 6 | mlget-bak.yml 7 | .vscode/* -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.24.1 20 | 21 | - name: Build 22 | run: go get -u && go build -v ./... 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module xorhex.com/mlget 2 | 3 | go 1.21.3 4 | 5 | require ( 6 | github.com/spf13/pflag v1.0.6 7 | github.com/yeka/zip v0.0.0-20180914125537-d046722c6feb 8 | golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 9 | gopkg.in/yaml.v2 v2.4.0 10 | ) 11 | 12 | require ( 13 | github.com/kr/pretty v0.3.1 // indirect 14 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect 15 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Package", 9 | "type": "go", 10 | "debugAdapter": "dlv-dap", 11 | "request": "launch", 12 | "host": "127.0.0.1", 13 | "mode": "debug", 14 | "program": "${fileDirname}", 15 | "args":[ 16 | "--ud", "--t", "mimikatz", "B9601E60F87545441BF8579B2F62668C56507F4A" 17 | ] 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/release.yaml 2 | 3 | on: 4 | release: 5 | types: [created, draft] 6 | 7 | jobs: 8 | releases-matrix: 9 | name: Release Go Binary 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | # build and publish in parallel: linux/386, linux/amd64, windows/386, windows/amd64, darwin/amd64 14 | goos: [linux, windows, darwin] 15 | goarch: [amd64] 16 | steps: 17 | - uses: actions/checkout@v3 18 | - uses: wangyoucao577/go-release-action@v1 19 | with: 20 | github_token: ${{ secrets.GITHUB_TOKEN }} 21 | goos: ${{ matrix.goos }} 22 | goarch: ${{ matrix.goarch }} 23 | sha256sum: true 24 | build_command: go build 25 | build_flags: -v 26 | pre_command: go get -u 27 | binary_name: "mlget" -------------------------------------------------------------------------------- /web/scripts/script.js: -------------------------------------------------------------------------------- 1 | var table = $('#hashes').DataTable( { 2 | serverSide: true, 3 | ajax: '/data-source' 4 | } ); 5 | 6 | // Attach a submit handler to the form 7 | $( "#download" ).submit(function( event ) { 8 | 9 | // Stop form from submitting normally 10 | event.preventDefault(); 11 | 12 | // Get some values from elements on the page: 13 | var $form = $( this ), 14 | term = $form.find( "input[name='hashes']" ).val(), 15 | url = $form.attr( "action" ); 16 | 17 | // Send the data using post 18 | var posting = $.post( url, { hashes: term } ); 19 | 20 | // Put the results in a div 21 | posting.done(function( data ) { 22 | table.ajax.reload( null, false ); // user paging is not reset on reload 23 | }); 24 | }); 25 | 26 | 27 | setInterval( function () { 28 | table.ajax.reload( null, false ); // user paging is not reset on reload 29 | }, 30000 ); -------------------------------------------------------------------------------- /mlweb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "strings" 8 | "text/template" 9 | ) 10 | 11 | type Page struct { 12 | hashes []string 13 | tags []string 14 | comments []string 15 | } 16 | 17 | func indexHandler(w http.ResponseWriter, r *http.Request) { 18 | if r.Method == "POST" { 19 | values := strings.Split(r.PostFormValue("hashes"), "\r\n") 20 | tags := strings.Split(r.PostFormValue("tags"), "\r\n") 21 | comments := strings.Split(r.PostFormValue("comments"), "\r\n") 22 | go processMalwareDownloadRequest(values, tags, comments) 23 | } 24 | t, _ := template.ParseFiles("./web/templates/index.html") 25 | t.Execute(w, nil) 26 | } 27 | 28 | func processMalwareDownloadRequest(values []string, tags []string, comments []string) { 29 | //hashes := parseArgHashes(values, tags, comments) 30 | //downloadMalwareFromWebServer(hashes) 31 | } 32 | 33 | func runWebServer(bind string, port int) { 34 | 35 | http.HandleFunc("/styles/style.css", func(response http.ResponseWriter, request *http.Request) { 36 | http.ServeFile(response, request, "./web/styles/style.css") 37 | }) 38 | 39 | http.HandleFunc("/scripts/script.js", func(response http.ResponseWriter, request *http.Request) { 40 | http.ServeFile(response, request, "./web/scripts/script.js") 41 | }) 42 | 43 | http.HandleFunc("/", indexHandler) 44 | 45 | //http.HandleFunc("/download", postDataHandler) 46 | 47 | log.Fatal(http.ListenAndServe(fmt.Sprint(bind, ":", port), nil)) 48 | } 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## mlget 2 | 3 | ![image](https://api.xorhex.com/resource/png/Mlget-ReadMe/052cc80c1529236836d0bb7e35c30dff6567a285056564468d33b6197025d36c) 4 | 5 | 6 | ![Build](https://github.com/xorhex/mlget/actions/workflows/go.yml/badge.svg) 7 | 8 | ### What is it 9 | 10 | Use mlget to query multiple sources for a given malware hash and download it. The thought is to save time querying each source individually. 11 | 12 | ### Usage Instructions 13 | 14 | [Mlget Blog Post](https://blog.xorhex.com/mlget/) 15 | 16 | ### License 17 | 18 | MIT License 19 | 20 | Copyright (c) 2025 @xorhex 21 | 22 | Permission is hereby granted, free of charge, to any person obtaining a copy 23 | of this software and associated documentation files (the "Software"), to deal 24 | in the Software without restriction, including without limitation the rights 25 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 26 | copies of the Software, and to permit persons to whom the Software is 27 | furnished to do so, subject to the following conditions: 28 | 29 | The above copyright notice and this permission notice shall be included in all 30 | copies or substantial portions of the Software. 31 | 32 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 33 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 34 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 35 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 36 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 37 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 38 | SOFTWARE. 39 | 40 | -------------------------------------------------------------------------------- /web/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 10 | 11 | 13 | 14 | 27 | 28 | 29 |
30 |
31 |
32 |

MLGET - Download Malware

33 |
34 |
35 | 36 |
37 |
38 | 39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 | 47 |
48 |
49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 |
HashHash TypeFound On
60 |
61 |
-------------------------------------------------------------------------------- /mlget-test-config/samples.yaml: -------------------------------------------------------------------------------- 1 | test 1: 2 | name: TestJoeSandbox 3 | hash: 40541a03e910b21df681bec69cfe59678ebba86c 4 | test 2: 5 | name: TestObjectiveSee 6 | hash: 458a9ac086116fa011c1a7bd49ac15f386cd95e39eb6b7cd5c5125aef516c78c 7 | test 3: 8 | name: TestCapeSandbox 9 | hash: 28eefc36104bebb595fb38cae21a7d0a 10 | test 4: 11 | name: TestInquestLabsLookUp 12 | hash: b3f868fa1af24f270e3ecc0ecb79325e 13 | test 5: 14 | name: TestInquestLabsNoLookUp 15 | hash: 6b425804d43bb369211bbec59808807730a908804ca9b8c09081139179bbc868 16 | test 6: 17 | name: TestVirusTotal 18 | hash: 21cc9c0ae5f97b66d69f1ff99a4fed264551edfe0a5ce8d5449942bf8f0aefb2 19 | test 7: 20 | name: TestMWDB 21 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 22 | test 8: 23 | name: TestPolyswarm 24 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 25 | test 9: 26 | name: TestHybridAnalysis 27 | hash: ed2f501408a7a6e1a854c29c4b0bc5648a6aa8612432df829008931b3e34bf56 28 | test 10: 29 | name: TestTriage 30 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 31 | test 11: 32 | name: TestMalShare 33 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 34 | test 12: 35 | name: TestMalwareBazaar 36 | hash: 001bffcdd170c8328601006ad54a221d1073ba04fbdca556749cf1b041cfad97 37 | test 13: 38 | name: TestMalpedia 39 | hash: 78668c237097651d64c97b25fc86c74096bfe1ed53e1004445f118ea5feaa3ad 40 | test 14: 41 | name: TestUnpacme 42 | hash: 0219a79a2f47da42601568ee4a41392aa429f62a1fb01080cb68540074449c92 43 | test 15: 44 | name: TestVxShare 45 | hash: 1c11c963a417674e1414bac05fdbfa5cfa09f92c7b0d9882aeb55ce2a058d668 46 | test 16: 47 | name: TestFileScanIo 48 | hash: 2799af2efd698da215afc9c88da3b1e84b00137433d9444a5c11d69092b3f80d 49 | test 17: 50 | name: TestURLScanIo 51 | hash: 5b027ada26a610e97ab4ef9efb1118b377061712acec6db994d6aa1c78a332a8 52 | test 19: 53 | name: TestAssemblyLine 54 | hash: b78e786091f017510b44137961f3074fe7d5f950 55 | test 20: 56 | name: TestTriageV2 57 | hash: 0d8d46ec44e737e6ef6cd7df8edf95d83807e84be825ef76089307b399a6bcbb 58 | test 21: 59 | name: TestVirusExchange 60 | hash: 5d11b9be5daa65fe010cc7900d5d5eead7f62a7885e862a5971a005856ae9878 61 | test 22: 62 | name: TestHybridAnalysisNotFound 63 | hash: fc17c021f18ec73d1544ad46dde6a1f1949f126bf3e75f97e241f982e2b07c86 64 | test 23: 65 | name: TestVirusExchangeV2 66 | hash: 0cacdb88b24bd34b9d8ef600b06814f76206e60e70f975c8e4bdaa1ab7cebb80 67 | test 24: 68 | name: TestMalwareBazaarNotFound 69 | hash: fee889e9518d1c660bd6fa331c19aabada7eeff8f1c99f2ef4d64c662ed5805a 70 | test 25: 71 | name: TestMalwareBazaarMD5 72 | hash: 9078dcd62129e872c84ba4bc6574f607 -------------------------------------------------------------------------------- /history.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | func writeToFile(file io.ReadCloser, filename string) error { 15 | // Create the file 16 | out, err := os.Create(filename) 17 | if err != nil { 18 | return err 19 | } 20 | defer out.Close() 21 | 22 | // Write the body to file 23 | _, err = io.Copy(out, file) 24 | return err 25 | } 26 | 27 | // Code for this function came from - https://golangcode.com/how-to-check-if-a-string-is-a-url/ 28 | // isValidUrl tests a string to determine if it is a well-structured url or not. 29 | func isValidUrl(toTest string) bool { 30 | _, err := url.ParseRequestURI(toTest) 31 | if err != nil { 32 | return false 33 | } 34 | 35 | u, err := url.Parse(toTest) 36 | if err != nil || u.Scheme == "" || u.Host == "" { 37 | return false 38 | } 39 | 40 | return true 41 | } 42 | 43 | // https://golangcode.com/check-if-a-file-exists/ 44 | func fileExists(filename string) bool { 45 | info, err := os.Stat(filename) 46 | if os.IsNotExist(err) { 47 | return false 48 | } 49 | return !info.IsDir() 50 | } 51 | 52 | func downloadFromUrl(url string) (string, error) { 53 | 54 | filename := "mlget.download.data.tmp" 55 | 56 | r, err := http.Get(url) 57 | if err != nil { 58 | log.Println("Cannot get from URL", err) 59 | return "", err 60 | } 61 | 62 | defer r.Body.Close() 63 | 64 | if !fileExists(filename) { 65 | 66 | file, _ := os.Create(filename) 67 | defer file.Close() 68 | 69 | writer := bufio.NewWriter(file) 70 | io.Copy(writer, r.Body) 71 | writer.Flush() 72 | 73 | return filename, nil 74 | 75 | } else { 76 | return "", fmt.Errorf("file %s already exists - delete and try again", filename) 77 | } 78 | } 79 | 80 | func parseFileForHashEntries(filename string) ([]Hash, error) { 81 | hashes := []Hash{} 82 | var _filename string 83 | var err error 84 | 85 | fmt.Printf("Hashes Found in File:\n") 86 | 87 | if isValidUrl(filename) { 88 | _filename, err = downloadFromUrl(filename) 89 | if err != nil { 90 | return nil, err 91 | } 92 | } else { 93 | _filename = filename 94 | } 95 | 96 | file, err := os.Open(_filename) 97 | if err != nil { 98 | fmt.Println("Error reading file") 99 | fmt.Println(err) 100 | } 101 | 102 | defer func() ([]string, error) { 103 | if err = file.Close(); err != nil { 104 | fmt.Println(err) 105 | return nil, err 106 | } 107 | return nil, nil 108 | }() 109 | 110 | f := func(c rune) bool { 111 | return c == '|' 112 | } 113 | 114 | scanner := bufio.NewScanner(file) 115 | for scanner.Scan() { // internally, it advances token based on separator 116 | text := scanner.Text() 117 | if len(strings.TrimSpace(text)) > 0 { 118 | hash := strings.FieldsFunc(strings.TrimSpace(text), f)[0] 119 | tags := []string{} 120 | comments := []string{} 121 | if len(strings.FieldsFunc(text, f)) > 1 { 122 | fields := strings.FieldsFunc(text, f)[1:len(strings.FieldsFunc(text, f))] 123 | tagSection := false 124 | commentSection := false 125 | for _, f := range fields { 126 | if f == "TAGS" { 127 | tagSection = true 128 | commentSection = false 129 | } else if f == "COMMENTS" { 130 | tagSection = false 131 | commentSection = true 132 | } else if f != "TAGS" && f != "COMMENTS" && tagSection { 133 | tags = append(tags, f) 134 | } else if f != "TAGS" && f != "COMMENTS" && commentSection { 135 | comments = append(comments, f) 136 | } 137 | } 138 | } 139 | pHash := Hash{} 140 | pHash, err = parseFileHashEntry(hash, tags, comments) 141 | if err == nil { 142 | hashes = append(hashes, pHash) 143 | } else { 144 | // Try splitting on \t and check to see if any of the values match a hash 145 | // This is useful for reading files from the web that list sample hashes 146 | // This still assumes there is only one hash per line as it stops after the 147 | // first hash is found on that line 148 | s := func(c rune) bool { 149 | return c == '\t' 150 | } 151 | 152 | line := strings.FieldsFunc(strings.TrimSpace(text), s) 153 | if len(line) > 0 { 154 | for _, element := range line { 155 | lHash := Hash{} 156 | lHash, err := parseFileHashEntry(strings.TrimSpace(element), tags, comments) 157 | if err == nil { 158 | hashes = append(hashes, lHash) 159 | break 160 | } else { 161 | 162 | matches, err := extractHashes(strings.TrimSpace(element)) 163 | if err != nil { 164 | fmt.Println(err) 165 | } 166 | for m := range matches { 167 | tags := []string{} 168 | comments := []string{} 169 | lHash, err = parseFileHashEntry(matches[m], tags, comments) 170 | if err == nil { 171 | hashes = append(hashes, lHash) 172 | } 173 | } 174 | } 175 | } 176 | } 177 | } 178 | } 179 | } 180 | 181 | if _filename == "mlget.download.data.tmp" { 182 | os.Remove(_filename) 183 | } 184 | 185 | fmt.Println("") 186 | 187 | return hashes, nil 188 | } 189 | 190 | func writeUnfoundHashesToFile(filename string, hashes Hashes) error { 191 | f, err := os.Create(filename) 192 | if err != nil { 193 | return err 194 | } 195 | defer f.Close() 196 | 197 | w := bufio.NewWriter(f) 198 | defer w.Flush() 199 | 200 | for _, h := range hashes.Hashes { 201 | w.WriteString(h.Hash + "|TAGS|" + strings.Join(h.Tags, "|") + "|COMMENTS|" + strings.Join(h.Comments, "|") + "\n") 202 | } 203 | return nil 204 | } 205 | 206 | func parseFileHashEntry(hash string, tags []string, comments []string) (Hash, error) { 207 | ht, err := hashType(hash) 208 | if err != nil { 209 | fmt.Printf("\n Skipping %s because it's %s\n", hash, err) 210 | return Hash{}, err 211 | } 212 | fmt.Printf(" - %s\n", hash) // token in unicode-char 213 | hashS := Hash{Hash: hash, HashType: ht} 214 | if len(tags) > 0 { 215 | hashS.Tags = tags 216 | } 217 | if len(comments) > 0 { 218 | hashS.Comments = comments 219 | } 220 | return hashS, nil 221 | } 222 | -------------------------------------------------------------------------------- /hashes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "crypto" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | "regexp" 12 | "strings" 13 | ) 14 | 15 | var alwaysDeleteInvalidFile = false 16 | 17 | type Hashes struct { 18 | Hashes []Hash 19 | } 20 | 21 | type Hash struct { 22 | Hash string 23 | HashType HashTypeOption 24 | Tags []string 25 | Comments []string 26 | Local bool // True if found locally on the filesystem (used with the -precheckdir flag). Default False 27 | LocalFile string // Full file name if file found on local system (used with the -precheckdir flag) 28 | } 29 | 30 | type HashTypeOption int64 31 | 32 | const ( 33 | NotAValidHashType HashTypeOption = iota 34 | md5 35 | sha1 36 | sha256 37 | ) 38 | 39 | func (hto HashTypeOption) String() string { 40 | switch hto { 41 | case md5: 42 | return "md5" 43 | case sha1: 44 | return "sha1" 45 | case sha256: 46 | return "sha256" 47 | } 48 | return "" 49 | } 50 | 51 | func addHash(hashes Hashes, hash Hash) (Hashes, error) { 52 | if hashes.hashExists(hash.Hash) { 53 | hsh, err := hashes.getByHash(hash.Hash) 54 | if err != nil { 55 | return hashes, err 56 | } 57 | for _, t := range hash.Tags { 58 | if !hsh.TagExists(t) { 59 | hsh.Tags = append(hsh.Tags, t) 60 | } 61 | } 62 | 63 | } else { 64 | hashes.Hashes = append(hashes.Hashes, hash) 65 | } 66 | return hashes, nil 67 | } 68 | 69 | func (hs Hashes) updateLocalFile(hash string, filename string) { 70 | for idx, h := range hs.Hashes { 71 | if h.Hash == hash { 72 | hs.Hashes[idx].Local = true 73 | hs.Hashes[idx].LocalFile = filename 74 | } 75 | } 76 | } 77 | 78 | func (hs Hashes) hashExists(hash string) bool { 79 | for _, h := range hs.Hashes { 80 | if h.Hash == hash { 81 | return true 82 | } 83 | } 84 | return false 85 | } 86 | 87 | func (hs Hashes) getByHash(hash string) (Hash, error) { 88 | for idx, h := range hs.Hashes { 89 | if h.Hash == hash { 90 | return hs.Hashes[idx], nil 91 | } 92 | } 93 | return Hash{}, fmt.Errorf("Hash not found") 94 | } 95 | 96 | func (h Hash) TagExists(tag string) bool { 97 | for _, t := range h.Tags { 98 | if t == tag { 99 | return true 100 | } 101 | } 102 | return false 103 | } 104 | 105 | func (h Hash) ValidateFile(filename string) (bool, string) { 106 | f, err := os.Open(filename) 107 | if err != nil { 108 | log.Fatal(err) 109 | } 110 | defer f.Close() 111 | 112 | var sum []byte 113 | 114 | if h.HashType == md5 { 115 | hasher := crypto.MD5.New() 116 | if _, err := io.Copy(hasher, f); err != nil { 117 | log.Fatal(err) 118 | } 119 | sum = hasher.Sum(nil) 120 | } else if h.HashType == sha1 { 121 | hasher := crypto.SHA1.New() 122 | if _, err := io.Copy(hasher, f); err != nil { 123 | log.Fatal(err) 124 | } 125 | sum = hasher.Sum(nil) 126 | } else if h.HashType == sha256 { 127 | hasher := crypto.SHA256.New() 128 | if _, err := io.Copy(hasher, f); err != nil { 129 | log.Fatal(err) 130 | } 131 | sum = hasher.Sum(nil) 132 | } 133 | if (fmt.Sprintf("%x", sum)) == strings.ToLower(h.Hash) { 134 | return true, fmt.Sprintf("%x", sum) 135 | } else { 136 | return false, fmt.Sprintf("%x", sum) 137 | } 138 | } 139 | 140 | func (h Hash) Validate(bytes []byte) (bool, string) { 141 | var sum []byte 142 | 143 | if h.HashType == md5 { 144 | hasher := crypto.MD5.New() 145 | hasher.Write(bytes) 146 | sum = hasher.Sum(nil) 147 | } else if h.HashType == sha1 { 148 | hasher := crypto.SHA1.New() 149 | hasher.Write(bytes) 150 | sum = hasher.Sum(nil) 151 | } else if h.HashType == sha256 { 152 | hasher := crypto.SHA256.New() 153 | hasher.Write(bytes) 154 | sum = hasher.Sum(nil) 155 | } 156 | if (fmt.Sprintf("%x", sum)) == strings.ToLower(h.Hash) { 157 | return true, fmt.Sprintf("%x", sum) 158 | } else { 159 | return false, fmt.Sprintf("%x", sum) 160 | } 161 | } 162 | 163 | func deleteInvalidFile(filename string) { 164 | ok := YesNoAlwaysDeleteInvalidFilePrompt(" [?] Delete invalid file?", true) 165 | if ok { 166 | os.Remove(filename) 167 | fmt.Printf(" [!] Deleted invalid file\n") 168 | } else { 169 | fmt.Printf(" [!] Keeping invalid file\n") 170 | } 171 | } 172 | 173 | func hashType(hash string) (HashTypeOption, error) { 174 | match, _ := regexp.MatchString("^[A-Fa-f0-9]{64}$", hash) 175 | if match { 176 | return sha256, nil 177 | } 178 | match, _ = regexp.MatchString("^[A-Fa-f0-9]{40}$", hash) 179 | if match { 180 | return sha1, nil 181 | } 182 | match, _ = regexp.MatchString("^[A-Fa-f0-9]{32}$", hash) 183 | if match { 184 | return md5, nil 185 | } 186 | return NotAValidHashType, errors.New("not a valid hash") 187 | } 188 | 189 | func extractHashes(text string) ([]string, error) { 190 | hashes := make([]string, 0) 191 | 192 | re := regexp.MustCompile(`>\s*[A-Fa-f0-9]{64}\s*<`) 193 | matches := re.FindAllStringSubmatch(text, 100) 194 | for m := range matches { 195 | hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) 196 | } 197 | re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{40}\s*<`) 198 | matches = re.FindAllStringSubmatch(text, 100) 199 | for m := range matches { 200 | hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) 201 | } 202 | re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{32}\s*<`) 203 | matches = re.FindAllStringSubmatch(text, 100) 204 | for m := range matches { 205 | hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) 206 | } 207 | 208 | if len(hashes) > 0 { 209 | return hashes, fmt.Errorf("no hashes found") 210 | } 211 | 212 | return hashes, nil 213 | } 214 | 215 | func YesNoAlwaysDeleteInvalidFilePrompt(label string, def bool) bool { 216 | if alwaysDeleteInvalidFile { 217 | return true 218 | } 219 | 220 | choices := "a - always /Y - Yes /n - no" 221 | if !def { 222 | choices = "a - always /y - yes /N - No" 223 | } 224 | 225 | r := bufio.NewReader(os.Stdin) 226 | var s string 227 | 228 | for { 229 | fmt.Fprintf(os.Stderr, "%s (%s) ", label, choices) 230 | s, _ = r.ReadString('\n') 231 | s = strings.TrimSpace(s) 232 | if s == "" { 233 | return def 234 | } 235 | s = strings.ToLower(s) 236 | if s == "y" || s == "yes" || s == "Y" { 237 | return true 238 | } 239 | if s == "n" || s == "no" || s == "N" { 240 | return false 241 | } 242 | if s == "a" || s == "always" || s == "A" { 243 | alwaysDeleteInvalidFile = true 244 | return true 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /upload.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "mime/multipart" 11 | "net/http" 12 | "net/url" 13 | "os" 14 | "strings" 15 | ) 16 | 17 | func doesSampleExistInAssemblyLine(uri string, api string, user string, hash Hash, ignoreTLSErrors bool) bool { 18 | if api == "" { 19 | fmt.Println(" [!] !! Missing Key !!") 20 | return false 21 | } 22 | if user == "" { 23 | fmt.Println(" [!] !! Missing User !!") 24 | return false 25 | } 26 | 27 | request, error := http.NewRequest("GET", uri+"/hash_search/"+url.PathEscape(hash.Hash)+"/", nil) 28 | if error != nil { 29 | fmt.Println(error) 30 | return false 31 | } 32 | 33 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 34 | request.Header.Set("x-user", user) 35 | request.Header.Set("x-apikey", api) 36 | 37 | tr := &http.Transport{} 38 | if ignoreTLSErrors { 39 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 40 | } 41 | 42 | client := &http.Client{Transport: tr} 43 | response, error := client.Do(request) 44 | if error != nil { 45 | fmt.Printf(" [!] Error with querying AssemblyLine for hash : %s\n", error) 46 | return false 47 | } 48 | defer response.Body.Close() 49 | 50 | if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { 51 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 52 | return false 53 | } 54 | 55 | byteValue, _ := io.ReadAll(response.Body) 56 | 57 | var data = AssemblyLineQuery{} 58 | error = json.Unmarshal(byteValue, &data) 59 | 60 | if error != nil { 61 | fmt.Println(error) 62 | return false 63 | } 64 | 65 | if data.Response.AL == nil { 66 | return false 67 | } 68 | 69 | if len(data.Response.AL.Items) > 0 { 70 | return true 71 | } 72 | return false 73 | } 74 | 75 | func UploadSampleToAssemblyLine(repos []RepositoryConfigEntry, filename string, hash Hash, deleteFromDisk bool, forceResubmission bool) error { 76 | matchingConfigRepos := getConfigsByType(UploadAssemblyLine, repos) 77 | if len(matchingConfigRepos) == 0 { 78 | return fmt.Errorf(" upload to assemblyline config entry not found") 79 | } 80 | for _, mcr := range matchingConfigRepos { 81 | if !forceResubmission && doesSampleExistInAssemblyLine(mcr.Host, mcr.Api, mcr.User, hash, mcr.IgnoreTLSErrors) { 82 | fmt.Println(" Sample Already Exist in AssemblyLine. Not Reuploading.") 83 | continue 84 | } 85 | 86 | bodyBuf := &bytes.Buffer{} 87 | bodyWriter := multipart.NewWriter(bodyBuf) 88 | 89 | // this step is very important 90 | fileWriter, err := bodyWriter.CreateFormFile("bin", filename) 91 | if err != nil { 92 | return fmt.Errorf("error writing to buffer") 93 | } 94 | 95 | // open file handle 96 | fh, err := os.Open(filename) 97 | if err != nil { 98 | return fmt.Errorf("error opening file") 99 | } 100 | defer fh.Close() 101 | 102 | //iocopy 103 | _, err = io.Copy(fileWriter, fh) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | contentType := bodyWriter.FormDataContentType() 109 | bodyWriter.Close() 110 | 111 | request, error := http.NewRequest("POST", mcr.Host+"/submit/", bodyBuf) 112 | if error != nil { 113 | return error 114 | } 115 | 116 | request.Header.Set("accept", "application/json") 117 | request.Header.Set("x-user", mcr.User) 118 | request.Header.Set("x-apikey", mcr.Api) 119 | request.Header.Set("Content-Type", contentType) 120 | 121 | tr := &http.Transport{} 122 | if mcr.IgnoreTLSErrors { 123 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 124 | } 125 | client := &http.Client{Transport: tr} 126 | resp, err := client.Do(request) 127 | if err != nil { 128 | return err 129 | } 130 | 131 | defer resp.Body.Close() 132 | 133 | if resp.StatusCode != http.StatusOK { 134 | return fmt.Errorf("error uploading file - status code %d returned", resp.StatusCode) 135 | 136 | } else { 137 | fmt.Printf(" %s uploaded to AssemblyLine (%s)\n", filename, mcr.Host) 138 | if deleteFromDisk { 139 | os.Remove(filename) 140 | fmt.Printf(" %s deleted from disk\n", filename) 141 | } 142 | } 143 | } 144 | return nil 145 | } 146 | 147 | type CommentItemResponse struct { 148 | Author string `json:"author"` 149 | Comment string `json:"comment"` 150 | Id int32 `json:"id"` 151 | Timestamp string `json:"timestamp"` 152 | } 153 | 154 | func doesSampleExistInMWDB(uri string, api string, hash string) bool { 155 | query := uri + "/file/" + hash 156 | 157 | request, err := http.NewRequest("GET", query, nil) 158 | if err != nil { 159 | fmt.Println(err) 160 | return false 161 | } 162 | 163 | request.Header.Set("Authorization", "Bearer "+api) 164 | client := &http.Client{} 165 | response, error := client.Do(request) 166 | if error != nil { 167 | fmt.Println(error) 168 | return false 169 | } 170 | defer response.Body.Close() 171 | 172 | if response.StatusCode == http.StatusOK { 173 | fmt.Printf(" [!] File %s already exists in MWDB: %s \n", hash, uri) 174 | return true 175 | } else { 176 | fmt.Println("") 177 | return false 178 | } 179 | } 180 | 181 | func UploadSampleToMWDBs(repos []RepositoryConfigEntry, filename string, hash Hash, deleteFromDisk bool) error { 182 | matchingConfigRepos := getConfigsByType(UploadMWDB, repos) 183 | for _, mcr := range matchingConfigRepos { 184 | if doesSampleExistInMWDB(mcr.Host, mcr.Api, hash.Hash) { 185 | continue 186 | } 187 | 188 | bodyBuf := &bytes.Buffer{} 189 | bodyWriter := multipart.NewWriter(bodyBuf) 190 | 191 | // this step is very important 192 | fileWriter, err := bodyWriter.CreateFormFile("file", filename) 193 | if err != nil { 194 | fmt.Println("error writing to buffer") 195 | return err 196 | } 197 | 198 | // open file handle 199 | fh, err := os.Open(filename) 200 | if err != nil { 201 | fmt.Println("error opening file") 202 | return err 203 | } 204 | defer fh.Close() 205 | 206 | //iocopy 207 | _, err = io.Copy(fileWriter, fh) 208 | if err != nil { 209 | return err 210 | } 211 | 212 | contentType := bodyWriter.FormDataContentType() 213 | bodyWriter.Close() 214 | 215 | request, error := http.NewRequest("POST", mcr.Host+"/file", bodyBuf) 216 | if error != nil { 217 | fmt.Println(error) 218 | return error 219 | } 220 | 221 | request.Header.Set("Authorization", "Bearer "+mcr.Api) 222 | request.Header.Set("Content-Type", contentType) 223 | 224 | client := &http.Client{} 225 | resp, err := client.Do(request) 226 | if err != nil { 227 | return err 228 | } 229 | 230 | defer resp.Body.Close() 231 | 232 | if resp.StatusCode != http.StatusOK { 233 | return fmt.Errorf("error uploading file - status code %d returned", resp.StatusCode) 234 | 235 | } else { 236 | fmt.Printf(" [-] %s uploaded to MWDB (%s)\n", filename, mcr.Host) 237 | if deleteFromDisk { 238 | os.Remove(filename) 239 | fmt.Printf(" [-] %s deleted from disk\n", filename) 240 | } 241 | } 242 | 243 | err = addTagsToSampleInMWDB(hash, mcr.Host, mcr.Api) 244 | if err != nil { 245 | return err 246 | } 247 | 248 | err = addCommentsToSampleInMWDB(hash, mcr.Host, mcr.Api) 249 | if err != nil { 250 | return err 251 | } 252 | } 253 | return nil 254 | } 255 | 256 | func AddTagsToSamplesAcrossMWDBs(repos []RepositoryConfigEntry, hash Hash) { 257 | matchingConfigRepos := getConfigsByType(UploadMWDB, repos) 258 | for _, mcr := range matchingConfigRepos { 259 | if !doesSampleExistInMWDB(mcr.Host, mcr.Api, hash.Hash) { 260 | continue 261 | } 262 | err := addTagsToSampleInMWDB(hash, mcr.Host, mcr.Api) 263 | if err != nil { 264 | fmt.Printf("Error occurred while tagging %s on %s (%s)\n", hash.Hash, mcr.Type, mcr.Host) 265 | } 266 | } 267 | } 268 | 269 | func addTagsToSampleInMWDB(hash Hash, mwdbServer string, auth string) error { 270 | for _, t := range hash.Tags { 271 | query := mwdbServer + "/file/" + hash.Hash + "/tag" 272 | 273 | _, error := url.ParseQuery(query) 274 | if error != nil { 275 | fmt.Println(error) 276 | return error 277 | } 278 | 279 | value_json := "{\"tag\":\"" + t + "\"}" 280 | request, error := http.NewRequest("PUT", query, strings.NewReader(value_json)) 281 | if error != nil { 282 | fmt.Println(error) 283 | return error 284 | } 285 | 286 | request.Header.Set("Authorization", "Bearer "+auth) 287 | 288 | client := &http.Client{} 289 | respTag, err := client.Do(request) 290 | if err != nil { 291 | return err 292 | } 293 | 294 | if respTag.StatusCode == http.StatusOK { 295 | fmt.Printf(" [-] %s tagged as %s\n", hash.Hash, t) 296 | } else { 297 | fmt.Printf(" [!] Failed to tag %s as %s\n", hash.Hash, t) 298 | } 299 | } 300 | return nil 301 | } 302 | 303 | func AddCommentsToSamplesAcrossMWDBs(repos []RepositoryConfigEntry, hash Hash) { 304 | matchingConfigRepos := getConfigsByType(UploadMWDB, repos) 305 | for _, mcr := range matchingConfigRepos { 306 | if !doesSampleExistInMWDB(mcr.Host, mcr.Api, hash.Hash) { 307 | continue 308 | } 309 | err := addCommentsToSampleInMWDB(hash, mcr.Host, mcr.Api) 310 | if err != nil { 311 | fmt.Printf("Error occurred while adding comments to %s on %s (%s)\n", hash.Hash, mcr.Type, mcr.Host) 312 | } 313 | } 314 | } 315 | 316 | func addCommentsToSampleInMWDB(hash Hash, mwdbServer string, auth string) error { 317 | 318 | // Get existing comments 319 | getQuery := mwdbServer + "/file/" + hash.Hash + "/comment" 320 | _, error := url.ParseQuery(getQuery) 321 | if error != nil { 322 | fmt.Println(error) 323 | return error 324 | } 325 | getRequest, error := http.NewRequest("GET", getQuery, nil) 326 | if error != nil { 327 | fmt.Println(error) 328 | return error 329 | } 330 | getRequest.Header.Set("Authorization", "Bearer "+auth) 331 | 332 | getClient := &http.Client{} 333 | getResponse, error := getClient.Do(getRequest) 334 | if error != nil { 335 | fmt.Println(error) 336 | return error 337 | } 338 | defer getResponse.Body.Close() 339 | 340 | var getData []CommentItemResponse 341 | if getResponse.StatusCode == http.StatusOK { 342 | 343 | byteValue, error := ioutil.ReadAll(getResponse.Body) 344 | if error != nil { 345 | fmt.Println(error) 346 | return error 347 | } 348 | 349 | error = json.Unmarshal(byteValue, &getData) 350 | 351 | if error != nil { 352 | fmt.Println(error) 353 | return error 354 | } 355 | } 356 | 357 | for _, c := range hash.Comments { 358 | 359 | // Check to make sure the comment does not already exists before added it, if it does exist continue on to the next comment 360 | commentExists := false 361 | if len(getData) > 0 { 362 | for _, existingComment := range getData { 363 | if c == existingComment.Comment { 364 | commentExists = true 365 | break 366 | } 367 | } 368 | } 369 | if commentExists { 370 | fmt.Printf(" [!] %s comment already exists for %s\n", c, hash.Hash) 371 | continue 372 | } 373 | 374 | // Add net comment to sample 375 | query := mwdbServer + "/file/" + hash.Hash + "/comment" 376 | 377 | _, error := url.ParseQuery(query) 378 | if error != nil { 379 | fmt.Println(error) 380 | return error 381 | } 382 | 383 | value_json := "{\"comment\":\"" + c + "\"}" 384 | request, error := http.NewRequest("POST", query, strings.NewReader(value_json)) 385 | if error != nil { 386 | fmt.Println(error) 387 | return error 388 | } 389 | 390 | request.Header.Set("Authorization", "Bearer "+auth) 391 | 392 | client := &http.Client{} 393 | respTag, err := client.Do(request) 394 | if err != nil { 395 | return err 396 | } 397 | 398 | if respTag.StatusCode == http.StatusOK { 399 | fmt.Printf(" [-] %s comment added for %s\n", c, hash.Hash) 400 | } else { 401 | fmt.Printf(" [!] Failed to comment %s for %s\n", c, hash.Hash) 402 | } 403 | } 404 | return nil 405 | } 406 | -------------------------------------------------------------------------------- /mlget.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "path" 10 | "sync" 11 | "time" 12 | 13 | flag "github.com/spf13/pflag" 14 | "golang.org/x/exp/slices" 15 | ) 16 | 17 | var apiFlag string 18 | var helpFlag bool 19 | var checkConfFlag bool 20 | var AddConfigEntryFlag bool 21 | var doNotExtractFlag bool 22 | var inputFileFlag string 23 | var outputFileFlag bool 24 | var uploadToMWDBAndDeleteFlag bool 25 | var downloadOnlyFlag bool 26 | var uploadToMWDBFlag bool 27 | var readFromFileAndUpdateWithNotFoundHashesFlag string 28 | var tagsFlag []string 29 | var commentsFlag []string 30 | var versionFlag bool 31 | var noValidationFlag bool 32 | var precheckdir bool 33 | var uploadToAssemblyLineFlag bool 34 | var uploadToAssemblyLineAndDeleteFlag bool 35 | var forceResubmission bool 36 | 37 | var version string = "3.4.5" 38 | 39 | func usage() { 40 | fmt.Println("mlget - A command line tool to download malware from a variety of sources") 41 | fmt.Println("") 42 | 43 | fmt.Printf("Usage: %s [OPTIONS] hash_arguments...\n", os.Args[0]) 44 | flag.PrintDefaults() 45 | 46 | fmt.Println("") 47 | fmt.Println("Example Usage: mlget ") 48 | fmt.Println("Example Usage: mlget --from mb ") 49 | fmt.Println("Example Usage: mlget --tag tag_one --tag tag_two --uploaddelete ") 50 | } 51 | 52 | func init() { 53 | flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - al (AssemblyLine)\n - cs (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tr (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - ve (VExchange)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") 54 | flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)") 55 | flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (requires using --read)") 56 | flag.BoolVar(&helpFlag, "help", false, "Print the help message") 57 | flag.BoolVar(&checkConfFlag, "config", false, "Parse and print the config file") 58 | flag.BoolVar(&AddConfigEntryFlag, "addtoconfig", false, "Add entry to the config file") 59 | flag.BoolVar(&doNotExtractFlag, "noextraction", false, "Do not extract malware from archive file.\nCurrently this only effects MalwareBazaar and HybridAnalysis") 60 | flag.BoolVar(&uploadToMWDBFlag, "uploadmwdb", false, "Upload downloaded files to the Upload MWDB instances specified in the mlget.yml file.") 61 | flag.BoolVar(&uploadToAssemblyLineFlag, "uploadal", false, "Upload downloaded files to the Upload AssemblyLine instances specified in the mlget.yml file.") 62 | flag.StringVar(&readFromFileAndUpdateWithNotFoundHashesFlag, "readupdate", "", "Read hashes from file to download. Replace entries in the file with just the hashes that were not found (for next time).") 63 | flag.BoolVar(&uploadToMWDBAndDeleteFlag, "uploaddeletemwdb", false, "Upload downloaded files to the Upload MWDB instance specified in the mlget.yml file.\nDelete the files after successful upload") 64 | flag.BoolVar(&uploadToAssemblyLineAndDeleteFlag, "uploaddeleteal", false, "Upload downloaded files to the Upload AssemblyLine instances specified in the mlget.yml file.\nDelete the files after successful upload") 65 | flag.BoolVar(&forceResubmission, "f", false, "Force resubmission to AssemblyLine when the files already exists on the AssemblyLine instance.") 66 | flag.StringSliceVar(&tagsFlag, "tag", []string{}, "Tag the sample when uploading to your own instance of MWDB.") 67 | flag.StringSliceVar(&commentsFlag, "comment", []string{}, "Add comment to the sample when uploading to your own instance of MWDB.") 68 | flag.BoolVar(&downloadOnlyFlag, "downloadonly", false, "Download from any source, including your personal instance of MWDB.\nWhen this flag is set; it will NOT update any output file with the hashes not found.\nAnd it will not upload to any of the UploadMWDB.") 69 | flag.BoolVar(&versionFlag, "version", false, "Print the version number") 70 | flag.BoolVar(&noValidationFlag, "novalidation", false, "Turn off post download hash check verification") 71 | flag.BoolVar(&precheckdir, "precheckdir", false, "Search current dir for files matching the hashes provided, if found don't redownload") 72 | } 73 | 74 | func main() { 75 | 76 | doNotValidatehash := []MalwareRepoType{ObjectiveSee} 77 | 78 | homeDir, err := os.UserHomeDir() 79 | if err != nil { 80 | fmt.Println(err) 81 | return 82 | } 83 | 84 | configFileName := path.Join(homeDir, ".mlget.yml") 85 | 86 | cfg, err := LoadConfig(configFileName) 87 | if err != nil { 88 | fmt.Println(err) 89 | return 90 | } 91 | 92 | flag.Parse() 93 | 94 | if versionFlag { 95 | fmt.Printf("Mlget version: %s\n", version) 96 | return 97 | } 98 | 99 | if helpFlag { 100 | usage() 101 | return 102 | } 103 | 104 | if AddConfigEntryFlag { 105 | AddToConfig(configFileName) 106 | return 107 | } 108 | 109 | if checkConfFlag { 110 | fmt.Printf("%+v", cfg) 111 | return 112 | } 113 | 114 | args := flag.Args() 115 | /* 116 | if webserver { 117 | runWebServer(ip, port) 118 | } else {*/ 119 | downloadMalwareFromCLI(args, cfg, doNotValidatehash, precheckdir) 120 | // } 121 | 122 | } 123 | 124 | func parseArgHashes(hashes []string, tags []string, comments []string) Hashes { 125 | parsedHashes := Hashes{} 126 | fmt.Printf("Hashes Passed Via the Command Line:\n") 127 | for _, h := range hashes { 128 | ht, err := hashType(h) 129 | if err != nil { 130 | fmt.Printf("\n Skipping %s because it's %s\n", h, err) 131 | continue 132 | } 133 | fmt.Printf(" - %s\n", h) // token in unicode-char 134 | hash := Hash{Hash: h, HashType: ht, Local: false} 135 | if len(tags) > 0 { 136 | hash.Tags = tags 137 | } 138 | if len(comments) > 0 { 139 | hash.Comments = comments 140 | } 141 | parsedHashes, _ = addHash(parsedHashes, hash) 142 | } 143 | fmt.Println("") 144 | return parsedHashes 145 | } 146 | 147 | func downloadMalwareFromCLI(args []string, cfg []RepositoryConfigEntry, doNotValidatehash []MalwareRepoType, precheckdir bool) { 148 | if apiFlag != "" { 149 | flaggedRepo := getMalwareRepoByFlagName(apiFlag) 150 | if flaggedRepo == NotSupported { 151 | fmt.Printf("Invalid or unsupported malware repo type: %s\nCheck the help for the values to pass to the --from parameter\n", apiFlag) 152 | return 153 | } 154 | } 155 | 156 | if apiFlag != "" && downloadOnlyFlag { 157 | fmt.Printf(("Can't use both the --from flag and the --downloadonly flag together")) 158 | return 159 | } 160 | 161 | hashes := parseArgHashes(args, tagsFlag, commentsFlag) 162 | var err error 163 | 164 | if inputFileFlag != "" { 165 | hshs, err := parseFileForHashEntries(inputFileFlag) 166 | if err != nil { 167 | fmt.Printf("Error reading from %s\n", inputFileFlag) 168 | fmt.Println(err) 169 | } else { 170 | for idx, hsh := range hshs { 171 | if len(hshs) > 100 { 172 | fmt.Printf("Adding Hash %d of %d\n", idx, len(hshs)) 173 | } 174 | hashes, _ = addHash(hashes, hsh) 175 | } 176 | } 177 | } 178 | if readFromFileAndUpdateWithNotFoundHashesFlag != "" { 179 | hshs, err := parseFileForHashEntries(readFromFileAndUpdateWithNotFoundHashesFlag) 180 | if err != nil { 181 | fmt.Printf("Error reading from %s\n", readFromFileAndUpdateWithNotFoundHashesFlag) 182 | fmt.Println(err) 183 | } else { 184 | for idx, hsh := range hshs { 185 | if len(hshs) > 100 { 186 | fmt.Printf("Adding Hash %d of %d\n", idx, len(hshs)) 187 | } 188 | hashes, _ = addHash(hashes, hsh) 189 | } 190 | } 191 | } 192 | 193 | var notFoundHashes Hashes 194 | 195 | if len(hashes.Hashes) == 0 { 196 | fmt.Println("No hashes found; displaying Help") 197 | usage() 198 | return 199 | } 200 | 201 | var osq ObjectiveSeeQuery 202 | osConfigs := getConfigsByType(ObjectiveSee, cfg) 203 | // Can have multiple Objective-See configs but only the first one to load will be used 204 | for _, osc := range osConfigs { 205 | osq, err = loadObjectiveSeeJson(osc.Host) 206 | if err != nil { 207 | fmt.Println("Unable to load Objective-See json data. Skipping...") 208 | continue 209 | } 210 | fmt.Println("") 211 | break 212 | } 213 | 214 | if doNotExtractFlag { 215 | doNotValidatehash = append(doNotValidatehash, VxShare, MalwareBazaar, HybridAnalysis, FileScanIo, Triage) 216 | } 217 | 218 | if precheckdir { 219 | files, err := os.ReadDir(".") 220 | if err != nil { 221 | fmt.Println(err) 222 | } 223 | 224 | filesHashing := 0 225 | ch := make(chan Hash, len(files)*3) 226 | var wg sync.WaitGroup 227 | 228 | go func() { 229 | wg.Wait() 230 | close(ch) 231 | }() 232 | 233 | for _, file := range files { 234 | if file.IsDir() { 235 | continue 236 | } 237 | filesHashing++ 238 | wg.Add(1) 239 | 240 | name := file.Name() 241 | 242 | go func() { 243 | defer wg.Done() 244 | hashFileAndCheck(name, ch) 245 | }() 246 | } 247 | 248 | fmt.Println("\nFiles Matching Hashes Found Locally:") 249 | 250 | for i := range ch { 251 | if hashes.hashExists(i.Hash) { 252 | fmt.Printf(" - Found %s in current folder\n", i.Hash) 253 | fmt.Printf(" File Name: %s\n", i.LocalFile) 254 | hashes.updateLocalFile(i.Hash, i.LocalFile) 255 | 256 | h, _ := hashes.getByHash(i.Hash) 257 | 258 | if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { 259 | err := UploadSampleToMWDBs(cfg, h.LocalFile, h, uploadToMWDBAndDeleteFlag) 260 | if err != nil { 261 | fmt.Printf(" ! %s\n", err) 262 | } 263 | } 264 | if (uploadToAssemblyLineFlag || uploadToAssemblyLineAndDeleteFlag) && !downloadOnlyFlag { 265 | err := UploadSampleToAssemblyLine(cfg, h.LocalFile, h, uploadToAssemblyLineAndDeleteFlag, forceResubmission) 266 | if err != nil { 267 | fmt.Printf(" ! %s\n", err) 268 | } 269 | } 270 | } 271 | } 272 | } 273 | 274 | for idx, h := range hashes.Hashes { 275 | 276 | if h.Local { 277 | continue 278 | } 279 | 280 | fmt.Printf("\nLook up %s (%s) - (%d of %d)\n", h.Hash, h.HashType, idx+1, len(hashes.Hashes)) 281 | 282 | if apiFlag != "" { 283 | flaggedRepo := getMalwareRepoByFlagName(apiFlag) 284 | 285 | fmt.Printf("Looking on %s\n", getMalwareRepoByFlagName(apiFlag)) 286 | 287 | found, filename, checkedRepo := flaggedRepo.QueryAndDownload(cfg, h, doNotExtractFlag, osq) 288 | if !found { 289 | fmt.Println(" [!] Not Found") 290 | notFoundHashes, _ = addHash(notFoundHashes, h) 291 | } else if found { 292 | if !noValidationFlag { 293 | if slices.Contains(doNotValidatehash, checkedRepo) { 294 | if checkedRepo == ObjectiveSee { 295 | fmt.Printf(" [!] Not able to validate hash for repo %s\n", checkedRepo.String()) 296 | } else { 297 | fmt.Printf(" [!] Not able to validate hash for repo %s when noextraction flag is set to %t\n", checkedRepo.String(), doNotExtractFlag) 298 | } 299 | } else { 300 | valid, calculatedHash := h.ValidateFile(filename) 301 | if !valid { 302 | fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\n", calculatedHash, h.Hash) 303 | deleteInvalidFile(filename) 304 | notFoundHashes, _ = addHash(notFoundHashes, h) 305 | continue 306 | } else { 307 | fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", h.Hash) 308 | } 309 | } 310 | } 311 | if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { 312 | err := UploadSampleToMWDBs(cfg, filename, h, uploadToMWDBAndDeleteFlag) 313 | if err != nil { 314 | fmt.Printf(" ! %s", err) 315 | } 316 | } 317 | if (uploadToAssemblyLineFlag || uploadToAssemblyLineAndDeleteFlag) && !downloadOnlyFlag { 318 | err := UploadSampleToAssemblyLine(cfg, filename, h, uploadToAssemblyLineAndDeleteFlag, forceResubmission) 319 | if err != nil { 320 | fmt.Printf(" ! %s", err) 321 | } 322 | } 323 | } 324 | 325 | } else { 326 | fmt.Println("Querying all services") 327 | 328 | found, filename, _ := queryAndDownloadAll(cfg, h, doNotExtractFlag, !downloadOnlyFlag, osq, noValidationFlag, doNotValidatehash) 329 | if found { 330 | if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { 331 | err := UploadSampleToMWDBs(cfg, filename, h, uploadToMWDBAndDeleteFlag) 332 | if err != nil { 333 | fmt.Printf(" ! %s", err) 334 | } 335 | } 336 | if (uploadToAssemblyLineFlag || uploadToAssemblyLineAndDeleteFlag) && !downloadOnlyFlag { 337 | err := UploadSampleToAssemblyLine(cfg, filename, h, uploadToAssemblyLineAndDeleteFlag, forceResubmission) 338 | if err != nil { 339 | fmt.Printf(" ! %s", err) 340 | } 341 | } 342 | continue 343 | } 344 | 345 | notFoundHashes, _ = addHash(notFoundHashes, h) 346 | } 347 | } 348 | 349 | if len(notFoundHashes.Hashes) > 0 { 350 | fmt.Printf("\nHashes not found!\n") 351 | for i, s := range notFoundHashes.Hashes { 352 | fmt.Printf(" %d: %s\n", i, s.Hash) 353 | } 354 | } 355 | if !downloadOnlyFlag { 356 | if readFromFileAndUpdateWithNotFoundHashesFlag != "" && !isValidUrl(readFromFileAndUpdateWithNotFoundHashesFlag) { 357 | err := writeUnfoundHashesToFile(readFromFileAndUpdateWithNotFoundHashesFlag, notFoundHashes) 358 | if err != nil { 359 | fmt.Println("Error writing not found hashes to file") 360 | fmt.Println(err) 361 | } 362 | fmt.Printf("\n\n%s refreshed to show only the hashes not found.\n", readFromFileAndUpdateWithNotFoundHashesFlag) 363 | 364 | } else if outputFileFlag && len(notFoundHashes.Hashes) > 0 { 365 | var filename string 366 | if inputFileFlag != "" { 367 | filename = time.Now().Format("2006-01-02__3_4_5__pm__") + inputFileFlag 368 | } else { 369 | filename = time.Now().Format("2006-01-02__3_4_5__pm") + "_not_found_hashes.txt" 370 | } 371 | err := writeUnfoundHashesToFile(filename, notFoundHashes) 372 | if err != nil { 373 | fmt.Println("Error writing unfound hashes to file") 374 | fmt.Println(err) 375 | } 376 | fmt.Printf("\n\nUnfound hashes written to %s\n", filename) 377 | } else if isValidUrl(readFromFileAndUpdateWithNotFoundHashesFlag) { 378 | fmt.Println("File specified is a URL - can't update file. Please use --read and --output flags instead if you want to capture the hashes not found.") 379 | } 380 | } 381 | } 382 | 383 | func hashFileAndCheck(file string, c chan Hash) { 384 | 385 | f, err := os.Open(file) 386 | 387 | if err != nil { 388 | log.Fatal(err) 389 | } 390 | defer f.Close() 391 | 392 | hasherMD5 := crypto.MD5.New() 393 | if _, err := io.Copy(hasherMD5, f); err != nil { 394 | log.Fatal(err) 395 | } 396 | sumMD5 := hasherMD5.Sum(nil) 397 | 398 | c <- Hash{Hash: fmt.Sprintf("%x", sumMD5), HashType: md5, LocalFile: file} 399 | 400 | f.Seek(0, 0) 401 | hasherSHA1 := crypto.SHA1.New() 402 | if _, err := io.Copy(hasherSHA1, f); err != nil { 403 | log.Fatal(err) 404 | } 405 | sumSHA1 := hasherSHA1.Sum(nil) 406 | 407 | c <- Hash{Hash: fmt.Sprintf("%x", sumSHA1), HashType: sha1, LocalFile: file} 408 | 409 | f.Seek(0, 0) 410 | hasherSHA256 := crypto.SHA256.New() 411 | if _, err := io.Copy(hasherSHA256, f); err != nil { 412 | log.Fatal(err) 413 | } 414 | sumSHA256 := hasherSHA256.Sum(nil) 415 | 416 | c <- Hash{Hash: fmt.Sprintf("%x", sumSHA256), HashType: sha256, LocalFile: file} 417 | } 418 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "sort" 8 | "strconv" 9 | "strings" 10 | 11 | "golang.org/x/exp/slices" 12 | "gopkg.in/yaml.v2" 13 | ) 14 | 15 | type MalwareRepoType int64 16 | 17 | const ( 18 | NotSupported MalwareRepoType = iota //NotSupported must always be first, or other things won't work as expected 19 | 20 | AssemblyLine 21 | CapeSandbox 22 | FileScanIo 23 | HybridAnalysis 24 | InQuest 25 | JoeSandbox 26 | Malpedia 27 | Malshare 28 | MalwareBazaar 29 | MWDB 30 | ObjectiveSee 31 | Polyswarm 32 | Triage 33 | UnpacMe 34 | URLScanIO 35 | VirusExchange 36 | VirusTotal 37 | VxShare 38 | 39 | UploadAssemblyLine 40 | //UploadMWDB must always be last, or other things won't work as expected 41 | UploadMWDB 42 | ) 43 | 44 | // var MalwareRepoList = []MalwareRepoType{CapeSandbox, HybridAnalysis, InQuest, JoeSandbox, Malpedia, Malshare, MalwareBazaar, MWDB, ObjectiveSee, Polyswarm, Triage, UnpacMe, VirusTotal, UploadMWDB} 45 | func getMalwareRepoList() []MalwareRepoType { 46 | var malwareRepoList []MalwareRepoType 47 | for repo := range [UploadMWDB + 1]int64{} { 48 | if int64(repo) > int64(NotSupported) && int64(repo) <= int64(UploadMWDB) { 49 | malwareRepoList = append(malwareRepoList, MalwareRepoType(repo)) 50 | } 51 | } 52 | return malwareRepoList 53 | } 54 | 55 | func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, osq ObjectiveSeeQuery) (bool, string, MalwareRepoType) { 56 | matchingConfigRepos := getConfigsByType(malrepo, repos) 57 | if len(matchingConfigRepos) == 0 { 58 | fmt.Printf(" [!] %s is not found in the yml config file\n", malrepo) 59 | } 60 | for _, mcr := range matchingConfigRepos { 61 | found := false 62 | filename := "" 63 | checkedRepo := NotSupported 64 | fmt.Printf(" [*] %s: %s\n", mcr.Type, mcr.Host) 65 | switch malrepo { 66 | case MalwareBazaar: 67 | found, filename = malwareBazaar(mcr.Host, mcr.Api, hash, doNotExtract, "infected") 68 | checkedRepo = MalwareBazaar 69 | case MWDB: 70 | found, filename = mwdb(mcr.Host, mcr.Api, hash) 71 | checkedRepo = MWDB 72 | case Malshare: 73 | found, filename = malshare(mcr.Host, mcr.Api, hash) 74 | checkedRepo = Malshare 75 | case Triage: 76 | found, filename = triage(mcr.Host, mcr.Api, hash) 77 | checkedRepo = Triage 78 | case InQuest: 79 | found, filename = inquestlabs(mcr.Host, mcr.Api, hash) 80 | checkedRepo = InQuest 81 | case HybridAnalysis: 82 | found, filename = hybridAnalysis(mcr.Host, mcr.Api, hash) 83 | checkedRepo = HybridAnalysis 84 | case Polyswarm: 85 | found, filename = polyswarm(mcr.Host, mcr.Api, hash) 86 | checkedRepo = Polyswarm 87 | case VirusTotal: 88 | found, filename = virustotal(mcr.Host, mcr.Api, hash) 89 | checkedRepo = VirusTotal 90 | case JoeSandbox: 91 | found, filename = joesandbox(mcr.Host, mcr.Api, hash) 92 | checkedRepo = JoeSandbox 93 | case CapeSandbox: 94 | found, filename = capesandbox(mcr.Host, mcr.Api, hash) 95 | checkedRepo = CapeSandbox 96 | case ObjectiveSee: 97 | if len(osq.Malware) > 0 { 98 | found, filename = objectivesee(osq, hash, doNotExtract) 99 | checkedRepo = ObjectiveSee 100 | } 101 | case UnpacMe: 102 | found, filename = unpacme(mcr.Host, mcr.Api, hash) 103 | checkedRepo = UnpacMe 104 | case Malpedia: 105 | found, filename = malpedia(mcr.Host, mcr.Api, hash) 106 | checkedRepo = Malpedia 107 | case VxShare: 108 | found, filename = vxshare(mcr.Host, mcr.Api, hash, doNotExtract, "infected") 109 | checkedRepo = VxShare 110 | case FileScanIo: 111 | found, filename = filescanio(mcr.Host, mcr.Api, hash, doNotExtract, "infected") 112 | checkedRepo = FileScanIo 113 | case URLScanIO: 114 | found, filename = urlscanio(mcr.Host, mcr.Api, hash) 115 | checkedRepo = URLScanIO 116 | case VirusExchange: 117 | found, filename = virusexchange(mcr.Host, mcr.Api, hash) 118 | checkedRepo = VirusExchange 119 | case AssemblyLine: 120 | found, filename = assemblyline(mcr.Host, mcr.User, mcr.Api, mcr.IgnoreTLSErrors, hash) 121 | checkedRepo = AssemblyLine 122 | case UploadAssemblyLine: 123 | found, filename = assemblyline(mcr.Host, mcr.User, mcr.Api, mcr.IgnoreTLSErrors, hash) 124 | checkedRepo = UploadAssemblyLine 125 | case UploadMWDB: 126 | found, filename = mwdb(mcr.Host, mcr.Api, hash) 127 | checkedRepo = UploadMWDB 128 | } 129 | 130 | if found { 131 | return found, filename, checkedRepo 132 | } 133 | } 134 | return false, "", NotSupported 135 | } 136 | 137 | func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool { 138 | switch malrepo { 139 | case NotSupported: 140 | return false 141 | case ObjectiveSee: 142 | if repo.Host != "" { 143 | return true 144 | } 145 | case AssemblyLine: 146 | if repo.Host != "" && repo.Api != "" && repo.User != "" { 147 | return true 148 | } 149 | case UploadAssemblyLine: 150 | if repo.Host != "" && repo.Api != "" && repo.User != "" { 151 | return true 152 | } 153 | default: 154 | if repo.Host != "" && repo.Api != "" { 155 | return true 156 | } 157 | } 158 | return false 159 | } 160 | 161 | func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { 162 | var host string 163 | var api string 164 | var user string 165 | tls := false 166 | 167 | var default_url string 168 | 169 | switch malrepo { 170 | case NotSupported: 171 | return RepositoryConfigEntry{}, fmt.Errorf("malware repository rype, %s, is not supported", malrepo.String()) 172 | case MalwareBazaar: 173 | default_url = "https://mb-api.abuse.ch/api/v1/" 174 | case Malshare: 175 | default_url = "https://malshare.com" 176 | case MWDB: 177 | default_url = "https://mwdb.cert.pl/api" 178 | case CapeSandbox: 179 | default_url = "" 180 | case JoeSandbox: 181 | default_url = "https://jbxcloud.joesecurity.org/api/v2" 182 | case InQuest: 183 | default_url = "https://labs.inquest.net/api" 184 | case HybridAnalysis: 185 | default_url = "https://www.hybrid-analysis.com/api/v2" 186 | case Triage: 187 | default_url = "https://api.tria.ge/v0" 188 | case VirusTotal: 189 | default_url = "https://www.virustotal.com/api/v3" 190 | case Polyswarm: 191 | default_url = "https://api.polyswarm.network/v3" 192 | case ObjectiveSee: 193 | default_url = "https://objective-see.com/malware.json" 194 | case UnpacMe: 195 | default_url = "https://api.unpac.me/api/v1" 196 | case Malpedia: 197 | default_url = "https://malpedia.caad.fkie.fraunhofer.de/api" 198 | case VxShare: 199 | default_url = "https://virusshare.com/apiv2" 200 | case FileScanIo: 201 | default_url = "https://www.filescan.io/api" 202 | case URLScanIO: 203 | default_url = "https://urlscan.io/downloads" 204 | case VirusExchange: 205 | default_url = "https://virus.exchange/api" 206 | } 207 | if default_url != "" { 208 | fmt.Printf("Enter Host [ Press enter for default - %s ]:\n", default_url) 209 | } else { 210 | fmt.Printf("Enter Host:\n") 211 | } 212 | fmt.Print(">> ") 213 | fmt.Scanln(&host) 214 | if host == "" { 215 | fmt.Println("Using the default url") 216 | host = default_url 217 | } 218 | if malrepo == AssemblyLine || malrepo == UploadAssemblyLine { 219 | fmt.Println("Enter User Name:") 220 | fmt.Print(">> ") 221 | fmt.Scanln(&user) 222 | for { 223 | fmt.Println("Disable TLS Verification (true|false):") 224 | fmt.Print(">> ") 225 | var tlss string 226 | fmt.Scanln(&tlss) 227 | boolvalue, err := strconv.ParseBool(tlss) 228 | if err == nil { 229 | tls = boolvalue 230 | break 231 | } 232 | fmt.Println("Invalid option entered") 233 | } 234 | } 235 | if malrepo != ObjectiveSee { 236 | fmt.Println("Enter API Key:") 237 | fmt.Print(">> ") 238 | fmt.Scanln(&api) 239 | } 240 | return RepositoryConfigEntry{Host: host, User: user, Api: api, Type: malrepo.String(), IgnoreTLSErrors: tls}, nil 241 | } 242 | 243 | func (malrepo MalwareRepoType) String() string { 244 | switch malrepo { 245 | case JoeSandbox: 246 | return "JoeSandbox" 247 | case MWDB: 248 | return "MWDB" 249 | case HybridAnalysis: 250 | return "HybridAnalysis" 251 | case CapeSandbox: 252 | return "CapeSandbox" 253 | case InQuest: 254 | return "InQuest" 255 | case MalwareBazaar: 256 | return "MalwareBazaar" 257 | case Triage: 258 | return "Triage" 259 | case Malshare: 260 | return "Malshare" 261 | case VirusTotal: 262 | return "VirusTotal" 263 | case Polyswarm: 264 | return "Polyswarm" 265 | case ObjectiveSee: 266 | return "ObjectiveSee" 267 | case UnpacMe: 268 | return "UnpacMe" 269 | case Malpedia: 270 | return "Malpedia" 271 | case VxShare: 272 | return "VxShare" 273 | case FileScanIo: 274 | return "FileScanIo" 275 | case URLScanIO: 276 | return "URLScanIO" 277 | case AssemblyLine: 278 | return "AssemblyLine" 279 | case UploadAssemblyLine: 280 | return "UploadAssemblyLine" 281 | case UploadMWDB: 282 | return "UploadMWDB" 283 | case VirusExchange: 284 | return "VirusExchange" 285 | } 286 | return "NotSupported" 287 | } 288 | 289 | func allowedMalwareRepoTypes() { 290 | for _, mr := range getMalwareRepoList() { 291 | fmt.Printf(" %s\n", mr.String()) 292 | } 293 | } 294 | 295 | func printAllowedMalwareRepoTypeOptions() { 296 | fmt.Println("") 297 | for _, mr := range getMalwareRepoList() { 298 | fmt.Printf(" [%d] %s\n", mr, mr.String()) 299 | } 300 | } 301 | 302 | func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, skipUpload bool, osq ObjectiveSeeQuery, doNotValidateHash bool, doNotValidateHashList []MalwareRepoType) (bool, string, MalwareRepoType) { 303 | found := false 304 | filename := "" 305 | checkedRepo := NotSupported 306 | sort.Slice(repos[:], func(i, j int) bool { 307 | return repos[i].QueryOrder < repos[j].QueryOrder 308 | }) 309 | 310 | // Hack for now 311 | // Due to Multiple entries of the same type, for each type instance in the config it will 312 | // try to download for type the number of entries for type in config squared 313 | // This array is meant to ensure that for each type it will only try it once 314 | var completedTypes []MalwareRepoType 315 | 316 | for _, repo := range repos { 317 | if (repo.Type == UploadMWDB.String() || repo.Type == UploadAssemblyLine.String()) && skipUpload { 318 | continue 319 | } 320 | mr := getMalwareRepoByName(repo.Type) 321 | if !contains(completedTypes, mr) { 322 | found, filename, checkedRepo = mr.QueryAndDownload(repos, hash, doNotExtract, osq) 323 | if found { 324 | if !doNotValidateHash { 325 | if slices.Contains(doNotValidateHashList, checkedRepo) { 326 | if checkedRepo == ObjectiveSee { 327 | fmt.Printf(" [!] Not able to validate hash for repo %s\n", checkedRepo.String()) 328 | } else { 329 | fmt.Printf(" [!] Not able to validate hash for repo %s when noextraction flag is set to %t\n", checkedRepo.String(), doNotExtractFlag) 330 | } 331 | break 332 | } else { 333 | valid, calculatedHash := hash.ValidateFile(filename) 334 | if !valid { 335 | fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\nTrying another source.\n", calculatedHash, hash.Hash) 336 | deleteInvalidFile(filename) 337 | continue 338 | } else { 339 | fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", hash.Hash) 340 | break 341 | } 342 | } 343 | } 344 | break 345 | } 346 | completedTypes = append(completedTypes, mr) 347 | } 348 | } 349 | return found, filename, checkedRepo 350 | } 351 | 352 | func getMalwareRepoByFlagName(name string) MalwareRepoType { 353 | switch strings.ToLower(name) { 354 | case strings.ToLower("js"): 355 | return JoeSandbox 356 | case strings.ToLower("md"): 357 | return MWDB 358 | case strings.ToLower("ha"): 359 | return HybridAnalysis 360 | case strings.ToLower("cs"): 361 | return CapeSandbox 362 | case strings.ToLower("iq"): 363 | return InQuest 364 | case strings.ToLower("mb"): 365 | return MalwareBazaar 366 | case strings.ToLower("tr"): 367 | return Triage 368 | case strings.ToLower("ms"): 369 | return Malshare 370 | case strings.ToLower("vt"): 371 | return VirusTotal 372 | case strings.ToLower("ps"): 373 | return Polyswarm 374 | case strings.ToLower("os"): 375 | return ObjectiveSee 376 | case strings.ToLower("um"): 377 | return UnpacMe 378 | case strings.ToLower("mp"): 379 | return Malpedia 380 | case strings.ToLower("vx"): 381 | return VxShare 382 | case strings.ToLower("fs"): 383 | return FileScanIo 384 | case strings.ToLower("us"): 385 | return URLScanIO 386 | case strings.ToLower("al"): 387 | return AssemblyLine 388 | case strings.ToLower("ve"): 389 | return VirusExchange 390 | } 391 | return NotSupported 392 | } 393 | 394 | func getMalwareRepoByName(name string) MalwareRepoType { 395 | switch strings.ToLower(name) { 396 | case strings.ToLower("JoeSandbox"): 397 | return JoeSandbox 398 | case strings.ToLower("MWDB"): 399 | return MWDB 400 | case strings.ToLower("HybridAnalysis"): 401 | return HybridAnalysis 402 | case strings.ToLower("CapeSandbox"): 403 | return CapeSandbox 404 | case strings.ToLower("InQuest"): 405 | return InQuest 406 | case strings.ToLower("MalwareBazaar"): 407 | return MalwareBazaar 408 | case strings.ToLower("Triage"): 409 | return Triage 410 | case strings.ToLower("Malshare"): 411 | return Malshare 412 | case strings.ToLower("VirusTotal"): 413 | return VirusTotal 414 | case strings.ToLower("Polyswarm"): 415 | return Polyswarm 416 | case strings.ToLower("ObjectiveSee"): 417 | return ObjectiveSee 418 | case strings.ToLower("UnpacMe"): 419 | return UnpacMe 420 | case strings.ToLower("Malpedia"): 421 | return Malpedia 422 | case strings.ToLower("VxShare"): 423 | return VxShare 424 | case strings.ToLower("FileScanIo"): 425 | return FileScanIo 426 | case strings.ToLower("URLScanIO"): 427 | return URLScanIO 428 | case strings.ToLower("AssemblyLine"): 429 | return AssemblyLine 430 | case strings.ToLower("UploadAssemblyLine"): 431 | return UploadAssemblyLine 432 | case strings.ToLower("UploadMWDB"): 433 | return UploadMWDB 434 | case strings.ToLower("VirusExchange"): 435 | return VirusExchange 436 | } 437 | return NotSupported 438 | } 439 | 440 | func getConfigsByType(repoType MalwareRepoType, repos []RepositoryConfigEntry) []RepositoryConfigEntry { 441 | var filteredRepos []RepositoryConfigEntry 442 | for _, v := range repos { 443 | if v.Type == repoType.String() { 444 | filteredRepos = append(filteredRepos, v) 445 | } 446 | } 447 | return filteredRepos 448 | } 449 | 450 | type RepositoryConfigEntry struct { 451 | Type string `yaml:"type"` 452 | Host string `yaml:"url"` 453 | Api string `yaml:"api"` 454 | QueryOrder int `yaml:"queryorder"` 455 | Password string `yaml:"pwd"` 456 | User string `yaml:"user"` 457 | IgnoreTLSErrors bool `yaml:"ignoretlserrors"` 458 | } 459 | 460 | func LoadConfig(filename string) ([]RepositoryConfigEntry, error) { 461 | cfg, err := parseFile(filename) 462 | if os.IsNotExist(err) { 463 | fmt.Printf("%s does not exists. Creating...\n", filename) 464 | filename, err = initConfig(filename) 465 | if err != nil { 466 | log.Fatal(err) 467 | return nil, err 468 | } 469 | cfg, err = parseFile(filename) 470 | if err != nil { 471 | log.Fatal(err) 472 | return nil, err 473 | } 474 | } else if err != nil { 475 | log.Fatal(err) 476 | return nil, err 477 | } 478 | return verifyConfig(cfg) 479 | } 480 | 481 | func verifyConfig(repos map[string]RepositoryConfigEntry) ([]RepositoryConfigEntry, error) { 482 | var verifiedConfigRepos []RepositoryConfigEntry 483 | 484 | for k, v := range repos { 485 | mr := getMalwareRepoByName(v.Type) 486 | if mr == NotSupported { 487 | fmt.Printf("%s is not a supported type. Skipping...\n\nSupported types include:\n", v.Type) 488 | allowedMalwareRepoTypes() 489 | fmt.Println("") 490 | } else { 491 | valid := mr.VerifyRepoParams(v) 492 | if !valid { 493 | fmt.Printf(" Skipping %s (Type: %s, URL: %s, API: %s) as it's missing a parameter.\n", k, v.Type, v.Host, v.Api) 494 | } else { 495 | verifiedConfigRepos = append(verifiedConfigRepos, v) 496 | } 497 | } 498 | } 499 | return verifiedConfigRepos, nil 500 | } 501 | 502 | func parseFile(path string) (map[string]RepositoryConfigEntry, error) { 503 | 504 | _, err := os.Stat(path) 505 | if os.IsNotExist(err) { 506 | return nil, err 507 | } 508 | 509 | f, err := os.ReadFile(path) 510 | if err != nil { 511 | fmt.Printf("%v", err) 512 | return nil, err 513 | } 514 | 515 | data := make(map[string]RepositoryConfigEntry) 516 | 517 | err = yaml.Unmarshal(f, &data) 518 | if err != nil { 519 | fmt.Printf("%v", err) 520 | return nil, err 521 | } 522 | 523 | return data, nil 524 | } 525 | 526 | func AddToConfig(filename string) (string, error) { 527 | repoConfigEntries, err := parseFile(filename) 528 | if err != nil { 529 | fmt.Printf("Error parsing %s - %v", filename, err) 530 | return "", err 531 | } 532 | 533 | data, err := createNewEntries(0) 534 | if err != nil { 535 | fmt.Printf("Error creating new config entries : %v", err) 536 | return "", err 537 | } 538 | 539 | finalConfigEntryList := make(map[string]RepositoryConfigEntry) 540 | 541 | entryNumber := 0 542 | 543 | // Add items from reporConfigEntries (pre-existing items) to the final list 544 | for _, v := range repoConfigEntries { 545 | finalConfigEntryList["repository "+fmt.Sprint(entryNumber)] = v 546 | entryNumber++ 547 | } 548 | 549 | // Add items not found in repoConfigEnties (the pre-existing items) 550 | for _, v1 := range data { 551 | found := false 552 | for _, v2 := range repoConfigEntries { 553 | if v1.Type == v2.Type && v1.Host == v1.Api { 554 | found = true 555 | } 556 | } 557 | if !found { 558 | finalConfigEntryList["repository "+fmt.Sprint(entryNumber)] = v1 559 | entryNumber++ 560 | } 561 | } 562 | return writeConfigToFile(filename, finalConfigEntryList) 563 | } 564 | 565 | func initConfig(filename string) (string, error) { 566 | data, err := createNewEntries(0) 567 | if err != nil { 568 | fmt.Printf("Error creating new Repository Config Entries: %v\n", err) 569 | return "", err 570 | } 571 | return writeConfigToFile(filename, data) 572 | } 573 | 574 | func createNewEntries(entryNumber int) (map[string]RepositoryConfigEntry, error) { 575 | data := make(map[string]RepositoryConfigEntry) 576 | 577 | var option int64 578 | 579 | for { 580 | 581 | fmt.Printf("\nEnter the corresponding Repository Config Entry number you want to add to .mlget.yml.\n") 582 | fmt.Printf("Enter 0 to exit.\n") 583 | printAllowedMalwareRepoTypeOptions() 584 | 585 | fmt.Print(">> ") 586 | 587 | fmt.Scan(&option) 588 | 589 | if option > int64(NotSupported) && option <= int64(UploadMWDB) { 590 | entry, err := MalwareRepoType(option).CreateEntry() 591 | if err != nil { 592 | continue 593 | } 594 | data["repository "+fmt.Sprint(entryNumber)] = entry 595 | entryNumber++ 596 | } else if option == 0 { 597 | break 598 | } 599 | } 600 | return data, nil 601 | } 602 | 603 | func writeConfigToFile(filename string, repoConfigEntries map[string]RepositoryConfigEntry) (string, error) { 604 | _, err := os.Stat(filename) 605 | if err == nil { 606 | os.Remove(filename) 607 | } 608 | 609 | file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0600) 610 | if err != nil { 611 | fmt.Printf("error creating file: %v\n", err) 612 | return "", err 613 | } 614 | defer file.Close() 615 | 616 | enc := yaml.NewEncoder(file) 617 | 618 | err = enc.Encode(repoConfigEntries) 619 | if err != nil { 620 | fmt.Printf("error encoding: %v\n", err) 621 | return "", err 622 | } else { 623 | fmt.Printf("Config written to %s\n\n", file.Name()) 624 | } 625 | 626 | return file.Name(), nil 627 | } 628 | 629 | func contains(list []MalwareRepoType, x MalwareRepoType) bool { 630 | for _, item := range list { 631 | if item == x { 632 | return true 633 | } 634 | } 635 | return false 636 | } 637 | -------------------------------------------------------------------------------- /mlget_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path" 9 | "testing" 10 | 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | type TestConfigEntry struct { 15 | Name string `yaml:"name"` 16 | Hash string `yaml:"hash"` 17 | } 18 | 19 | func parseTestConfig(path string, testName string) (TestConfigEntry, error) { 20 | var tce TestConfigEntry 21 | 22 | _, err := os.Stat(path) 23 | if os.IsNotExist(err) { 24 | return tce, err 25 | } 26 | 27 | f, err := os.ReadFile(path) 28 | if err != nil { 29 | fmt.Printf("%v", err) 30 | return tce, err 31 | } 32 | 33 | data := make(map[string]TestConfigEntry) 34 | 35 | err = yaml.Unmarshal(f, &data) 36 | if err != nil { 37 | fmt.Printf("%v", err) 38 | return tce, err 39 | } 40 | 41 | var filteredTestConfig []TestConfigEntry 42 | for _, v := range data { 43 | if v.Name == testName { 44 | filteredTestConfig = append(filteredTestConfig, v) 45 | } 46 | } 47 | 48 | if len(filteredTestConfig) != 1 { 49 | return tce, errors.New("No test config entry found") 50 | } 51 | return filteredTestConfig[0], nil 52 | } 53 | 54 | func TestJoeSandbox(t *testing.T) { 55 | home, _ := os.UserHomeDir() 56 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 57 | if err != nil { 58 | log.Fatal() 59 | t.Errorf("%v", err) 60 | } 61 | 62 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 63 | if err != nil { 64 | log.Fatal() 65 | t.Errorf("%v", err) 66 | } 67 | 68 | ht, _ := hashType(scfg.Hash) 69 | hash := Hash{HashType: ht, Hash: scfg.Hash} 70 | 71 | var osq ObjectiveSeeQuery 72 | result, filename, _ := JoeSandbox.QueryAndDownload(cfg, hash, false, osq) 73 | 74 | if !result { 75 | t.Errorf("JoeSandbox failed") 76 | } else { 77 | valid, errmsg := hash.ValidateFile(filename) 78 | 79 | if !valid { 80 | os.Remove(hash.Hash) 81 | t.Errorf(errmsg) 82 | } else { 83 | os.Remove(hash.Hash) 84 | } 85 | } 86 | } 87 | 88 | func TestObjectiveSee(t *testing.T) { 89 | home, _ := os.UserHomeDir() 90 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 91 | if err != nil { 92 | log.Fatal() 93 | t.Errorf("%v", err) 94 | } 95 | 96 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 97 | if err != nil { 98 | log.Fatal() 99 | t.Errorf("%v", err) 100 | } 101 | 102 | ht, _ := hashType(scfg.Hash) 103 | hash := Hash{HashType: ht, Hash: scfg.Hash} 104 | 105 | osq, _ := loadObjectiveSeeJson(getConfigsByType(ObjectiveSee, cfg)[0].Host) 106 | result, _, _ := ObjectiveSee.QueryAndDownload(cfg, hash, true, osq) 107 | 108 | if !result { 109 | t.Errorf("Objective-See failed") 110 | } else { 111 | os.Remove(hash.Hash) 112 | } 113 | } 114 | 115 | func TestCapeSandbox(t *testing.T) { 116 | home, _ := os.UserHomeDir() 117 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 118 | if err != nil { 119 | log.Fatal() 120 | t.Errorf("%v", err) 121 | } 122 | 123 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 124 | if err != nil { 125 | log.Fatal() 126 | t.Errorf("%v", err) 127 | } 128 | 129 | ht, _ := hashType(scfg.Hash) 130 | hash := Hash{HashType: ht, Hash: scfg.Hash} 131 | 132 | var osq ObjectiveSeeQuery 133 | result, filename, _ := CapeSandbox.QueryAndDownload(cfg, hash, false, osq) 134 | 135 | if !result { 136 | t.Errorf("CapeSandbox failed") 137 | } else { 138 | valid, errmsg := hash.ValidateFile(filename) 139 | 140 | if !valid { 141 | os.Remove(hash.Hash) 142 | t.Errorf(errmsg) 143 | } else { 144 | os.Remove(hash.Hash) 145 | } 146 | } 147 | } 148 | 149 | func TestInquestLabsLookUp(t *testing.T) { 150 | home, _ := os.UserHomeDir() 151 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 152 | if err != nil { 153 | log.Fatal() 154 | t.Errorf("%v", err) 155 | } 156 | 157 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 158 | if err != nil { 159 | log.Fatal() 160 | t.Errorf("%v", err) 161 | } 162 | 163 | ht, _ := hashType(scfg.Hash) 164 | hash := Hash{HashType: ht, Hash: scfg.Hash} 165 | 166 | var osq ObjectiveSeeQuery 167 | result, filename, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) 168 | 169 | if !result { 170 | t.Errorf("InquestLabs failed") 171 | } else { 172 | valid, errmsg := hash.ValidateFile(filename) 173 | 174 | if !valid { 175 | os.Remove(hash.Hash) 176 | t.Errorf(errmsg) 177 | } else { 178 | os.Remove(hash.Hash) 179 | } 180 | } 181 | } 182 | 183 | func TestInquestLabsNoLookUp(t *testing.T) { 184 | home, _ := os.UserHomeDir() 185 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 186 | if err != nil { 187 | log.Fatal() 188 | t.Errorf("%v", err) 189 | } 190 | 191 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 192 | if err != nil { 193 | log.Fatal() 194 | t.Errorf("%v", err) 195 | } 196 | 197 | ht, _ := hashType(scfg.Hash) 198 | hash := Hash{HashType: ht, Hash: scfg.Hash} 199 | 200 | var osq ObjectiveSeeQuery 201 | result, filename, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) 202 | 203 | if !result { 204 | t.Errorf("InquestLabs failed") 205 | } else { 206 | valid, errmsg := hash.ValidateFile(filename) 207 | 208 | if !valid { 209 | os.Remove(hash.Hash) 210 | t.Errorf(errmsg) 211 | } else { 212 | os.Remove(hash.Hash) 213 | } 214 | } 215 | } 216 | 217 | func TestVirusTotal(t *testing.T) { 218 | home, _ := os.UserHomeDir() 219 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 220 | if err != nil { 221 | log.Fatal() 222 | t.Errorf("%v", err) 223 | } 224 | 225 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 226 | if err != nil { 227 | log.Fatal() 228 | t.Errorf("%v", err) 229 | } 230 | 231 | ht, _ := hashType(scfg.Hash) 232 | hash := Hash{HashType: ht, Hash: scfg.Hash} 233 | 234 | var osq ObjectiveSeeQuery 235 | result, filename, _ := VirusTotal.QueryAndDownload(cfg, hash, false, osq) 236 | 237 | if !result { 238 | t.Errorf("VirusTotal failed") 239 | } else { 240 | valid, errmsg := hash.ValidateFile(filename) 241 | 242 | if !valid { 243 | os.Remove(hash.Hash) 244 | t.Errorf(errmsg) 245 | } else { 246 | os.Remove(hash.Hash) 247 | } 248 | } 249 | } 250 | 251 | func TestMWDB(t *testing.T) { 252 | home, _ := os.UserHomeDir() 253 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 254 | if err != nil { 255 | log.Fatal() 256 | t.Errorf("%v", err) 257 | } 258 | 259 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 260 | if err != nil { 261 | log.Fatal() 262 | t.Errorf("%v", err) 263 | } 264 | 265 | ht, _ := hashType(scfg.Hash) 266 | hash := Hash{HashType: ht, Hash: scfg.Hash} 267 | 268 | var osq ObjectiveSeeQuery 269 | result, filename, _ := MWDB.QueryAndDownload(cfg, hash, false, osq) 270 | 271 | if !result { 272 | t.Errorf("MWDB failed") 273 | } else { 274 | valid, errmsg := hash.ValidateFile(filename) 275 | 276 | if !valid { 277 | os.Remove(hash.Hash) 278 | t.Errorf(errmsg) 279 | } else { 280 | os.Remove(hash.Hash) 281 | } 282 | } 283 | } 284 | 285 | func TestPolyswarm(t *testing.T) { 286 | home, _ := os.UserHomeDir() 287 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 288 | if err != nil { 289 | log.Fatal() 290 | t.Errorf("%v", err) 291 | } 292 | 293 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 294 | if err != nil { 295 | log.Fatal() 296 | t.Errorf("%v", err) 297 | } 298 | 299 | ht, _ := hashType(scfg.Hash) 300 | hash := Hash{HashType: ht, Hash: scfg.Hash} 301 | 302 | var osq ObjectiveSeeQuery 303 | result, filename, _ := Polyswarm.QueryAndDownload(cfg, hash, false, osq) 304 | 305 | if !result { 306 | t.Errorf("PolySwarm failed") 307 | } else { 308 | valid, errmsg := hash.ValidateFile(filename) 309 | 310 | if !valid { 311 | os.Remove(hash.Hash) 312 | t.Errorf(errmsg) 313 | } else { 314 | os.Remove(hash.Hash) 315 | } 316 | } 317 | } 318 | 319 | func TestHybridAnalysis(t *testing.T) { 320 | home, _ := os.UserHomeDir() 321 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 322 | if err != nil { 323 | log.Fatal() 324 | t.Errorf("%v", err) 325 | } 326 | 327 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 328 | if err != nil { 329 | log.Fatal() 330 | t.Errorf("%v", err) 331 | } 332 | 333 | ht, _ := hashType(scfg.Hash) 334 | hash := Hash{HashType: ht, Hash: scfg.Hash} 335 | 336 | var osq ObjectiveSeeQuery 337 | result, filename, _ := HybridAnalysis.QueryAndDownload(cfg, hash, false, osq) 338 | 339 | if !result { 340 | t.Errorf("HybridAnalysis failed") 341 | } else { 342 | valid, errmsg := hash.ValidateFile(filename) 343 | 344 | if !valid { 345 | os.Remove(hash.Hash) 346 | t.Errorf(errmsg) 347 | } else { 348 | os.Remove(hash.Hash) 349 | } 350 | } 351 | } 352 | 353 | func TestHybridAnalysisNotFound(t *testing.T) { 354 | home, _ := os.UserHomeDir() 355 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 356 | if err != nil { 357 | log.Fatal() 358 | t.Errorf("%v", err) 359 | } 360 | 361 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 362 | if err != nil { 363 | log.Fatal() 364 | t.Errorf("%v", err) 365 | } 366 | 367 | ht, _ := hashType(scfg.Hash) 368 | hash := Hash{HashType: ht, Hash: scfg.Hash} 369 | 370 | var osq ObjectiveSeeQuery 371 | result, _, _ := HybridAnalysis.QueryAndDownload(cfg, hash, false, osq) 372 | 373 | if result { 374 | t.Errorf("HybridAnalysis failed") 375 | } 376 | } 377 | 378 | func TestTriage(t *testing.T) { 379 | home, _ := os.UserHomeDir() 380 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 381 | if err != nil { 382 | log.Fatal() 383 | t.Errorf("%v", err) 384 | } 385 | 386 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 387 | if err != nil { 388 | log.Fatal() 389 | t.Errorf("%v", err) 390 | } 391 | 392 | ht, _ := hashType(scfg.Hash) 393 | hash := Hash{HashType: ht, Hash: scfg.Hash} 394 | 395 | var osq ObjectiveSeeQuery 396 | result, filename, _ := Triage.QueryAndDownload(cfg, hash, false, osq) 397 | 398 | if !result { 399 | t.Errorf("Triage failed") 400 | } else { 401 | valid, errmsg := hash.ValidateFile(filename) 402 | 403 | if !valid { 404 | os.Remove(hash.Hash) 405 | t.Errorf(errmsg) 406 | } else { 407 | os.Remove(hash.Hash) 408 | } 409 | } 410 | } 411 | 412 | func TestTriageV2(t *testing.T) { 413 | home, _ := os.UserHomeDir() 414 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 415 | if err != nil { 416 | log.Fatal() 417 | t.Errorf("%v", err) 418 | } 419 | 420 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 421 | if err != nil { 422 | log.Fatal() 423 | t.Errorf("%v", err) 424 | } 425 | 426 | ht, _ := hashType(scfg.Hash) 427 | hash := Hash{HashType: ht, Hash: scfg.Hash} 428 | 429 | var osq ObjectiveSeeQuery 430 | result, filename, _ := Triage.QueryAndDownload(cfg, hash, true, osq) 431 | 432 | if !result { 433 | t.Errorf("Triage failed") 434 | } else { 435 | if filename == "0d8d46ec44e737e6ef6cd7df8edf95d83807e84be825ef76089307b399a6bcbb" { 436 | os.Remove(hash.Hash) 437 | } else { 438 | os.Remove(hash.Hash) 439 | t.Errorf("File name not found") 440 | } 441 | } 442 | } 443 | 444 | func TestMalShare(t *testing.T) { 445 | home, _ := os.UserHomeDir() 446 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 447 | if err != nil { 448 | log.Fatal() 449 | t.Errorf("%v", err) 450 | } 451 | 452 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 453 | if err != nil { 454 | log.Fatal() 455 | t.Errorf("%v", err) 456 | } 457 | 458 | ht, _ := hashType(scfg.Hash) 459 | hash := Hash{HashType: ht, Hash: scfg.Hash} 460 | 461 | var osq ObjectiveSeeQuery 462 | result, filename, _ := Malshare.QueryAndDownload(cfg, hash, false, osq) 463 | 464 | if !result { 465 | t.Errorf("Malshare failed") 466 | } else { 467 | valid, errmsg := hash.ValidateFile(filename) 468 | 469 | if !valid { 470 | os.Remove(hash.Hash) 471 | t.Errorf(errmsg) 472 | } else { 473 | os.Remove(hash.Hash) 474 | } 475 | } 476 | } 477 | 478 | func TestMalwareBazaar(t *testing.T) { 479 | home, _ := os.UserHomeDir() 480 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 481 | if err != nil { 482 | log.Fatal() 483 | t.Errorf("%v", err) 484 | } 485 | 486 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 487 | if err != nil { 488 | log.Fatal() 489 | t.Errorf("%v", err) 490 | } 491 | 492 | ht, _ := hashType(scfg.Hash) 493 | hash := Hash{HashType: ht, Hash: scfg.Hash} 494 | 495 | var osq ObjectiveSeeQuery 496 | result, filename, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) 497 | 498 | if !result { 499 | t.Errorf("MalwareBazaar failed") 500 | } else { 501 | valid, errmsg := hash.ValidateFile(filename) 502 | 503 | if !valid { 504 | os.Remove(hash.Hash) 505 | t.Errorf(errmsg) 506 | } else { 507 | os.Remove(hash.Hash) 508 | } 509 | } 510 | } 511 | 512 | func TestMalwareBazaarMD5(t *testing.T) { 513 | home, _ := os.UserHomeDir() 514 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 515 | if err != nil { 516 | log.Fatal() 517 | t.Errorf("%v", err) 518 | } 519 | 520 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 521 | if err != nil { 522 | log.Fatal() 523 | t.Errorf("%v", err) 524 | } 525 | 526 | ht, _ := hashType(scfg.Hash) 527 | hash := Hash{HashType: ht, Hash: scfg.Hash} 528 | 529 | var osq ObjectiveSeeQuery 530 | result, filename, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) 531 | 532 | if !result { 533 | t.Errorf("MalwareBazaar failed") 534 | } else { 535 | valid, errmsg := hash.ValidateFile(filename) 536 | 537 | if !valid { 538 | os.Remove(filename) 539 | t.Errorf(errmsg) 540 | } else { 541 | os.Remove(filename) 542 | } 543 | } 544 | } 545 | 546 | func TestMalwareBazaarNotFound(t *testing.T) { 547 | home, _ := os.UserHomeDir() 548 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 549 | if err != nil { 550 | log.Fatal() 551 | t.Errorf("%v", err) 552 | } 553 | 554 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 555 | if err != nil { 556 | log.Fatal() 557 | t.Errorf("%v", err) 558 | } 559 | 560 | ht, _ := hashType(scfg.Hash) 561 | hash := Hash{HashType: ht, Hash: scfg.Hash} 562 | 563 | var osq ObjectiveSeeQuery 564 | result, filename, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) 565 | 566 | if result { 567 | os.Remove(filename) 568 | t.Errorf("MalwareBazaar failed") 569 | } 570 | } 571 | 572 | func TestMalpedia(t *testing.T) { 573 | home, _ := os.UserHomeDir() 574 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 575 | if err != nil { 576 | log.Fatal() 577 | t.Errorf("%v", err) 578 | } 579 | 580 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 581 | if err != nil { 582 | log.Fatal() 583 | t.Errorf("%v", err) 584 | } 585 | 586 | ht, _ := hashType(scfg.Hash) 587 | hash := Hash{HashType: ht, Hash: scfg.Hash} 588 | 589 | var osq ObjectiveSeeQuery 590 | result, filename, _ := Malpedia.QueryAndDownload(cfg, hash, false, osq) 591 | 592 | if !result { 593 | t.Errorf("Malpedia failed") 594 | } else { 595 | valid, errmsg := hash.ValidateFile(filename) 596 | 597 | if !valid { 598 | os.Remove(hash.Hash) 599 | t.Errorf(errmsg) 600 | } else { 601 | os.Remove(hash.Hash) 602 | } 603 | } 604 | } 605 | 606 | func TestUnpacme(t *testing.T) { 607 | home, _ := os.UserHomeDir() 608 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 609 | if err != nil { 610 | log.Fatal() 611 | t.Errorf("%v", err) 612 | } 613 | 614 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 615 | if err != nil { 616 | log.Fatal() 617 | t.Errorf("%v", err) 618 | } 619 | 620 | ht, _ := hashType(scfg.Hash) 621 | hash := Hash{HashType: ht, Hash: scfg.Hash} 622 | 623 | var osq ObjectiveSeeQuery 624 | result, filename, _ := UnpacMe.QueryAndDownload(cfg, hash, false, osq) 625 | 626 | if !result { 627 | t.Errorf("Unpacme failed") 628 | } else { 629 | valid, errmsg := hash.ValidateFile(filename) 630 | 631 | if !valid { 632 | os.Remove(hash.Hash) 633 | t.Errorf(errmsg) 634 | } else { 635 | os.Remove(hash.Hash) 636 | } 637 | } 638 | } 639 | 640 | func TestVxShare(t *testing.T) { 641 | home, _ := os.UserHomeDir() 642 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 643 | if err != nil { 644 | log.Fatal() 645 | t.Errorf("%v", err) 646 | } 647 | 648 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 649 | if err != nil { 650 | log.Fatal() 651 | t.Errorf("%v", err) 652 | } 653 | 654 | ht, _ := hashType(scfg.Hash) 655 | hash := Hash{HashType: ht, Hash: scfg.Hash} 656 | 657 | var osq ObjectiveSeeQuery 658 | result, filename, _ := VxShare.QueryAndDownload(cfg, hash, false, osq) 659 | 660 | if !result { 661 | t.Errorf("VxShare failed") 662 | } else { 663 | valid, errmsg := hash.ValidateFile(filename) 664 | 665 | if !valid { 666 | os.Remove(hash.Hash) 667 | t.Errorf(errmsg) 668 | } else { 669 | os.Remove(hash.Hash) 670 | } 671 | } 672 | } 673 | 674 | func TestFileScanIo(t *testing.T) { 675 | home, _ := os.UserHomeDir() 676 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 677 | if err != nil { 678 | log.Fatal() 679 | t.Errorf("%v", err) 680 | } 681 | 682 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 683 | if err != nil { 684 | log.Fatal() 685 | t.Errorf("%v", err) 686 | } 687 | 688 | ht, _ := hashType(scfg.Hash) 689 | hash := Hash{HashType: ht, Hash: scfg.Hash} 690 | 691 | var osq ObjectiveSeeQuery 692 | result, filename, _ := FileScanIo.QueryAndDownload(cfg, hash, false, osq) 693 | 694 | if !result { 695 | t.Errorf("FileScanIo failed") 696 | } else { 697 | valid, errmsg := hash.ValidateFile(filename) 698 | 699 | if !valid { 700 | os.Remove(hash.Hash) 701 | t.Errorf(errmsg) 702 | } else { 703 | os.Remove(hash.Hash) 704 | } 705 | } 706 | 707 | } 708 | 709 | func TestURLScanIo(t *testing.T) { 710 | home, _ := os.UserHomeDir() 711 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 712 | if err != nil { 713 | log.Fatal() 714 | t.Errorf("%v", err) 715 | } 716 | 717 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 718 | if err != nil { 719 | log.Fatal() 720 | t.Errorf("%v", err) 721 | } 722 | 723 | ht, _ := hashType(scfg.Hash) 724 | hash := Hash{HashType: ht, Hash: scfg.Hash} 725 | 726 | var osq ObjectiveSeeQuery 727 | result, filename, _ := URLScanIO.QueryAndDownload(cfg, hash, false, osq) 728 | 729 | if !result { 730 | t.Errorf("URLScanIO failed") 731 | } else { 732 | valid, errmsg := hash.ValidateFile(filename) 733 | 734 | if !valid { 735 | os.Remove(hash.Hash) 736 | t.Errorf(errmsg) 737 | } else { 738 | os.Remove(hash.Hash) 739 | } 740 | } 741 | 742 | } 743 | 744 | func TestAssemblyLine(t *testing.T) { 745 | home, _ := os.UserHomeDir() 746 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 747 | if err != nil { 748 | log.Fatal() 749 | t.Errorf("%v", err) 750 | } 751 | 752 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 753 | if err != nil { 754 | log.Fatal() 755 | t.Errorf("%v", err) 756 | } 757 | 758 | ht, _ := hashType(scfg.Hash) 759 | hash := Hash{HashType: ht, Hash: scfg.Hash} 760 | 761 | var osq ObjectiveSeeQuery 762 | result, filename, _ := AssemblyLine.QueryAndDownload(cfg, hash, false, osq) 763 | 764 | if !result { 765 | t.Errorf("Assemblyline failed") 766 | } else { 767 | valid, errmsg := hash.ValidateFile(filename) 768 | 769 | if !valid { 770 | os.Remove(filename) 771 | t.Errorf(errmsg) 772 | } else { 773 | os.Remove(filename) 774 | } 775 | } 776 | 777 | } 778 | 779 | func TestVirusExchange(t *testing.T) { 780 | home, _ := os.UserHomeDir() 781 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 782 | if err != nil { 783 | log.Fatal() 784 | t.Errorf("%v", err) 785 | } 786 | 787 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 788 | if err != nil { 789 | log.Fatal() 790 | t.Errorf("%v", err) 791 | } 792 | 793 | ht, _ := hashType(scfg.Hash) 794 | hash := Hash{HashType: ht, Hash: scfg.Hash} 795 | 796 | var osq ObjectiveSeeQuery 797 | result, filename, _ := VirusExchange.QueryAndDownload(cfg, hash, false, osq) 798 | 799 | if result { 800 | t.Errorf("VirusExchange was a success - this is unexpected. This means the link returned by the API was fixed or there is another issue going on.") 801 | valid, errmsg := hash.ValidateFile(filename) 802 | 803 | if !valid { 804 | os.Remove(filename) 805 | t.Errorf(errmsg) 806 | } else { 807 | os.Remove(filename) 808 | } 809 | } 810 | } 811 | 812 | func TestVirusExchangeV2(t *testing.T) { 813 | home, _ := os.UserHomeDir() 814 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 815 | if err != nil { 816 | log.Fatal() 817 | t.Errorf("%v", err) 818 | } 819 | 820 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 821 | if err != nil { 822 | log.Fatal() 823 | t.Errorf("%v", err) 824 | } 825 | 826 | ht, _ := hashType(scfg.Hash) 827 | hash := Hash{HashType: ht, Hash: scfg.Hash} 828 | 829 | var osq ObjectiveSeeQuery 830 | result, filename, _ := VirusExchange.QueryAndDownload(cfg, hash, false, osq) 831 | 832 | if !result { 833 | t.Errorf("VirusExchange failed") 834 | } else { 835 | valid, errmsg := hash.ValidateFile(filename) 836 | 837 | if !valid { 838 | os.Remove(filename) 839 | t.Errorf(errmsg) 840 | } else { 841 | os.Remove(filename) 842 | } 843 | } 844 | } 845 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/tls" 7 | "encoding/base64" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "log" 13 | "net/http" 14 | "net/url" 15 | "os" 16 | "regexp" 17 | "strings" 18 | "syscall" 19 | 20 | "github.com/yeka/zip" 21 | ) 22 | 23 | type JoeSandboxQuery struct { 24 | Data []JoeSandboxQueryData `'json:"data"` 25 | } 26 | 27 | type JoeSandboxQueryData struct { 28 | Webid string `json:"webid"` 29 | } 30 | 31 | type InquestLabsQuery struct { 32 | Data []InquestLabsQueryData `json:"data"` 33 | Success bool `json:"success"` 34 | } 35 | 36 | type InquestLabsQueryData struct { 37 | Sha256 string `json:"sha256"` 38 | } 39 | 40 | type HybridAnalysisQuery struct { 41 | Submit_name string `json:"submit_name"` 42 | Md5 string `json:"md5"` 43 | Sha1 string `json:"sha1"` 44 | Sha256 string `json:"sha256"` 45 | Sha512 string `json:"sha512"` 46 | } 47 | 48 | type MalwareBazarQuery struct { 49 | Data []MalwareBazarQueryData `json:"data"` 50 | } 51 | 52 | type MalwareBazarQueryData struct { 53 | Sha256_hash string `json:"sha256_hash"` 54 | Sha3_384_hash string `json:"sha3_384_hash"` 55 | Sha1_hash string `json:"sha1_hash"` 56 | Md5_hash string `json:"md5_hash"` 57 | File_name string `json:"file_name"` 58 | } 59 | 60 | type MalwareBazaarQueryStatus struct { 61 | Status string `json:"query_status"` 62 | } 63 | 64 | type AssemblyLineQuery struct { 65 | Error_message string `json:"api_error_message"` 66 | Response *AssemblyLineQueryResponse `json:"api_response"` 67 | Server_version string `json:"api_server_version"` 68 | Status_Code int `json:"api_status_code"` 69 | } 70 | 71 | type AssemblyLineQueryResponse struct { 72 | AL *AssemblyLineQueryALResponse `json:"al"` 73 | } 74 | 75 | type AssemblyLineQueryALResponse struct { 76 | Error string `json:"error"` 77 | Items []AssemblyLineQueryItem `json:"items"` 78 | } 79 | 80 | type AssemblyLineQueryItem struct { 81 | Classification string `json:"classification"` 82 | Data *AssemblyLineQueryData `json:"data"` 83 | } 84 | 85 | type AssemblyLineQueryData struct { 86 | Md5 string `json:"md5"` 87 | Sha1 string `json:"sha1"` 88 | Sha256 string `json:"sha256"` 89 | } 90 | 91 | type TriageQuery struct { 92 | Data []TriageQueryData `json:"data"` 93 | } 94 | 95 | type TriageQueryData struct { 96 | Id string `json:"id"` 97 | Kind string `json:"kind"` 98 | Filename string `json:"filename"` 99 | } 100 | 101 | type ObjectiveSeeQuery struct { 102 | Malware []ObjectiveSeeData `json:"malware"` 103 | } 104 | 105 | type ObjectiveSeeData struct { 106 | Name string `json:"name"` 107 | Type string `json:"type"` 108 | VirusTotal string `json:"virusTotal"` 109 | MoreInfo string `json:"moreInfo"` 110 | Download string `json:"download"` 111 | Sha256 string 112 | } 113 | 114 | type MalpediaData struct { 115 | Name string 116 | FileBytes []byte 117 | } 118 | 119 | type VirusExchangeData struct { 120 | Md5 string `json:"md5"` 121 | Size int `json:"size"` 122 | Type string `json:"type"` 123 | Sha512 string `json:"sha512"` 124 | Sha256 string `json:"sha256"` 125 | Sha1 string `json:"sha1"` 126 | Download_Link string `json:"download_link"` 127 | } 128 | 129 | func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) { 130 | 131 | fmt.Printf("Downloading Objective-See Malware json from: %s\n\n", uri) 132 | 133 | client := &http.Client{} 134 | response, error := client.Get(uri) 135 | if error != nil { 136 | fmt.Println(error) 137 | return ObjectiveSeeQuery{}, error 138 | } 139 | 140 | defer response.Body.Close() 141 | 142 | if response.StatusCode == http.StatusOK { 143 | byteValue, _ := io.ReadAll(response.Body) 144 | 145 | var data = ObjectiveSeeQuery{} 146 | error = json.Unmarshal(byteValue, &data) 147 | 148 | var unmarshalTypeError *json.UnmarshalTypeError 149 | if errors.As(error, &unmarshalTypeError) { 150 | fmt.Printf(" [!] Failed unmarshaling json. Likely due to the format of the Objective-See json file changing\n") 151 | fmt.Printf(" %s\n", byteValue) 152 | 153 | } else if error != nil { 154 | fmt.Println(error) 155 | return ObjectiveSeeQuery{}, error 156 | } 157 | 158 | fmt.Printf(" Parsing VirusTotal Links for sha256 hashes\n") 159 | re := regexp.MustCompile("[A-Fa-f0-9]{64}") 160 | for k, item := range data.Malware { 161 | if len(item.VirusTotal) > 0 { 162 | matches := re.FindStringSubmatch(item.VirusTotal) 163 | if len(matches) == 1 { 164 | data.Malware[k].Sha256 = matches[0] 165 | } 166 | } 167 | if len(data.Malware[k].Sha256) == 0 { 168 | fmt.Printf(" [!] SHA256 not found for %s : %s\n VirusTotal Link: %s\n", item.Name, item.Type, item.VirusTotal) 169 | } 170 | } 171 | 172 | return data, nil 173 | } else { 174 | return ObjectiveSeeQuery{}, fmt.Errorf("unable to download objective-see json file") 175 | } 176 | } 177 | 178 | func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool) (bool, string) { 179 | if hash.HashType != sha256 { 180 | fmt.Printf(" [!] Objective-See only supports SHA256\n Skipping\n") 181 | } 182 | 183 | item, found := findHashInObjectiveSeeList(data.Malware, hash) 184 | 185 | if !found { 186 | return false, "" 187 | } 188 | 189 | if !doNotExtract { 190 | fmt.Printf(" [!] Extraction is not supported for Objective-See\n Try again but with the --noextraction flag\n") 191 | return false, "" 192 | } 193 | 194 | client := &http.Client{} 195 | response, error := client.Get(item.Download) 196 | if error != nil { 197 | fmt.Println(error) 198 | return false, "" 199 | } 200 | 201 | defer response.Body.Close() 202 | 203 | if response.StatusCode != http.StatusOK { 204 | return false, "" 205 | } 206 | 207 | error = writeToFile(response.Body, hash.Hash+".zip") 208 | if error != nil { 209 | fmt.Println(error) 210 | return false, "" 211 | } 212 | 213 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") 214 | if doNotExtract { 215 | return true, hash.Hash + ".zip" 216 | } else { 217 | return false, "" 218 | } 219 | } 220 | 221 | func joesandbox(uri string, api string, hash Hash) (bool, string) { 222 | if api == "" { 223 | fmt.Println(" [!] !! Missing Key !!") 224 | return false, "" 225 | } 226 | 227 | fmt.Printf(" [-] Looking up sandbox ID for: %s\n", hash.Hash) 228 | 229 | query := uri + "/analysis/search" 230 | 231 | _, error := url.ParseRequestURI(query) 232 | if error != nil { 233 | fmt.Printf(" [!] Error when parsing the query uri (%s). Check the value in the config file.\n", query) 234 | fmt.Println(error) 235 | return false, "" 236 | } 237 | 238 | queryData := "q=" + hash.Hash + "&" + "apikey=" + api 239 | values, error := url.ParseQuery(queryData) 240 | if error != nil { 241 | fmt.Printf(" [!] Error when parsing the query data (%s).\n", queryData) 242 | fmt.Println(error) 243 | return false, "" 244 | } 245 | 246 | client := &http.Client{} 247 | response, error := client.PostForm(query, values) 248 | if error != nil { 249 | fmt.Println(error) 250 | return false, "" 251 | } 252 | 253 | defer response.Body.Close() 254 | 255 | if response.StatusCode == http.StatusOK { 256 | 257 | byteValue, error := io.ReadAll(response.Body) 258 | if error != nil { 259 | fmt.Println(error) 260 | return false, "" 261 | } 262 | 263 | var data = JoeSandboxQuery{} 264 | error = json.Unmarshal(byteValue, &data) 265 | 266 | if error != nil { 267 | fmt.Println(error) 268 | return false, "" 269 | } 270 | 271 | if len(data.Data) > 0 { 272 | sandboxid := data.Data[0].Webid 273 | fmt.Printf(" [-] Hash %s Sandbox ID: %s\n", hash.Hash, sandboxid) 274 | 275 | // Download Sample using Sample ID 276 | return joesandboxDownload(uri, api, sandboxid, hash) 277 | } else { 278 | return false, "" 279 | } 280 | } else if response.StatusCode == http.StatusForbidden { 281 | fmt.Printf(" [!] Not authorized. Check the URL in the config.\n JoeSandbox does have more than one API endpoint.\n Check your documentation.\n") 282 | return false, "" 283 | } else { 284 | return false, "" 285 | } 286 | 287 | } 288 | 289 | func joesandboxDownload(uri string, api string, sandboxid string, hash Hash) (bool, string) { 290 | query := uri + "/analysis/download" 291 | 292 | _, error := url.ParseRequestURI(query) 293 | if error != nil { 294 | fmt.Printf(" [!] Error when parsing the query uri (%s). Check the value in the config file.\n", query) 295 | fmt.Println(error) 296 | return false, "" 297 | } 298 | 299 | queryData := "webid=" + sandboxid + "&" + "apikey=" + api + "&" + "type=sample" 300 | values, error := url.ParseQuery(queryData) 301 | if error != nil { 302 | fmt.Printf(" [!] Error when parsing the query data (%s).\n", queryData) 303 | fmt.Println(error) 304 | return false, "" 305 | } 306 | 307 | client := &http.Client{} 308 | response, error := client.PostForm(query, values) 309 | if error != nil { 310 | fmt.Println(error) 311 | return false, "" 312 | } 313 | 314 | defer response.Body.Close() 315 | 316 | if response.StatusCode == http.StatusOK { 317 | error = writeToFile(response.Body, hash.Hash) 318 | if error != nil { 319 | fmt.Println(error) 320 | return false, "" 321 | } 322 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 323 | return true, hash.Hash 324 | } else { 325 | return false, hash.Hash 326 | } 327 | } 328 | 329 | func capesandbox(uri string, api string, hash Hash) (bool, string) { 330 | if api == "" { 331 | fmt.Println(" [!] !! Missing Key !!") 332 | return false, "" 333 | } 334 | return capesandboxDownload(uri, api, hash) 335 | } 336 | 337 | func capesandboxDownload(uri string, api string, hash Hash) (bool, string) { 338 | query := uri + "/files/get/" + url.QueryEscape(hash.HashType.String()) + "/" + url.QueryEscape(hash.Hash) + "/" 339 | 340 | request, err := http.NewRequest("GET", query, nil) 341 | if err != nil { 342 | fmt.Println(err) 343 | return false, "" 344 | } 345 | 346 | request.Header.Set("Authorization", "Token "+api) 347 | client := &http.Client{} 348 | response, error := client.Do(request) 349 | if error != nil { 350 | fmt.Println(error) 351 | return false, "" 352 | } 353 | defer response.Body.Close() 354 | 355 | if response.StatusCode == http.StatusOK { 356 | 357 | if response.Header["Content-Type"][0] == "application/json" { 358 | return false, "" 359 | } 360 | 361 | error = writeToFile(response.Body, hash.Hash) 362 | if error != nil { 363 | fmt.Println(error) 364 | return false, "" 365 | } 366 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 367 | return true, hash.Hash 368 | } else if response.StatusCode == http.StatusForbidden { 369 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 370 | return false, "" 371 | } else { 372 | return false, "" 373 | } 374 | } 375 | 376 | func inquestlabs(uri string, api string, hash Hash) (bool, string) { 377 | if api == "" { 378 | fmt.Println(" [!] !! Missing Key !!") 379 | return false, "" 380 | } 381 | 382 | if hash.HashType != sha256 { 383 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 384 | 385 | query := uri + "/dfi/search/hash/" + url.PathEscape(hash.HashType.String()) + "?hash=" + url.QueryEscape(hash.Hash) 386 | 387 | _, error := url.ParseQuery(query) 388 | if error != nil { 389 | fmt.Println(" [!] Issue creating hash lookup query url") 390 | fmt.Println(error) 391 | return false, "" 392 | } 393 | 394 | request, err := http.NewRequest("GET", query, nil) 395 | if err != nil { 396 | fmt.Println(err) 397 | return false, "" 398 | } 399 | 400 | client := &http.Client{} 401 | response, error := client.Do(request) 402 | if error != nil { 403 | fmt.Println(error) 404 | return false, "" 405 | } 406 | defer response.Body.Close() 407 | 408 | if response.StatusCode == http.StatusOK { 409 | 410 | byteValue, _ := io.ReadAll(response.Body) 411 | 412 | var data = InquestLabsQuery{} 413 | error = json.Unmarshal(byteValue, &data) 414 | 415 | var unmarshalTypeError *json.UnmarshalTypeError 416 | if errors.As(error, &unmarshalTypeError) { 417 | fmt.Printf(" [!] Failed unmarshaling json. This could be due to the API changing or\n just no data inside the data array was returned - aka. sha256 hash was not found.\n") 418 | fmt.Printf(" %s\n", byteValue) 419 | 420 | } else if error != nil { 421 | fmt.Println(error) 422 | return false, "" 423 | } 424 | 425 | if !data.Success { 426 | return false, "" 427 | } 428 | 429 | if len(data.Data) == 0 { 430 | return false, "" 431 | } 432 | 433 | if data.Data[0].Sha256 == "" { 434 | return false, "" 435 | } 436 | hash.HashType = sha256 437 | hash.Hash = data.Data[0].Sha256 438 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 439 | 440 | } else if response.StatusCode == http.StatusForbidden { 441 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 442 | return false, "" 443 | } 444 | } 445 | 446 | if hash.HashType == sha256 { 447 | return inquestlabsDownload(uri, api, hash) 448 | } 449 | return false, "" 450 | } 451 | 452 | func inquestlabsDownload(uri string, api string, hash Hash) (bool, string) { 453 | query := uri + "/dfi/download?sha256=" + url.QueryEscape(hash.Hash) 454 | 455 | _, error := url.ParseQuery(query) 456 | if error != nil { 457 | fmt.Println(error) 458 | return false, "" 459 | } 460 | 461 | request, err := http.NewRequest("GET", query, nil) 462 | if err != nil { 463 | fmt.Println(err) 464 | return false, "" 465 | } 466 | 467 | request.Header.Set("Authorization", api) 468 | client := &http.Client{} 469 | response, error := client.Do(request) 470 | if error != nil { 471 | fmt.Println(error) 472 | return false, "" 473 | } 474 | defer response.Body.Close() 475 | 476 | if response.StatusCode == http.StatusOK { 477 | 478 | error = writeToFile(response.Body, hash.Hash) 479 | if error != nil { 480 | fmt.Println(error) 481 | return false, "" 482 | } 483 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 484 | return true, hash.Hash 485 | } else if response.StatusCode == http.StatusForbidden { 486 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 487 | return false, "" 488 | } else { 489 | return false, "" 490 | } 491 | } 492 | 493 | func virustotal(uri string, api string, hash Hash) (bool, string) { 494 | if api == "" { 495 | fmt.Println(" [!] !! Missing Key !!") 496 | return false, "" 497 | } 498 | return virustotalDownload(uri, api, hash) 499 | } 500 | 501 | func virustotalDownload(uri string, api string, hash Hash) (bool, string) { 502 | query := uri + "/files/" + url.PathEscape(hash.Hash) + "/download" 503 | 504 | request, err := http.NewRequest("GET", query, nil) 505 | if err != nil { 506 | fmt.Println(err) 507 | return false, "" 508 | } 509 | 510 | request.Header.Set("x-apikey", api) 511 | client := &http.Client{} 512 | response, error := client.Do(request) 513 | if error != nil { 514 | fmt.Println(error) 515 | return false, "" 516 | } 517 | defer response.Body.Close() 518 | 519 | if response.StatusCode == http.StatusOK { 520 | 521 | error = writeToFile(response.Body, hash.Hash) 522 | if error != nil { 523 | fmt.Println(error) 524 | return false, "" 525 | } 526 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 527 | return true, hash.Hash 528 | } else if response.StatusCode == http.StatusForbidden { 529 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 530 | return false, "" 531 | } else { 532 | return false, "" 533 | } 534 | } 535 | 536 | func mwdb(uri string, api string, hash Hash) (bool, string) { 537 | if api == "" { 538 | fmt.Println(" [!] !! Missing Key !!") 539 | return false, "" 540 | } 541 | return mwdbDownload(uri, api, hash) 542 | } 543 | 544 | func mwdbDownload(uri string, api string, hash Hash) (bool, string) { 545 | query := uri + "/file/" + url.PathEscape(hash.Hash) + "/download" 546 | 547 | request, err := http.NewRequest("GET", query, nil) 548 | if err != nil { 549 | fmt.Println(err) 550 | return false, "" 551 | } 552 | 553 | request.Header.Set("Authorization", "Bearer "+api) 554 | client := &http.Client{} 555 | response, error := client.Do(request) 556 | if error != nil { 557 | fmt.Println(error) 558 | return false, "" 559 | } 560 | defer response.Body.Close() 561 | 562 | if response.StatusCode == http.StatusOK { 563 | 564 | error = writeToFile(response.Body, hash.Hash) 565 | if error != nil { 566 | fmt.Println(error) 567 | return false, "" 568 | } 569 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 570 | return true, hash.Hash 571 | } else if response.StatusCode == http.StatusForbidden { 572 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 573 | return false, "" 574 | } else { 575 | return false, "" 576 | } 577 | } 578 | 579 | func polyswarm(uri string, api string, hash Hash) (bool, string) { 580 | if api == "" { 581 | fmt.Println(" [!] !! Missing Key !!") 582 | return false, "" 583 | } 584 | return polyswarmDownload(uri, api, hash) 585 | } 586 | 587 | func polyswarmDownload(uri string, api string, hash Hash) (bool, string) { 588 | query := "/consumer/download/" + url.PathEscape(hash.HashType.String()) + "/" + url.PathEscape(hash.Hash) 589 | 590 | _, error := url.ParseQuery(query) 591 | if error != nil { 592 | fmt.Println(error) 593 | return false, "" 594 | } 595 | 596 | request, err := http.NewRequest("GET", uri+query, nil) 597 | if err != nil { 598 | fmt.Println(err) 599 | return false, "" 600 | } 601 | 602 | request.Header.Set("Authorization", api) 603 | client := &http.Client{} 604 | response, error := client.Do(request) 605 | if error != nil { 606 | fmt.Println(error) 607 | return false, "" 608 | } 609 | defer response.Body.Close() 610 | 611 | if response.StatusCode == http.StatusOK { 612 | 613 | error = writeToFile(response.Body, hash.Hash) 614 | if error != nil { 615 | fmt.Println(error) 616 | return false, "" 617 | } 618 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 619 | return true, hash.Hash 620 | } else if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { 621 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 622 | return false, "" 623 | } else { 624 | return false, "" 625 | } 626 | } 627 | 628 | func hybridAnalysis(uri string, api string, hash Hash) (bool, string) { 629 | if api == "" { 630 | fmt.Println(" [!] !! Missing Key !!") 631 | return false, "" 632 | } 633 | 634 | if hash.HashType != sha256 { 635 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 636 | 637 | pData := []byte("hash=" + hash.Hash) 638 | request, error := http.NewRequest("POST", uri+"/search/hash", bytes.NewBuffer(pData)) 639 | 640 | if error != nil { 641 | fmt.Println(error) 642 | return false, "" 643 | } 644 | 645 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 646 | request.Header.Set("user-agent", "Falcon Sandbox") 647 | request.Header.Set("api-key", api) 648 | client := &http.Client{} 649 | response, error := client.Do(request) 650 | if error != nil { 651 | fmt.Println(error) 652 | return false, "" 653 | } 654 | defer response.Body.Close() 655 | 656 | if response.StatusCode == http.StatusForbidden { 657 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 658 | return false, "" 659 | } 660 | byteValue, _ := io.ReadAll(response.Body) 661 | 662 | var data = HybridAnalysisQuery{} 663 | error = json.Unmarshal(byteValue, &data) 664 | 665 | if error != nil { 666 | fmt.Println(error) 667 | return false, "" 668 | } 669 | 670 | if data.Sha256 == "" { 671 | return false, "" 672 | } 673 | hash.Hash = data.Sha256 674 | hash.HashType = sha256 675 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 676 | 677 | } 678 | 679 | if hash.HashType == sha256 { 680 | return hybridAnalysisDownload(uri, api, hash) 681 | } 682 | return false, "" 683 | } 684 | 685 | func hybridAnalysisDownload(uri string, api string, hash Hash) (bool, string) { 686 | request, error := http.NewRequest("GET", uri+"/overview/"+url.PathEscape(hash.Hash)+"/sample", nil) 687 | 688 | request.Header.Set("accept", "application/gzip") 689 | request.Header.Set("user-agent", "Falcon Sandbox") 690 | request.Header.Set("api-key", api) 691 | 692 | if error != nil { 693 | fmt.Println(error) 694 | return false, "" 695 | } 696 | client := &http.Client{} 697 | response, error := client.Do(request) 698 | if error != nil { 699 | fmt.Println(error) 700 | return false, "" 701 | } 702 | defer response.Body.Close() 703 | 704 | if response.StatusCode == http.StatusForbidden { 705 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\nCould also be that the sample is not allowed to be downloaded.\n") 706 | return false, "" 707 | } else if response.StatusCode == http.StatusNotFound { 708 | fmt.Printf(" [!] Hash not found\n") 709 | return false, "" 710 | } else if response.StatusCode != http.StatusOK { 711 | return false, "" 712 | } 713 | 714 | error = writeToFile(response.Body, hash.Hash+".gzip") 715 | if error != nil { 716 | fmt.Println(error) 717 | return false, "" 718 | } 719 | if doNotExtractFlag { 720 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".gzip") 721 | return true, hash.Hash + ".gzip" 722 | } else { 723 | fmt.Println(" [-] Extracting...") 724 | err := extractGzip(hash.Hash) 725 | if err != nil { 726 | fmt.Println(error) 727 | return false, "" 728 | } else { 729 | fmt.Printf(" [-] Extracted %s\n", hash.Hash) 730 | } 731 | os.Remove(hash.Hash + ".gzip") 732 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 733 | return true, hash.Hash 734 | 735 | } 736 | } 737 | 738 | func triage(uri string, api string, hash Hash) (bool, string) { 739 | if api == "" { 740 | fmt.Println(" [!] !! Missing Key !!") 741 | return false, "" 742 | } 743 | 744 | // Look up hash to get Sample ID 745 | query := "query=" + url.QueryEscape(hash.HashType.String()) + ":" + url.QueryEscape(hash.Hash) 746 | _, error := url.ParseQuery(query) 747 | if error != nil { 748 | fmt.Println(error) 749 | return false, "" 750 | } 751 | request, error := http.NewRequest("GET", uri+"/search?"+query, nil) 752 | if error != nil { 753 | fmt.Println(error) 754 | return false, "" 755 | } 756 | 757 | request.Header.Set("Authorization", "Bearer "+api) 758 | 759 | client := &http.Client{} 760 | response, error := client.Do(request) 761 | if error != nil { 762 | fmt.Println(error) 763 | return false, "" 764 | } 765 | defer response.Body.Close() 766 | 767 | if response.StatusCode == http.StatusOK { 768 | 769 | byteValue, error := io.ReadAll(response.Body) 770 | if error != nil { 771 | fmt.Println(error) 772 | return false, "" 773 | } 774 | 775 | var data = TriageQuery{} 776 | error = json.Unmarshal(byteValue, &data) 777 | 778 | if error != nil { 779 | fmt.Println(error) 780 | return false, "" 781 | } 782 | 783 | if len(data.Data) > 0 { 784 | sampleId := data.Data[0].Id 785 | fmt.Printf(" [-] Hash %s Sample ID: %s\n", hash.Hash, sampleId) 786 | 787 | // Download Sample using Sample ID 788 | return traigeDownload(uri, api, sampleId, hash) 789 | } else { 790 | return false, "" 791 | } 792 | } else if response.StatusCode == http.StatusForbidden { 793 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 794 | return false, "" 795 | } else { 796 | return false, "" 797 | } 798 | } 799 | 800 | func traigeDownload(uri string, api string, sampleId string, hash Hash) (bool, string) { 801 | request, error := http.NewRequest("GET", uri+"/samples/"+url.PathEscape(sampleId)+"/sample", nil) 802 | if error != nil { 803 | fmt.Println(error) 804 | return false, "" 805 | } 806 | 807 | request.Header.Set("Authorization", "Bearer "+api) 808 | 809 | client := &http.Client{} 810 | response, error := client.Do(request) 811 | if error != nil { 812 | fmt.Println(error) 813 | return false, "" 814 | } 815 | 816 | defer response.Body.Close() 817 | 818 | error = writeToFile(response.Body, hash.Hash) 819 | if error != nil { 820 | fmt.Println(error) 821 | return false, "" 822 | } 823 | // Triage will download the sample directly - no password protected zip file. 824 | hashMatch, dhash := hash.ValidateFile(hash.Hash) 825 | if !hashMatch { 826 | fmt.Printf(" [!] Sample ID %s (%s)\n contains the file in question, further processing of the sample is needed to get the hash requested.\n", sampleId, dhash) 827 | //ok := YesNoPrompt(fmt.Sprintf(" [?] Keep the file %s or delete it and continue looking for sample?", dhash), false) 828 | //if ok { 829 | return true, hash.Hash 830 | // } else { 831 | //return false, "" 832 | //} 833 | 834 | } else { 835 | return true, hash.Hash 836 | } 837 | } 838 | 839 | func malshare(url string, api string, hash Hash) (bool, string) { 840 | if api == "" { 841 | fmt.Println(" [!] !! Missing Key !!") 842 | return false, "" 843 | } 844 | 845 | return malshareDownload(url, api, hash) 846 | } 847 | 848 | func malshareDownload(uri string, api string, hash Hash) (bool, string) { 849 | query := "api_key=" + url.QueryEscape(api) + "&action=getfile&hash=" + url.QueryEscape(hash.Hash) 850 | 851 | _, error := url.ParseQuery(query) 852 | if error != nil { 853 | fmt.Println(error) 854 | return false, "" 855 | } 856 | 857 | client := &http.Client{} 858 | response, error := client.Get(uri + "/api.php?" + query) 859 | if error != nil { 860 | fmt.Println(error) 861 | return false, "" 862 | } 863 | defer response.Body.Close() 864 | 865 | if response.StatusCode == http.StatusOK { 866 | 867 | error = writeToFile(response.Body, hash.Hash) 868 | if error != nil { 869 | fmt.Println(error) 870 | return false, "" 871 | } 872 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 873 | return true, hash.Hash 874 | } else if response.StatusCode == http.StatusForbidden { 875 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 876 | return false, "" 877 | } else { 878 | return false, "" 879 | } 880 | } 881 | 882 | func malwareBazaar(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 883 | if hash.HashType != sha256 { 884 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 885 | 886 | query := "query=get_info&hash=" + url.QueryEscape(hash.Hash) 887 | values, error := url.ParseQuery(query) 888 | if error != nil { 889 | fmt.Println(error) 890 | return false, "" 891 | } 892 | 893 | request, err := http.NewRequest("POST", uri, strings.NewReader(values.Encode())) 894 | if err != nil { 895 | fmt.Println(err) 896 | return false, "" 897 | } 898 | 899 | request.Header.Set("Auth-Key", api) 900 | request.Header.Set("Content-Type", "application/x-www-form-urlencoded") 901 | 902 | client := &http.Client{} 903 | response, error := client.Do(request) 904 | if error != nil { 905 | fmt.Println(error) 906 | return false, "" 907 | } 908 | defer response.Body.Close() 909 | 910 | if response.StatusCode == http.StatusForbidden { 911 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 912 | return false, "" 913 | } 914 | 915 | byteValue, _ := io.ReadAll(response.Body) 916 | 917 | var data = MalwareBazarQuery{} 918 | error = json.Unmarshal(byteValue, &data) 919 | 920 | if error != nil { 921 | fmt.Println(error) 922 | return false, "" 923 | } 924 | 925 | if data.Data == nil { 926 | return false, "" 927 | } 928 | hash.Hash = data.Data[0].Sha256_hash 929 | hash.HashType = sha256 930 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 931 | 932 | } 933 | 934 | if hash.HashType == sha256 { 935 | return malwareBazaarDownload(uri, api, hash, doNotExtract, password) 936 | } 937 | return false, "" 938 | } 939 | 940 | func malwareBazaarDownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 941 | query := "query=get_file&sha256_hash=" + url.QueryEscape(hash.Hash) 942 | values, error := url.ParseQuery(query) 943 | if error != nil { 944 | fmt.Println(error) 945 | return false, "" 946 | } 947 | 948 | request, err := http.NewRequest("POST", uri, strings.NewReader(values.Encode())) 949 | if err != nil { 950 | fmt.Println(err) 951 | return false, "" 952 | } 953 | 954 | request.Header.Set("Auth-Key", api) 955 | request.Header.Set("Content-Type", "application/x-www-form-urlencoded") 956 | 957 | client := &http.Client{} 958 | response, error := client.Do(request) 959 | if error != nil { 960 | fmt.Println(error) 961 | return false, "" 962 | } 963 | 964 | defer response.Body.Close() 965 | 966 | if response.StatusCode == http.StatusUnauthorized { 967 | fmt.Printf(" [!] Unauthorized - Correct API key and try again\n") 968 | return false, "" 969 | } 970 | 971 | if response.Header["Content-Type"][0] == "application/json" { 972 | if response.StatusCode == http.StatusMethodNotAllowed { 973 | if !strings.HasSuffix(uri, "/") { 974 | fmt.Printf(" [!] Trying again with a trailing slash: %s/\n", uri) 975 | return malwareBazaarDownload(uri+"/", api, hash, doNotExtract, password) 976 | } else { 977 | fmt.Printf(" [!] Normally the response code: %s means that the provided URL %s needs a trailing slash (to avoid the redirect), but this already has a trailing slash.\nPlease file a bug report at https://github.com/xorhex/mlget/issues\n", response.Status, uri) 978 | } 979 | } else { 980 | byteValue, _ := io.ReadAll(response.Body) 981 | 982 | var data = MalwareBazaarQueryStatus{} 983 | error = json.Unmarshal(byteValue, &data) 984 | 985 | if error == nil { 986 | if data.Status == "file_not_found" { 987 | return false, "" 988 | } 989 | } else { 990 | err = writeToFile(io.NopCloser(bytes.NewReader(byteValue)), hash.Hash+".zip") 991 | if err != nil { 992 | fmt.Println(err) 993 | return false, "" 994 | } 995 | 996 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") 997 | if doNotExtract { 998 | return true, hash.Hash + ".zip" 999 | } else { 1000 | fmt.Println(" [-] Extracting...") 1001 | files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) 1002 | if err != nil { 1003 | fmt.Println(err) 1004 | return false, "" 1005 | } else { 1006 | for _, f := range files { 1007 | fmt.Printf(" [-] Extracted %s\n", f.Name) 1008 | } 1009 | } 1010 | os.Remove(hash.Hash + ".zip") 1011 | return true, hash.Hash 1012 | } 1013 | } 1014 | } 1015 | } 1016 | return false, "" 1017 | } 1018 | 1019 | func filescanio(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1020 | if api == "" { 1021 | fmt.Println(" [!] !! Missing Key !!") 1022 | return false, "" 1023 | } 1024 | return filescaniodownload(uri, api, hash, doNotExtract, password) 1025 | } 1026 | 1027 | func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1028 | query := "type=raw" 1029 | _, error := url.ParseQuery(query) 1030 | if error != nil { 1031 | fmt.Println(error) 1032 | return false, "" 1033 | } 1034 | 1035 | request, error := http.NewRequest("GET", uri+"/files/"+url.PathEscape(hash.Hash)+"?"+query, nil) 1036 | if error != nil { 1037 | fmt.Println(error) 1038 | return false, "" 1039 | } 1040 | 1041 | request.Header.Set("X-Api-Key", api) 1042 | 1043 | client := &http.Client{} 1044 | response, error := client.Do(request) 1045 | if error != nil { 1046 | fmt.Println(error) 1047 | return false, "" 1048 | } 1049 | 1050 | defer response.Body.Close() 1051 | 1052 | if response.StatusCode == 404 { 1053 | return false, "" 1054 | } else if response.StatusCode == 422 { 1055 | fmt.Printf(" [!] Validation Error.\n") 1056 | return false, "" 1057 | } else if response.StatusCode == http.StatusForbidden { 1058 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1059 | fmt.Printf(" [!] If you are sure this is correct, then test downloading a sample you've access to in their platform. It should work.\n") 1060 | fmt.Printf(" [!] Not sure why it does not just return a 404 instead; when the creds are correct but the file is not available.\n") 1061 | return false, "" 1062 | } 1063 | 1064 | error = writeToFile(response.Body, hash.Hash+".zip") 1065 | if error != nil { 1066 | fmt.Println(error) 1067 | return false, "" 1068 | } 1069 | 1070 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") 1071 | if doNotExtract { 1072 | return true, hash.Hash + ".zip" 1073 | } else { 1074 | fmt.Println(" [-] Extracting...") 1075 | files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) 1076 | if err != nil { 1077 | fmt.Println(err) 1078 | return false, "" 1079 | } else { 1080 | for _, f := range files { 1081 | fmt.Printf(" [-] Extracted %s\n", f.Name) 1082 | } 1083 | } 1084 | os.Remove(hash.Hash + ".zip") 1085 | return true, hash.Hash 1086 | } 1087 | } 1088 | 1089 | func vxshare(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1090 | if api == "" { 1091 | fmt.Println(" [!] !! Missing Key !!") 1092 | return false, "" 1093 | } 1094 | return vxsharedownload(uri, api, hash, doNotExtract, password) 1095 | } 1096 | 1097 | func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1098 | query := "apikey=" + url.QueryEscape(api) + "&hash=" + url.QueryEscape(hash.Hash) 1099 | _, error := url.ParseQuery(query) 1100 | if error != nil { 1101 | fmt.Println(error) 1102 | return false, "" 1103 | } 1104 | 1105 | client := &http.Client{} 1106 | response, error := client.Get(uri + "/download?" + query) 1107 | if error != nil { 1108 | fmt.Println(error) 1109 | return false, "" 1110 | } 1111 | 1112 | defer response.Body.Close() 1113 | 1114 | if response.StatusCode == 404 { 1115 | return false, "" 1116 | } else if response.StatusCode == http.StatusInternalServerError { 1117 | fmt.Printf(" [!] Internal service error. Skipping.\n") 1118 | return false, "" 1119 | } else if response.StatusCode == 204 { 1120 | fmt.Printf(" [!] Request rate limit exceeded. You are making more requests than are allowed or have exceeded your quota.\n") 1121 | return false, "" 1122 | } else if response.StatusCode == http.StatusForbidden { 1123 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1124 | return false, "" 1125 | } 1126 | 1127 | error = writeToFile(response.Body, hash.Hash+".zip") 1128 | if error != nil { 1129 | fmt.Println(error) 1130 | return false, "" 1131 | } 1132 | 1133 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1134 | if doNotExtract { 1135 | return true, hash.Hash + ".zip" 1136 | } else { 1137 | fmt.Println(" [-] Extracting...") 1138 | files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) 1139 | if err != nil { 1140 | fmt.Println(err) 1141 | return false, "" 1142 | } else { 1143 | for _, f := range files { 1144 | fmt.Printf(" [-] Extracted %s\n", f.Name) 1145 | } 1146 | } 1147 | os.Remove(hash.Hash + ".zip") 1148 | return true, hash.Hash 1149 | } 1150 | } 1151 | 1152 | func unpacme(uri string, api string, hash Hash) (bool, string) { 1153 | if api == "" { 1154 | fmt.Println(" [!] !! Missing Key !!") 1155 | return false, "" 1156 | } 1157 | 1158 | if hash.HashType != sha256 { 1159 | fmt.Printf(" [!] UnpacMe only supports SHA256\n Skipping\n") 1160 | return false, "" 1161 | } 1162 | 1163 | return unpacmeDownload(uri, api, hash) 1164 | } 1165 | 1166 | func unpacmeDownload(uri string, api string, hash Hash) (bool, string) { 1167 | request, error := http.NewRequest("GET", uri+"/private/download/"+url.PathEscape(hash.Hash), nil) 1168 | if error != nil { 1169 | fmt.Println(error) 1170 | return false, "" 1171 | } 1172 | 1173 | request.Header.Set("Authorization", "Key "+api) 1174 | 1175 | client := &http.Client{} 1176 | response, error := client.Do(request) 1177 | if error != nil { 1178 | fmt.Println(error) 1179 | return false, "" 1180 | } 1181 | 1182 | defer response.Body.Close() 1183 | 1184 | if response.StatusCode == 404 { 1185 | return false, "" 1186 | } else if response.StatusCode == http.StatusForbidden { 1187 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1188 | return false, "" 1189 | } else if response.StatusCode == http.StatusUnauthorized { 1190 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1191 | return false, "" 1192 | } 1193 | 1194 | error = writeToFile(response.Body, hash.Hash) 1195 | if error != nil { 1196 | fmt.Println(error) 1197 | return false, "" 1198 | } 1199 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1200 | return true, hash.Hash 1201 | } 1202 | 1203 | func urlscanio(uri string, api string, hash Hash) (bool, string) { 1204 | if api == "" { 1205 | fmt.Println(" [!] !! Missing Key !!") 1206 | return false, "" 1207 | } 1208 | 1209 | if hash.HashType != sha256 { 1210 | fmt.Printf(" [!] URLScanIO only supports SHA256\n Skipping\n") 1211 | } 1212 | 1213 | return urlscanioDownload(uri, api, hash) 1214 | } 1215 | 1216 | func urlscanioDownload(uri string, api string, hash Hash) (bool, string) { 1217 | //downloads/" 1218 | request, error := http.NewRequest("GET", uri+"/"+url.PathEscape(hash.Hash)+"/", nil) 1219 | if error != nil { 1220 | fmt.Println(error) 1221 | return false, "" 1222 | } 1223 | 1224 | request.Header.Set("API-Key", api) 1225 | 1226 | client := &http.Client{} 1227 | response, error := client.Do(request) 1228 | if error != nil { 1229 | fmt.Println(error) 1230 | return false, "" 1231 | } 1232 | 1233 | defer response.Body.Close() 1234 | 1235 | if response.StatusCode == 404 { 1236 | return false, "" 1237 | } else if response.StatusCode == http.StatusForbidden { 1238 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1239 | return false, "" 1240 | } 1241 | 1242 | error = writeToFile(response.Body, hash.Hash) 1243 | if error != nil { 1244 | fmt.Println(error) 1245 | return false, "" 1246 | } 1247 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1248 | return true, hash.Hash 1249 | } 1250 | 1251 | func malpedia(uri string, api string, hash Hash) (bool, string) { 1252 | if api == "" { 1253 | fmt.Println(" [!] !! Missing Key !!") 1254 | return false, "" 1255 | } 1256 | 1257 | if hash.HashType == sha1 { 1258 | fmt.Printf(" [!] Malpedia only supports MD5 and SHA256\n Skipping\n") 1259 | } 1260 | 1261 | return malpediaDownload(uri, api, hash) 1262 | 1263 | } 1264 | 1265 | func malpediaDownload(uri string, api string, hash Hash) (bool, string) { 1266 | ///get/sample//raw 1267 | // Malpedia returns a json file of the file and all of the related files (base64 encoded) 1268 | // No way to determine which file is which, so hashing each file found to identify the correct file 1269 | request, error := http.NewRequest("GET", uri+"/get/sample/"+url.PathEscape(hash.Hash)+"/raw", nil) 1270 | if error != nil { 1271 | fmt.Println(error) 1272 | return false, "" 1273 | } 1274 | 1275 | request.Header.Set("Authorization", "apitoken "+api) 1276 | 1277 | client := &http.Client{} 1278 | response, error := client.Do(request) 1279 | if error != nil { 1280 | fmt.Println(error) 1281 | return false, "" 1282 | } 1283 | 1284 | defer response.Body.Close() 1285 | 1286 | if response.StatusCode == 404 { 1287 | return false, "" 1288 | } else if response.StatusCode == http.StatusForbidden { 1289 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1290 | return false, "" 1291 | } 1292 | 1293 | byteValue, _ := io.ReadAll(response.Body) 1294 | jsonParseSuccesful, mpData := parseMalpediaJson(byteValue) 1295 | if jsonParseSuccesful { 1296 | for _, item := range mpData { 1297 | match, _ := hash.Validate(item.FileBytes) 1298 | if match { 1299 | error = writeBytesToFile(item.FileBytes, hash.Hash) 1300 | if error != nil { 1301 | fmt.Println(error) 1302 | return false, "" 1303 | } 1304 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1305 | return true, hash.Hash 1306 | } 1307 | } 1308 | } 1309 | return false, "" 1310 | } 1311 | 1312 | func parseMalpediaJson(byteValue []byte) (bool, []MalpediaData) { 1313 | // Code copied and modified fromm https://gist.github.com/mjohnsullivan/24647cae50928a34b5cc 1314 | // Unmarshal using a generic interface 1315 | var f interface{} 1316 | err := json.Unmarshal(byteValue, &f) 1317 | if err != nil { 1318 | fmt.Println("Error parsing JSON: ", err) 1319 | return false, nil 1320 | } 1321 | 1322 | // JSON object parses into a map with string keys 1323 | itemsMap := f.(map[string]interface{}) 1324 | var malpediaJsonItems []MalpediaData 1325 | 1326 | for key := range itemsMap { 1327 | var item MalpediaData 1328 | item.Name = key 1329 | rawDecodedValue, err := base64.StdEncoding.DecodeString(itemsMap[key].(string)) 1330 | if err != nil { 1331 | fmt.Println("Error base64 decoding the json value: ", err) 1332 | return false, nil 1333 | } 1334 | item.FileBytes = rawDecodedValue 1335 | malpediaJsonItems = append(malpediaJsonItems, item) 1336 | } 1337 | return true, malpediaJsonItems 1338 | } 1339 | 1340 | func assemblyline(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) { 1341 | if api == "" { 1342 | fmt.Println(" [!] !! Missing Key !!") 1343 | return false, "" 1344 | } 1345 | if user == "" { 1346 | fmt.Println(" [!] !! Missing User !!") 1347 | return false, "" 1348 | } 1349 | 1350 | if hash.HashType != sha256 { 1351 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 1352 | 1353 | request, error := http.NewRequest("GET", uri+"/hash_search/"+url.PathEscape(hash.Hash)+"/", nil) 1354 | if error != nil { 1355 | fmt.Println(error) 1356 | return false, "" 1357 | } 1358 | 1359 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 1360 | request.Header.Set("x-user", user) 1361 | request.Header.Set("x-apikey", api) 1362 | 1363 | tr := &http.Transport{} 1364 | if ignoretlserrors { 1365 | fmt.Printf(" [!] Ignoring Certificate Errors.\n") 1366 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 1367 | } 1368 | 1369 | client := &http.Client{Transport: tr} 1370 | response, error := client.Do(request) 1371 | if error != nil { 1372 | if errors.Is(error, syscall.ECONNREFUSED) { 1373 | fmt.Println(" [!] Connection Refused. Is the service online?") 1374 | return false, "" 1375 | } else { 1376 | fmt.Println(error) 1377 | return false, "" 1378 | } 1379 | } 1380 | defer response.Body.Close() 1381 | 1382 | if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { 1383 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 1384 | return false, "" 1385 | } 1386 | 1387 | byteValue, _ := io.ReadAll(response.Body) 1388 | 1389 | var data = AssemblyLineQuery{} 1390 | error = json.Unmarshal(byteValue, &data) 1391 | 1392 | if error != nil { 1393 | fmt.Println(error) 1394 | return false, "" 1395 | } 1396 | 1397 | if data.Response.AL == nil { 1398 | return false, "" 1399 | } 1400 | 1401 | if len(data.Response.AL.Items) > 0 { 1402 | hash.Hash = data.Response.AL.Items[0].Data.Sha256 1403 | hash.HashType = sha256 1404 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 1405 | } else { 1406 | return false, "" 1407 | } 1408 | } 1409 | 1410 | return assemblylineDownload(uri, user, api, ignoretlserrors, hash) 1411 | 1412 | } 1413 | 1414 | func assemblylineDownload(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) { 1415 | request, error := http.NewRequest("GET", uri+"/file/download/"+url.PathEscape(hash.Hash)+"/?encoding=raw", nil) 1416 | if error != nil { 1417 | fmt.Println(error) 1418 | return false, "" 1419 | } 1420 | 1421 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 1422 | request.Header.Set("x-user", user) 1423 | request.Header.Set("x-apikey", api) 1424 | 1425 | tr := &http.Transport{} 1426 | if ignoretlserrors { 1427 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 1428 | } 1429 | 1430 | client := &http.Client{Transport: tr} 1431 | response, error := client.Do(request) 1432 | if error != nil { 1433 | fmt.Println(error) 1434 | return false, "" 1435 | } 1436 | 1437 | defer response.Body.Close() 1438 | 1439 | if response.StatusCode == http.StatusNotFound { 1440 | return false, "" 1441 | } else if response.StatusCode == http.StatusForbidden { 1442 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 1443 | return false, "" 1444 | } else if response.StatusCode == http.StatusUnauthorized { 1445 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 1446 | return false, "" 1447 | } 1448 | 1449 | error = writeToFile(response.Body, hash.Hash) 1450 | if error != nil { 1451 | fmt.Println(error) 1452 | return false, "" 1453 | } 1454 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1455 | return true, hash.Hash 1456 | } 1457 | 1458 | func virusexchange(uri string, api string, hash Hash) (bool, string) { 1459 | if api == "" { 1460 | fmt.Println(" [!] !! Missing Key !!") 1461 | return false, "" 1462 | } 1463 | 1464 | if hash.HashType != sha256 { 1465 | fmt.Printf(" [!] VirusExchange only supports SHA256\n Skipping\n") 1466 | } 1467 | 1468 | request, error := http.NewRequest("GET", uri+"/samples/"+url.PathEscape(hash.Hash)+"/", nil) 1469 | if error != nil { 1470 | fmt.Println(error) 1471 | return false, "" 1472 | } 1473 | 1474 | request.Header.Set("Authorization", "Bearer "+api) 1475 | 1476 | client := &http.Client{} 1477 | response, error := client.Do(request) 1478 | if error != nil { 1479 | fmt.Println(error) 1480 | return false, "" 1481 | } 1482 | 1483 | defer response.Body.Close() 1484 | 1485 | if response.StatusCode == 404 { 1486 | return false, "" 1487 | } else if response.StatusCode == http.StatusForbidden { 1488 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1489 | return false, "" 1490 | } 1491 | 1492 | if response.StatusCode == http.StatusOK { 1493 | byteValue, _ := io.ReadAll(response.Body) 1494 | 1495 | var data = VirusExchangeData{} 1496 | error = json.Unmarshal(byteValue, &data) 1497 | 1498 | var unmarshalTypeError *json.UnmarshalTypeError 1499 | if errors.As(error, &unmarshalTypeError) { 1500 | fmt.Printf(" [!] Failed unmarshaling json. This could be due to the API changing or\n just no data inside the data array was returned\n") 1501 | fmt.Printf(" %s\n", byteValue) 1502 | 1503 | } else if error != nil { 1504 | fmt.Println(error) 1505 | return false, "" 1506 | } 1507 | 1508 | if data.Download_Link == "" { 1509 | return false, "" 1510 | } 1511 | return virusexchangeDownload(data.Download_Link, hash) 1512 | 1513 | } 1514 | // Sample not found 1515 | return false, "" 1516 | } 1517 | 1518 | func virusexchangeDownload(uri string, hash Hash) (bool, string) { 1519 | request, error := http.NewRequest("GET", uri, nil) 1520 | if error != nil { 1521 | fmt.Println(error) 1522 | return false, "" 1523 | } 1524 | 1525 | client := &http.Client{} 1526 | response, error := client.Do(request) 1527 | if error != nil { 1528 | fmt.Println(error) 1529 | return false, "" 1530 | } 1531 | 1532 | defer response.Body.Close() 1533 | 1534 | if response.StatusCode == http.StatusNotFound { 1535 | fmt.Printf(" [!] Invalid download link returned by the API.\n") 1536 | return false, "" 1537 | } else if response.StatusCode == http.StatusForbidden { 1538 | fmt.Printf(" [!] Not authorized for some reason.\n") 1539 | return false, "" 1540 | } else if response.StatusCode == http.StatusUnauthorized { 1541 | fmt.Printf(" [!] Not authorized for some reason\n") 1542 | return false, "" 1543 | } 1544 | 1545 | error = writeToFile(response.Body, hash.Hash) 1546 | if error != nil { 1547 | fmt.Println(error) 1548 | return false, "" 1549 | } 1550 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1551 | return true, hash.Hash 1552 | } 1553 | 1554 | func extractGzip(hash string) error { 1555 | r, err := os.Open(hash + ".gzip") 1556 | if err != nil { 1557 | log.Fatal(err) 1558 | } 1559 | defer r.Close() 1560 | 1561 | gzreader, e1 := gzip.NewReader(r) 1562 | if e1 != nil { 1563 | fmt.Println(e1) // Maybe panic here, depends on your error handling. 1564 | } 1565 | 1566 | err = writeToFile(io.NopCloser(gzreader), hash) 1567 | return err 1568 | } 1569 | 1570 | func extractPwdZip(file string, password string, renameFileAsHash bool, hash Hash) ([]*zip.File, error) { 1571 | 1572 | r, err := zip.OpenReader(file) 1573 | if err != nil { 1574 | log.Fatal(err) 1575 | } 1576 | defer r.Close() 1577 | 1578 | files := r.File 1579 | 1580 | for _, f := range r.File { 1581 | if f.IsEncrypted() { 1582 | f.SetPassword(password) 1583 | } 1584 | 1585 | r, err := f.Open() 1586 | if err != nil { 1587 | log.Fatal(err) 1588 | } 1589 | 1590 | var name string 1591 | 1592 | if !renameFileAsHash { 1593 | name = f.Name 1594 | } else { 1595 | name = hash.Hash 1596 | } 1597 | 1598 | out, error := os.Create(name) 1599 | if error != nil { 1600 | return nil, error 1601 | } 1602 | defer out.Close() 1603 | 1604 | _, err = io.Copy(out, r) 1605 | if err != nil { 1606 | return nil, err 1607 | } 1608 | } 1609 | return files, nil 1610 | } 1611 | 1612 | func findHashInObjectiveSeeList(list []ObjectiveSeeData, hash Hash) (ObjectiveSeeData, bool) { 1613 | for _, item := range list { 1614 | if item.Sha256 == hash.Hash { 1615 | return item, true 1616 | } 1617 | } 1618 | return ObjectiveSeeData{}, false 1619 | } 1620 | 1621 | func writeBytesToFile(bytes []byte, filename string) error { 1622 | // Create the file 1623 | out, err := os.Create(filename) 1624 | if err != nil { 1625 | return err 1626 | } 1627 | defer out.Close() 1628 | 1629 | _, err = out.Write(bytes) 1630 | return err 1631 | } 1632 | --------------------------------------------------------------------------------