├── .github └── workflows │ ├── go.yml │ └── release.yml ├── .gitignore ├── .vscode └── launch.json ├── README.md ├── config.go ├── download.go ├── go.mod ├── hashes.go ├── history.go ├── mlget-test-config └── samples.yaml ├── mlget.go ├── mlget_test.go ├── mlweb.go ├── upload.go └── web ├── scripts └── script.js ├── styles └── style.css └── templates └── index.html /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.21.6 20 | 21 | - name: Build 22 | run: go get -u && go build -v ./... 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/release.yaml 2 | 3 | on: 4 | release: 5 | types: [created, draft] 6 | 7 | jobs: 8 | releases-matrix: 9 | name: Release Go Binary 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | # build and publish in parallel: linux/386, linux/amd64, windows/386, windows/amd64, darwin/amd64 14 | goos: [linux, windows, darwin] 15 | goarch: [amd64] 16 | steps: 17 | - uses: actions/checkout@v3 18 | - uses: wangyoucao577/go-release-action@v1 19 | with: 20 | github_token: ${{ secrets.GITHUB_TOKEN }} 21 | goos: ${{ matrix.goos }} 22 | goarch: ${{ matrix.goarch }} 23 | sha256sum: true 24 | build_command: go build 25 | build_flags: -v 26 | pre_command: go get -u 27 | binary_name: "mlget" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | mlget 2 | mlget.yml 3 | go.sum 4 | .mlget.yml 5 | mlget.bak.yml 6 | mlget-bak.yml 7 | .vscode/* -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Package", 9 | "type": "go", 10 | "debugAdapter": "dlv-dap", 11 | "request": "launch", 12 | "host": "127.0.0.1", 13 | "mode": "debug", 14 | "program": "${fileDirname}", 15 | "args":[ 16 | "--ud", "--t", "mimikatz", "B9601E60F87545441BF8579B2F62668C56507F4A" 17 | ] 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## mlget 2 | 3 | ![image](https://api.xorhex.com/resource/png/Mlget-ReadMe/052cc80c1529236836d0bb7e35c30dff6567a285056564468d33b6197025d36c) 4 | 5 | 6 | ![Build](https://github.com/xorhex/mlget/actions/workflows/go.yml/badge.svg) 7 | 8 | ### What is it 9 | 10 | Use mlget to query multiple sources for a given malware hash and download it. The thought is to save time querying each source individually. 11 | 12 | ### Usage Instructions 13 | 14 | [Mlget Blog Post](https://blog.xorhex.com/mlget/) 15 | 16 | ### License 17 | 18 | MIT License 19 | 20 | Copyright (c) 2024 @xorhex 21 | 22 | Permission is hereby granted, free of charge, to any person obtaining a copy 23 | of this software and associated documentation files (the "Software"), to deal 24 | in the Software without restriction, including without limitation the rights 25 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 26 | copies of the Software, and to permit persons to whom the Software is 27 | furnished to do so, subject to the following conditions: 28 | 29 | The above copyright notice and this permission notice shall be included in all 30 | copies or substantial portions of the Software. 31 | 32 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 33 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 34 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 35 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 36 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 37 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 38 | SOFTWARE. 39 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "sort" 8 | "strconv" 9 | "strings" 10 | 11 | "golang.org/x/exp/slices" 12 | "gopkg.in/yaml.v2" 13 | ) 14 | 15 | type MalwareRepoType int64 16 | 17 | const ( 18 | NotSupported MalwareRepoType = iota //NotSupported must always be first, or other things won't work as expected 19 | 20 | AssemblyLine 21 | CapeSandbox 22 | FileScanIo 23 | HybridAnalysis 24 | InQuest 25 | JoeSandbox 26 | Malpedia 27 | Malshare 28 | MalwareBazaar 29 | MWDB 30 | ObjectiveSee 31 | Polyswarm 32 | Triage 33 | UnpacMe 34 | URLScanIO 35 | VirusExchange 36 | VirusTotal 37 | VxShare 38 | 39 | UploadAssemblyLine 40 | //UploadMWDB must always be last, or other things won't work as expected 41 | UploadMWDB 42 | ) 43 | 44 | // var MalwareRepoList = []MalwareRepoType{CapeSandbox, HybridAnalysis, InQuest, JoeSandbox, Malpedia, Malshare, MalwareBazaar, MWDB, ObjectiveSee, Polyswarm, Triage, UnpacMe, VirusTotal, UploadMWDB} 45 | func getMalwareRepoList() []MalwareRepoType { 46 | var malwareRepoList []MalwareRepoType 47 | for repo := range [UploadMWDB + 1]int64{} { 48 | if int64(repo) > int64(NotSupported) && int64(repo) <= int64(UploadMWDB) { 49 | malwareRepoList = append(malwareRepoList, MalwareRepoType(repo)) 50 | } 51 | } 52 | return malwareRepoList 53 | } 54 | 55 | func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, osq ObjectiveSeeQuery) (bool, string, MalwareRepoType) { 56 | matchingConfigRepos := getConfigsByType(malrepo, repos) 57 | if len(matchingConfigRepos) == 0 { 58 | fmt.Printf(" [!] %s is not found in the yml config file\n", malrepo) 59 | } 60 | for _, mcr := range matchingConfigRepos { 61 | found := false 62 | filename := "" 63 | checkedRepo := NotSupported 64 | fmt.Printf(" [*] %s: %s\n", mcr.Type, mcr.Host) 65 | switch malrepo { 66 | case MalwareBazaar: 67 | found, filename = malwareBazaar(mcr.Host, hash, doNotExtract, "infected") 68 | checkedRepo = MalwareBazaar 69 | case MWDB: 70 | found, filename = mwdb(mcr.Host, mcr.Api, hash) 71 | checkedRepo = MWDB 72 | case Malshare: 73 | found, filename = malshare(mcr.Host, mcr.Api, hash) 74 | checkedRepo = Malshare 75 | case Triage: 76 | found, filename = triage(mcr.Host, mcr.Api, hash) 77 | checkedRepo = Triage 78 | case InQuest: 79 | found, filename = inquestlabs(mcr.Host, mcr.Api, hash) 80 | checkedRepo = InQuest 81 | case HybridAnalysis: 82 | found, filename = hybridAnlysis(mcr.Host, mcr.Api, hash, doNotExtract) 83 | checkedRepo = HybridAnalysis 84 | case Polyswarm: 85 | found, filename = polyswarm(mcr.Host, mcr.Api, hash) 86 | checkedRepo = Polyswarm 87 | case VirusTotal: 88 | found, filename = virustotal(mcr.Host, mcr.Api, hash) 89 | checkedRepo = VirusTotal 90 | case JoeSandbox: 91 | found, filename = joesandbox(mcr.Host, mcr.Api, hash) 92 | checkedRepo = JoeSandbox 93 | case CapeSandbox: 94 | found, filename = capesandbox(mcr.Host, mcr.Api, hash) 95 | checkedRepo = CapeSandbox 96 | case ObjectiveSee: 97 | if len(osq.Malware) > 0 { 98 | found, filename = objectivesee(osq, hash, doNotExtract) 99 | checkedRepo = ObjectiveSee 100 | } 101 | case UnpacMe: 102 | found, filename = unpacme(mcr.Host, mcr.Api, hash) 103 | checkedRepo = UnpacMe 104 | case Malpedia: 105 | found, filename = malpedia(mcr.Host, mcr.Api, hash) 106 | checkedRepo = Malpedia 107 | case VxShare: 108 | found, filename = vxshare(mcr.Host, mcr.Api, hash, doNotExtract, "infected") 109 | checkedRepo = VxShare 110 | case FileScanIo: 111 | found, filename = filescanio(mcr.Host, mcr.Api, hash, doNotExtract, "infected") 112 | checkedRepo = FileScanIo 113 | case URLScanIO: 114 | found, filename = urlscanio(mcr.Host, mcr.Api, hash) 115 | checkedRepo = URLScanIO 116 | case VirusExchange: 117 | found, filename = virusexchange(mcr.Host, mcr.Api, hash) 118 | checkedRepo = VirusExchange 119 | case AssemblyLine: 120 | found, filename = assemblyline(mcr.Host, mcr.User, mcr.Api, mcr.IgnoreTLSErrors, hash) 121 | checkedRepo = AssemblyLine 122 | case UploadAssemblyLine: 123 | found, filename = assemblyline(mcr.Host, mcr.User, mcr.Api, mcr.IgnoreTLSErrors, hash) 124 | checkedRepo = UploadAssemblyLine 125 | case UploadMWDB: 126 | found, filename = mwdb(mcr.Host, mcr.Api, hash) 127 | checkedRepo = UploadMWDB 128 | } 129 | // So some repos we can't download from but we want to know that it exists at that service 130 | // At the moment, this is just Any.Run but suspect more will be added as time goes on 131 | if found { 132 | return found, filename, checkedRepo 133 | } 134 | } 135 | return false, "", NotSupported 136 | } 137 | 138 | func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool { 139 | switch malrepo { 140 | case NotSupported: 141 | return false 142 | case MalwareBazaar: 143 | if repo.Host != "" { 144 | return true 145 | } 146 | case ObjectiveSee: 147 | if repo.Host != "" { 148 | return true 149 | } 150 | case AssemblyLine: 151 | if repo.Host != "" && repo.Api != "" && repo.User != "" { 152 | return true 153 | } 154 | case UploadAssemblyLine: 155 | if repo.Host != "" && repo.Api != "" && repo.User != "" { 156 | return true 157 | } 158 | default: 159 | if repo.Host != "" && repo.Api != "" { 160 | return true 161 | } 162 | } 163 | return false 164 | } 165 | 166 | func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { 167 | var host string 168 | var api string 169 | var user string 170 | tls := false 171 | 172 | var default_url string 173 | 174 | switch malrepo { 175 | case NotSupported: 176 | return RepositoryConfigEntry{}, fmt.Errorf("malware repository rype, %s, is not supported", malrepo.String()) 177 | case MalwareBazaar: 178 | default_url = "https://mb-api.abuse.ch/api/v1/" 179 | case Malshare: 180 | default_url = "https://malshare.com" 181 | case MWDB: 182 | default_url = "https://mwdb.cert.pl/api" 183 | case CapeSandbox: 184 | default_url = "https://www.capesandbox.com/apiv2" 185 | case JoeSandbox: 186 | default_url = "https://jbxcloud.joesecurity.org/api/v2" 187 | case InQuest: 188 | default_url = "https://labs.inquest.net/api" 189 | case HybridAnalysis: 190 | default_url = "https://www.hybrid-analysis.com/api/v2" 191 | case Triage: 192 | default_url = "https://api.tria.ge/v0" 193 | case VirusTotal: 194 | default_url = "https://www.virustotal.com/api/v3" 195 | case Polyswarm: 196 | default_url = "https://api.polyswarm.network/v2" 197 | case ObjectiveSee: 198 | default_url = "https://objective-see.com/malware.json" 199 | case UnpacMe: 200 | default_url = "https://api.unpac.me/api/v1" 201 | case Malpedia: 202 | default_url = "https://malpedia.caad.fkie.fraunhofer.de/api" 203 | case VxShare: 204 | default_url = "https://virusshare.com/apiv2" 205 | case FileScanIo: 206 | default_url = "https://www.filescan.io/api" 207 | case URLScanIO: 208 | default_url = "https://urlscan.io/downloads" 209 | case VirusExchange: 210 | default_url = "https://virus.exchange/api" 211 | } 212 | if default_url != "" { 213 | fmt.Printf("Enter Host [ Press enter for default - %s ]:\n", default_url) 214 | } else { 215 | fmt.Printf("Enter Host:\n") 216 | } 217 | fmt.Print(">> ") 218 | fmt.Scanln(&host) 219 | if host == "" { 220 | fmt.Println("Using the default url") 221 | host = default_url 222 | } 223 | if malrepo == AssemblyLine || malrepo == UploadAssemblyLine { 224 | fmt.Println("Enter User Name:") 225 | fmt.Print(">> ") 226 | fmt.Scanln(&user) 227 | for { 228 | fmt.Println("Disable TLS Verification (true|false):") 229 | fmt.Print(">> ") 230 | var tlss string 231 | fmt.Scanln(&tlss) 232 | boolvalue, err := strconv.ParseBool(tlss) 233 | if err == nil { 234 | tls = boolvalue 235 | break 236 | } 237 | fmt.Println("Invalid option entered") 238 | } 239 | } 240 | if malrepo != MalwareBazaar && malrepo != ObjectiveSee { 241 | fmt.Println("Enter API Key:") 242 | fmt.Print(">> ") 243 | fmt.Scanln(&api) 244 | } 245 | return RepositoryConfigEntry{Host: host, User: user, Api: api, Type: malrepo.String(), IgnoreTLSErrors: tls}, nil 246 | } 247 | 248 | func (malrepo MalwareRepoType) String() string { 249 | switch malrepo { 250 | case JoeSandbox: 251 | return "JoeSandbox" 252 | case MWDB: 253 | return "MWDB" 254 | case HybridAnalysis: 255 | return "HybridAnalysis" 256 | case CapeSandbox: 257 | return "CapeSandbox" 258 | case InQuest: 259 | return "InQuest" 260 | case MalwareBazaar: 261 | return "MalwareBazaar" 262 | case Triage: 263 | return "Triage" 264 | case Malshare: 265 | return "Malshare" 266 | case VirusTotal: 267 | return "VirusTotal" 268 | case Polyswarm: 269 | return "Polyswarm" 270 | case ObjectiveSee: 271 | return "ObjectiveSee" 272 | case UnpacMe: 273 | return "UnpacMe" 274 | case Malpedia: 275 | return "Malpedia" 276 | case VxShare: 277 | return "VxShare" 278 | case FileScanIo: 279 | return "FileScanIo" 280 | case URLScanIO: 281 | return "URLScanIO" 282 | case AssemblyLine: 283 | return "AssemblyLine" 284 | case UploadAssemblyLine: 285 | return "UploadAssemblyLine" 286 | case UploadMWDB: 287 | return "UploadMWDB" 288 | case VirusExchange: 289 | return "VirusExchange" 290 | } 291 | return "NotSupported" 292 | } 293 | 294 | func allowedMalwareRepoTypes() { 295 | for _, mr := range getMalwareRepoList() { 296 | fmt.Printf(" %s\n", mr.String()) 297 | } 298 | } 299 | 300 | func printAllowedMalwareRepoTypeOptions() { 301 | fmt.Println("") 302 | for _, mr := range getMalwareRepoList() { 303 | fmt.Printf(" [%d] %s\n", mr, mr.String()) 304 | } 305 | } 306 | 307 | func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract bool, skipUpload bool, osq ObjectiveSeeQuery, doNotValidateHash bool, doNotValidateHashList []MalwareRepoType) (bool, string, MalwareRepoType) { 308 | found := false 309 | filename := "" 310 | checkedRepo := NotSupported 311 | sort.Slice(repos[:], func(i, j int) bool { 312 | return repos[i].QueryOrder < repos[j].QueryOrder 313 | }) 314 | 315 | // Hack for now 316 | // Due to Multiple entries of the same type, for each type instance in the config it will 317 | // try to download for type the number of entries for type in config squared 318 | // This array is meant to ensure that for each type it will only try it once 319 | var completedTypes []MalwareRepoType 320 | 321 | for _, repo := range repos { 322 | if (repo.Type == UploadMWDB.String() || repo.Type == UploadAssemblyLine.String()) && skipUpload { 323 | continue 324 | } 325 | mr := getMalwareRepoByName(repo.Type) 326 | if !contains(completedTypes, mr) { 327 | found, filename, checkedRepo = mr.QueryAndDownload(repos, hash, doNotExtract, osq) 328 | if found { 329 | if !doNotValidateHash { 330 | if slices.Contains(doNotValidateHashList, checkedRepo) { 331 | if checkedRepo == ObjectiveSee { 332 | fmt.Printf(" [!] Not able to validate hash for repo %s\n", checkedRepo.String()) 333 | } else { 334 | fmt.Printf(" [!] Not able to validate hash for repo %s when noextraction flag is set to %t\n", checkedRepo.String(), doNotExtractFlag) 335 | } 336 | break 337 | } else { 338 | valid, calculatedHash := hash.ValidateFile(filename) 339 | if !valid { 340 | fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\nTrying another source.\n", calculatedHash, hash.Hash) 341 | deleteInvalidFile(filename) 342 | continue 343 | } else { 344 | fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", hash.Hash) 345 | break 346 | } 347 | } 348 | } 349 | break 350 | } 351 | completedTypes = append(completedTypes, mr) 352 | } 353 | } 354 | return found, filename, checkedRepo 355 | } 356 | 357 | func getMalwareRepoByFlagName(name string) MalwareRepoType { 358 | switch strings.ToLower(name) { 359 | case strings.ToLower("js"): 360 | return JoeSandbox 361 | case strings.ToLower("md"): 362 | return MWDB 363 | case strings.ToLower("ha"): 364 | return HybridAnalysis 365 | case strings.ToLower("cs"): 366 | return CapeSandbox 367 | case strings.ToLower("iq"): 368 | return InQuest 369 | case strings.ToLower("mb"): 370 | return MalwareBazaar 371 | case strings.ToLower("tr"): 372 | return Triage 373 | case strings.ToLower("ms"): 374 | return Malshare 375 | case strings.ToLower("vt"): 376 | return VirusTotal 377 | case strings.ToLower("ps"): 378 | return Polyswarm 379 | case strings.ToLower("os"): 380 | return ObjectiveSee 381 | case strings.ToLower("um"): 382 | return UnpacMe 383 | case strings.ToLower("mp"): 384 | return Malpedia 385 | case strings.ToLower("vx"): 386 | return VxShare 387 | case strings.ToLower("fs"): 388 | return FileScanIo 389 | case strings.ToLower("us"): 390 | return URLScanIO 391 | case strings.ToLower("al"): 392 | return AssemblyLine 393 | case strings.ToLower("ve"): 394 | return VirusExchange 395 | } 396 | return NotSupported 397 | } 398 | 399 | func getMalwareRepoByName(name string) MalwareRepoType { 400 | switch strings.ToLower(name) { 401 | case strings.ToLower("JoeSandbox"): 402 | return JoeSandbox 403 | case strings.ToLower("MWDB"): 404 | return MWDB 405 | case strings.ToLower("HybridAnalysis"): 406 | return HybridAnalysis 407 | case strings.ToLower("CapeSandbox"): 408 | return CapeSandbox 409 | case strings.ToLower("InQuest"): 410 | return InQuest 411 | case strings.ToLower("MalwareBazaar"): 412 | return MalwareBazaar 413 | case strings.ToLower("Triage"): 414 | return Triage 415 | case strings.ToLower("Malshare"): 416 | return Malshare 417 | case strings.ToLower("VirusTotal"): 418 | return VirusTotal 419 | case strings.ToLower("Polyswarm"): 420 | return Polyswarm 421 | case strings.ToLower("ObjectiveSee"): 422 | return ObjectiveSee 423 | case strings.ToLower("UnpacMe"): 424 | return UnpacMe 425 | case strings.ToLower("Malpedia"): 426 | return Malpedia 427 | case strings.ToLower("VxShare"): 428 | return VxShare 429 | case strings.ToLower("FileScanIo"): 430 | return FileScanIo 431 | case strings.ToLower("URLScanIO"): 432 | return URLScanIO 433 | case strings.ToLower("AssemblyLine"): 434 | return AssemblyLine 435 | case strings.ToLower("UploadAssemblyLine"): 436 | return UploadAssemblyLine 437 | case strings.ToLower("UploadMWDB"): 438 | return UploadMWDB 439 | case strings.ToLower("VirusExchange"): 440 | return VirusExchange 441 | } 442 | return NotSupported 443 | } 444 | 445 | func getConfigsByType(repoType MalwareRepoType, repos []RepositoryConfigEntry) []RepositoryConfigEntry { 446 | var filteredRepos []RepositoryConfigEntry 447 | for _, v := range repos { 448 | if v.Type == repoType.String() { 449 | filteredRepos = append(filteredRepos, v) 450 | } 451 | } 452 | return filteredRepos 453 | } 454 | 455 | type RepositoryConfigEntry struct { 456 | Type string `yaml:"type"` 457 | Host string `yaml:"url"` 458 | Api string `yaml:"api"` 459 | QueryOrder int `yaml:"queryorder"` 460 | Password string `yaml:"pwd"` 461 | User string `yaml:"user"` 462 | IgnoreTLSErrors bool `yaml:"ignoretlserrors"` 463 | } 464 | 465 | func LoadConfig(filename string) ([]RepositoryConfigEntry, error) { 466 | cfg, err := parseFile(filename) 467 | if os.IsNotExist(err) { 468 | fmt.Printf("%s does not exists. Creating...\n", filename) 469 | filename, err = initConfig(filename) 470 | if err != nil { 471 | log.Fatal(err) 472 | return nil, err 473 | } 474 | cfg, err = parseFile(filename) 475 | if err != nil { 476 | log.Fatal(err) 477 | return nil, err 478 | } 479 | } else if err != nil { 480 | log.Fatal(err) 481 | return nil, err 482 | } 483 | return verifyConfig(cfg) 484 | } 485 | 486 | func verifyConfig(repos map[string]RepositoryConfigEntry) ([]RepositoryConfigEntry, error) { 487 | var verifiedConfigRepos []RepositoryConfigEntry 488 | 489 | for k, v := range repos { 490 | mr := getMalwareRepoByName(v.Type) 491 | if mr == NotSupported { 492 | fmt.Printf("%s is not a supported type. Skipping...\n\nSupported types include:\n", v.Type) 493 | allowedMalwareRepoTypes() 494 | fmt.Println("") 495 | } else { 496 | valid := mr.VerifyRepoParams(v) 497 | if !valid { 498 | fmt.Printf(" Skipping %s (Type: %s, URL: %s, API: %s) as it's missing a parameter.\n", k, v.Type, v.Host, v.Api) 499 | } else { 500 | verifiedConfigRepos = append(verifiedConfigRepos, v) 501 | } 502 | } 503 | } 504 | return verifiedConfigRepos, nil 505 | } 506 | 507 | func parseFile(path string) (map[string]RepositoryConfigEntry, error) { 508 | 509 | _, err := os.Stat(path) 510 | if os.IsNotExist(err) { 511 | return nil, err 512 | } 513 | 514 | f, err := os.ReadFile(path) 515 | if err != nil { 516 | fmt.Printf("%v", err) 517 | return nil, err 518 | } 519 | 520 | data := make(map[string]RepositoryConfigEntry) 521 | 522 | err = yaml.Unmarshal(f, &data) 523 | if err != nil { 524 | fmt.Printf("%v", err) 525 | return nil, err 526 | } 527 | 528 | return data, nil 529 | } 530 | 531 | func AddToConfig(filename string) (string, error) { 532 | repoConfigEntries, err := parseFile(filename) 533 | if err != nil { 534 | fmt.Printf("Error parsing %s - %v", filename, err) 535 | return "", err 536 | } 537 | 538 | data, err := createNewEntries(0) 539 | if err != nil { 540 | fmt.Printf("Error creating new config entries : %v", err) 541 | return "", err 542 | } 543 | 544 | finalConfigEntryList := make(map[string]RepositoryConfigEntry) 545 | 546 | entryNumber := 0 547 | 548 | // Add items from reporConfigEntries (pre-existing items) to the final list 549 | for _, v := range repoConfigEntries { 550 | finalConfigEntryList["repository "+fmt.Sprint(entryNumber)] = v 551 | entryNumber++ 552 | } 553 | 554 | // Add items not found in repoConfigEnties (the pre-existing items) 555 | for _, v1 := range data { 556 | found := false 557 | for _, v2 := range repoConfigEntries { 558 | if v1.Type == v2.Type && v1.Host == v1.Api { 559 | found = true 560 | } 561 | } 562 | if !found { 563 | finalConfigEntryList["repository "+fmt.Sprint(entryNumber)] = v1 564 | entryNumber++ 565 | } 566 | } 567 | return writeConfigToFile(filename, finalConfigEntryList) 568 | } 569 | 570 | func initConfig(filename string) (string, error) { 571 | data, err := createNewEntries(0) 572 | if err != nil { 573 | fmt.Printf("Error creating new Repository Config Entries: %v\n", err) 574 | return "", err 575 | } 576 | return writeConfigToFile(filename, data) 577 | } 578 | 579 | func createNewEntries(entryNumber int) (map[string]RepositoryConfigEntry, error) { 580 | data := make(map[string]RepositoryConfigEntry) 581 | 582 | var option int64 583 | 584 | for { 585 | 586 | fmt.Printf("\nEnter the corresponding Repository Config Entry number you want to add to .mlget.yml.\n") 587 | fmt.Printf("Enter 0 to exit.\n") 588 | printAllowedMalwareRepoTypeOptions() 589 | 590 | fmt.Print(">> ") 591 | 592 | fmt.Scan(&option) 593 | 594 | if option > int64(NotSupported) && option <= int64(UploadMWDB) { 595 | entry, err := MalwareRepoType(option).CreateEntry() 596 | if err != nil { 597 | continue 598 | } 599 | data["repository "+fmt.Sprint(entryNumber)] = entry 600 | entryNumber++ 601 | } else if option == 0 { 602 | break 603 | } 604 | } 605 | return data, nil 606 | } 607 | 608 | func writeConfigToFile(filename string, repoConfigEntries map[string]RepositoryConfigEntry) (string, error) { 609 | _, err := os.Stat(filename) 610 | if err == nil { 611 | os.Remove(filename) 612 | } 613 | 614 | file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0600) 615 | if err != nil { 616 | fmt.Printf("error creating file: %v\n", err) 617 | return "", err 618 | } 619 | defer file.Close() 620 | 621 | enc := yaml.NewEncoder(file) 622 | 623 | err = enc.Encode(repoConfigEntries) 624 | if err != nil { 625 | fmt.Printf("error encoding: %v\n", err) 626 | return "", err 627 | } else { 628 | fmt.Printf("Config written to %s\n\n", file.Name()) 629 | } 630 | 631 | return file.Name(), nil 632 | } 633 | 634 | func contains(list []MalwareRepoType, x MalwareRepoType) bool { 635 | for _, item := range list { 636 | if item == x { 637 | return true 638 | } 639 | } 640 | return false 641 | } 642 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/tls" 7 | "encoding/base64" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "log" 13 | "net/http" 14 | "net/url" 15 | "os" 16 | "regexp" 17 | "strings" 18 | 19 | "github.com/yeka/zip" 20 | ) 21 | 22 | type JoeSandboxQuery struct { 23 | Data []JoeSandboxQueryData `'json:"data"` 24 | } 25 | 26 | type JoeSandboxQueryData struct { 27 | Webid string `json:"webid"` 28 | } 29 | 30 | type InquestLabsQuery struct { 31 | Data []InquestLabsQueryData `json:"data"` 32 | Success bool `json:"success"` 33 | } 34 | 35 | type InquestLabsQueryData struct { 36 | Sha256 string `json:"sha256"` 37 | } 38 | 39 | type HybridAnalysisQuery struct { 40 | Submit_name string `json:"submit_name"` 41 | Md5 string `json:"md5"` 42 | Sha1 string `json:"sha1"` 43 | Sha256 string `json:"sha256"` 44 | Sha512 string `json:"sha512"` 45 | } 46 | 47 | type MalwareBazarQuery struct { 48 | Data *MalwareBazarQueryData `json:"data"` 49 | } 50 | 51 | type MalwareBazarQueryData struct { 52 | Sha256_hash string `json:"sha256_hash"` 53 | Sha3_384_hash string `json:"sha3_384_hash"` 54 | Sha1_hash string `json:"sha1_hash"` 55 | Md5_hash string `json:"md5_hash"` 56 | File_name string `json:"file_name"` 57 | } 58 | 59 | type AssemblyLineQuery struct { 60 | Error_message string `json:"api_error_message"` 61 | Response *AssemblyLineQueryResponse `json:"api_response"` 62 | Server_version string `json:"api_server_version"` 63 | Status_Code int `json:"api_status_code"` 64 | } 65 | 66 | type AssemblyLineQueryResponse struct { 67 | AL *AssemblyLineQueryALResponse `json:"al"` 68 | } 69 | 70 | type AssemblyLineQueryALResponse struct { 71 | Error string `json:"error"` 72 | Items []AssemblyLineQueryItem `json:"items"` 73 | } 74 | 75 | type AssemblyLineQueryItem struct { 76 | Classification string `json:"classification"` 77 | Data *AssemblyLineQueryData `json:"data"` 78 | } 79 | 80 | type AssemblyLineQueryData struct { 81 | Md5 string `json:"md5"` 82 | Sha1 string `json:"sha1"` 83 | Sha256 string `json:"sha256"` 84 | } 85 | 86 | type TriageQuery struct { 87 | Data []TriageQueryData `json:"data"` 88 | } 89 | 90 | type TriageQueryData struct { 91 | Id string `json:"id"` 92 | Kind string `json:"kind"` 93 | Filename string `json:"filename"` 94 | } 95 | 96 | type ObjectiveSeeQuery struct { 97 | Malware []ObjectiveSeeData `json:"malware"` 98 | } 99 | 100 | type ObjectiveSeeData struct { 101 | Name string `json:"name"` 102 | Type string `json:"type"` 103 | VirusTotal string `json:"virusTotal"` 104 | MoreInfo string `json:"moreInfo"` 105 | Download string `json:"download"` 106 | Sha256 string 107 | } 108 | 109 | type MalpediaData struct { 110 | Name string 111 | FileBytes []byte 112 | } 113 | 114 | type VirusExchangeData struct { 115 | Md5 string `json:"md5"` 116 | Size int `json:"size"` 117 | Type string `json:"type"` 118 | Sha512 string `json:"sha512"` 119 | Sha256 string `json:"sha256"` 120 | Sha1 string `json:"sha1"` 121 | Download_Link string `json:"download_link"` 122 | } 123 | 124 | func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) { 125 | 126 | fmt.Printf("Downloading Objective-See Malware json from: %s\n\n", uri) 127 | 128 | client := &http.Client{} 129 | response, error := client.Get(uri) 130 | if error != nil { 131 | fmt.Println(error) 132 | return ObjectiveSeeQuery{}, error 133 | } 134 | 135 | defer response.Body.Close() 136 | 137 | if response.StatusCode == http.StatusOK { 138 | byteValue, _ := io.ReadAll(response.Body) 139 | 140 | var data = ObjectiveSeeQuery{} 141 | error = json.Unmarshal(byteValue, &data) 142 | 143 | var unmarshalTypeError *json.UnmarshalTypeError 144 | if errors.As(error, &unmarshalTypeError) { 145 | fmt.Printf(" [!] Failed unmarshaling json. Likely due to the format of the Objective-See json file changing\n") 146 | fmt.Printf(" %s\n", byteValue) 147 | 148 | } else if error != nil { 149 | fmt.Println(error) 150 | return ObjectiveSeeQuery{}, error 151 | } 152 | 153 | fmt.Printf(" Parsing VirusTotal Links for sha256 hashes\n") 154 | re := regexp.MustCompile("[A-Fa-f0-9]{64}") 155 | for k, item := range data.Malware { 156 | if len(item.VirusTotal) > 0 { 157 | matches := re.FindStringSubmatch(item.VirusTotal) 158 | if len(matches) == 1 { 159 | data.Malware[k].Sha256 = matches[0] 160 | } 161 | } 162 | if len(data.Malware[k].Sha256) == 0 { 163 | fmt.Printf(" [!] SHA256 not found for %s : %s\n VirusTotal Link: %s\n", item.Name, item.Type, item.VirusTotal) 164 | } 165 | } 166 | 167 | return data, nil 168 | } else { 169 | return ObjectiveSeeQuery{}, fmt.Errorf("unable to download objective-see json file") 170 | } 171 | } 172 | 173 | func objectivesee(data ObjectiveSeeQuery, hash Hash, doNotExtract bool) (bool, string) { 174 | if hash.HashType != sha256 { 175 | fmt.Printf(" [!] Objective-See only supports SHA256\n Skipping\n") 176 | } 177 | 178 | item, found := findHashInObjectiveSeeList(data.Malware, hash) 179 | 180 | if !found { 181 | return false, "" 182 | } 183 | 184 | if !doNotExtract { 185 | fmt.Printf(" [!] Extraction is not supported for Objective-See\n Try again but with the --noextraction flag\n") 186 | return false, "" 187 | } 188 | 189 | client := &http.Client{} 190 | response, error := client.Get(item.Download) 191 | if error != nil { 192 | fmt.Println(error) 193 | return false, "" 194 | } 195 | 196 | defer response.Body.Close() 197 | 198 | if response.StatusCode != http.StatusOK { 199 | return false, "" 200 | } 201 | 202 | error = writeToFile(response.Body, hash.Hash+".zip") 203 | if error != nil { 204 | fmt.Println(error) 205 | return false, "" 206 | } 207 | 208 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") 209 | if doNotExtract { 210 | return true, hash.Hash + ".zip" 211 | } else { 212 | return false, "" 213 | } 214 | } 215 | 216 | func joesandbox(uri string, api string, hash Hash) (bool, string) { 217 | if api == "" { 218 | fmt.Println(" [!] !! Missing Key !!") 219 | return false, "" 220 | } 221 | 222 | fmt.Printf(" [-] Looking up sandbox ID for: %s\n", hash.Hash) 223 | 224 | query := uri + "/analysis/search" 225 | 226 | _, error := url.ParseRequestURI(query) 227 | if error != nil { 228 | fmt.Printf(" [!] Error when parsing the query uri (%s). Check the value in the config file.\n", query) 229 | fmt.Println(error) 230 | return false, "" 231 | } 232 | 233 | queryData := "q=" + hash.Hash + "&" + "apikey=" + api 234 | values, error := url.ParseQuery(queryData) 235 | if error != nil { 236 | fmt.Printf(" [!] Error when parsing the query data (%s).\n", queryData) 237 | fmt.Println(error) 238 | return false, "" 239 | } 240 | 241 | client := &http.Client{} 242 | response, error := client.PostForm(query, values) 243 | if error != nil { 244 | fmt.Println(error) 245 | return false, "" 246 | } 247 | 248 | defer response.Body.Close() 249 | 250 | if response.StatusCode == http.StatusOK { 251 | 252 | byteValue, error := io.ReadAll(response.Body) 253 | if error != nil { 254 | fmt.Println(error) 255 | return false, "" 256 | } 257 | 258 | var data = JoeSandboxQuery{} 259 | error = json.Unmarshal(byteValue, &data) 260 | 261 | if error != nil { 262 | fmt.Println(error) 263 | return false, "" 264 | } 265 | 266 | if len(data.Data) > 0 { 267 | sandboxid := data.Data[0].Webid 268 | fmt.Printf(" [-] Hash %s Sandbox ID: %s\n", hash.Hash, sandboxid) 269 | 270 | // Download Sample using Sample ID 271 | return joesandboxDownload(uri, api, sandboxid, hash) 272 | } else { 273 | return false, "" 274 | } 275 | } else if response.StatusCode == http.StatusForbidden { 276 | fmt.Printf(" [!] Not authorized. Check the URL in the config.\n JoeSandbox does have more than one API endpoint.\n Check your documentation.\n") 277 | return false, "" 278 | } else { 279 | return false, "" 280 | } 281 | 282 | } 283 | 284 | func joesandboxDownload(uri string, api string, sandboxid string, hash Hash) (bool, string) { 285 | query := uri + "/analysis/download" 286 | 287 | _, error := url.ParseRequestURI(query) 288 | if error != nil { 289 | fmt.Printf(" [!] Error when parsing the query uri (%s). Check the value in the config file.\n", query) 290 | fmt.Println(error) 291 | return false, "" 292 | } 293 | 294 | queryData := "webid=" + sandboxid + "&" + "apikey=" + api + "&" + "type=sample" 295 | values, error := url.ParseQuery(queryData) 296 | if error != nil { 297 | fmt.Printf(" [!] Error when parsing the query data (%s).\n", queryData) 298 | fmt.Println(error) 299 | return false, "" 300 | } 301 | 302 | client := &http.Client{} 303 | response, error := client.PostForm(query, values) 304 | if error != nil { 305 | fmt.Println(error) 306 | return false, "" 307 | } 308 | 309 | defer response.Body.Close() 310 | 311 | if response.StatusCode == http.StatusOK { 312 | error = writeToFile(response.Body, hash.Hash) 313 | if error != nil { 314 | fmt.Println(error) 315 | return false, "" 316 | } 317 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 318 | return true, hash.Hash 319 | } else { 320 | return false, hash.Hash 321 | } 322 | } 323 | 324 | func capesandbox(uri string, api string, hash Hash) (bool, string) { 325 | if api == "" { 326 | fmt.Println(" [!] !! Missing Key !!") 327 | return false, "" 328 | } 329 | return capesandboxDownload(uri, api, hash) 330 | } 331 | 332 | func capesandboxDownload(uri string, api string, hash Hash) (bool, string) { 333 | query := uri + "/files/get/" + url.QueryEscape(hash.HashType.String()) + "/" + url.QueryEscape(hash.Hash) + "/" 334 | 335 | request, err := http.NewRequest("GET", query, nil) 336 | if err != nil { 337 | fmt.Println(err) 338 | return false, "" 339 | } 340 | 341 | request.Header.Set("Authorization", "Token "+api) 342 | client := &http.Client{} 343 | response, error := client.Do(request) 344 | if error != nil { 345 | fmt.Println(error) 346 | return false, "" 347 | } 348 | defer response.Body.Close() 349 | 350 | if response.StatusCode == http.StatusOK { 351 | 352 | if response.Header["Content-Type"][0] == "application/json" { 353 | return false, "" 354 | } 355 | 356 | error = writeToFile(response.Body, hash.Hash) 357 | if error != nil { 358 | fmt.Println(error) 359 | return false, "" 360 | } 361 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 362 | return true, hash.Hash 363 | } else if response.StatusCode == http.StatusForbidden { 364 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 365 | return false, "" 366 | } else { 367 | return false, "" 368 | } 369 | } 370 | 371 | func inquestlabs(uri string, api string, hash Hash) (bool, string) { 372 | if api == "" { 373 | fmt.Println(" [!] !! Missing Key !!") 374 | return false, "" 375 | } 376 | 377 | if hash.HashType != sha256 { 378 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 379 | 380 | query := uri + "/dfi/search/hash/" + url.PathEscape(hash.HashType.String()) + "?hash=" + url.QueryEscape(hash.Hash) 381 | 382 | _, error := url.ParseQuery(query) 383 | if error != nil { 384 | fmt.Println(" [!] Issue creating hash lookup query url") 385 | fmt.Println(error) 386 | return false, "" 387 | } 388 | 389 | request, err := http.NewRequest("GET", query, nil) 390 | if err != nil { 391 | fmt.Println(err) 392 | return false, "" 393 | } 394 | 395 | client := &http.Client{} 396 | response, error := client.Do(request) 397 | if error != nil { 398 | fmt.Println(error) 399 | return false, "" 400 | } 401 | defer response.Body.Close() 402 | 403 | if response.StatusCode == http.StatusOK { 404 | 405 | byteValue, _ := io.ReadAll(response.Body) 406 | 407 | var data = InquestLabsQuery{} 408 | error = json.Unmarshal(byteValue, &data) 409 | 410 | var unmarshalTypeError *json.UnmarshalTypeError 411 | if errors.As(error, &unmarshalTypeError) { 412 | fmt.Printf(" [!] Failed unmarshaling json. This could be due to the API changing or\n just no data inside the data array was returned - aka. sha256 hash was not found.\n") 413 | fmt.Printf(" %s\n", byteValue) 414 | 415 | } else if error != nil { 416 | fmt.Println(error) 417 | return false, "" 418 | } 419 | 420 | if !data.Success { 421 | return false, "" 422 | } 423 | 424 | if len(data.Data) == 0 { 425 | return false, "" 426 | } 427 | 428 | if data.Data[0].Sha256 == "" { 429 | return false, "" 430 | } 431 | hash.HashType = sha256 432 | hash.Hash = data.Data[0].Sha256 433 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 434 | 435 | } else if response.StatusCode == http.StatusForbidden { 436 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 437 | return false, "" 438 | } 439 | } 440 | 441 | if hash.HashType == sha256 { 442 | return inquestlabsDownload(uri, api, hash) 443 | } 444 | return false, "" 445 | } 446 | 447 | func inquestlabsDownload(uri string, api string, hash Hash) (bool, string) { 448 | query := uri + "/dfi/download?sha256=" + url.QueryEscape(hash.Hash) 449 | 450 | _, error := url.ParseQuery(query) 451 | if error != nil { 452 | fmt.Println(error) 453 | return false, "" 454 | } 455 | 456 | request, err := http.NewRequest("GET", query, nil) 457 | if err != nil { 458 | fmt.Println(err) 459 | return false, "" 460 | } 461 | 462 | request.Header.Set("Authorization", api) 463 | client := &http.Client{} 464 | response, error := client.Do(request) 465 | if error != nil { 466 | fmt.Println(error) 467 | return false, "" 468 | } 469 | defer response.Body.Close() 470 | 471 | if response.StatusCode == http.StatusOK { 472 | 473 | error = writeToFile(response.Body, hash.Hash) 474 | if error != nil { 475 | fmt.Println(error) 476 | return false, "" 477 | } 478 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 479 | return true, hash.Hash 480 | } else if response.StatusCode == http.StatusForbidden { 481 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 482 | return false, "" 483 | } else { 484 | return false, "" 485 | } 486 | } 487 | 488 | func virustotal(uri string, api string, hash Hash) (bool, string) { 489 | if api == "" { 490 | fmt.Println(" [!] !! Missing Key !!") 491 | return false, "" 492 | } 493 | return virustotalDownload(uri, api, hash) 494 | } 495 | 496 | func virustotalDownload(uri string, api string, hash Hash) (bool, string) { 497 | query := uri + "/files/" + url.PathEscape(hash.Hash) + "/download" 498 | 499 | request, err := http.NewRequest("GET", query, nil) 500 | if err != nil { 501 | fmt.Println(err) 502 | return false, "" 503 | } 504 | 505 | request.Header.Set("x-apikey", api) 506 | client := &http.Client{} 507 | response, error := client.Do(request) 508 | if error != nil { 509 | fmt.Println(error) 510 | return false, "" 511 | } 512 | defer response.Body.Close() 513 | 514 | if response.StatusCode == http.StatusOK { 515 | 516 | error = writeToFile(response.Body, hash.Hash) 517 | if error != nil { 518 | fmt.Println(error) 519 | return false, "" 520 | } 521 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 522 | return true, hash.Hash 523 | } else if response.StatusCode == http.StatusForbidden { 524 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 525 | return false, "" 526 | } else { 527 | return false, "" 528 | } 529 | } 530 | 531 | func mwdb(uri string, api string, hash Hash) (bool, string) { 532 | if api == "" { 533 | fmt.Println(" [!] !! Missing Key !!") 534 | return false, "" 535 | } 536 | return mwdbDownload(uri, api, hash) 537 | } 538 | 539 | func mwdbDownload(uri string, api string, hash Hash) (bool, string) { 540 | query := uri + "/file/" + url.PathEscape(hash.Hash) + "/download" 541 | 542 | request, err := http.NewRequest("GET", query, nil) 543 | if err != nil { 544 | fmt.Println(err) 545 | return false, "" 546 | } 547 | 548 | request.Header.Set("Authorization", "Bearer "+api) 549 | client := &http.Client{} 550 | response, error := client.Do(request) 551 | if error != nil { 552 | fmt.Println(error) 553 | return false, "" 554 | } 555 | defer response.Body.Close() 556 | 557 | if response.StatusCode == http.StatusOK { 558 | 559 | error = writeToFile(response.Body, hash.Hash) 560 | if error != nil { 561 | fmt.Println(error) 562 | return false, "" 563 | } 564 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 565 | return true, hash.Hash 566 | } else if response.StatusCode == http.StatusForbidden { 567 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 568 | return false, "" 569 | } else { 570 | return false, "" 571 | } 572 | } 573 | 574 | func polyswarm(uri string, api string, hash Hash) (bool, string) { 575 | if api == "" { 576 | fmt.Println(" [!] !! Missing Key !!") 577 | return false, "" 578 | } 579 | return polyswarmDownload(uri, api, hash) 580 | } 581 | 582 | func polyswarmDownload(uri string, api string, hash Hash) (bool, string) { 583 | query := "/download/" + url.PathEscape(hash.HashType.String()) + "/" + url.PathEscape(hash.Hash) 584 | 585 | _, error := url.ParseQuery(query) 586 | if error != nil { 587 | fmt.Println(error) 588 | return false, "" 589 | } 590 | 591 | request, err := http.NewRequest("GET", uri+query, nil) 592 | if err != nil { 593 | fmt.Println(err) 594 | return false, "" 595 | } 596 | 597 | request.Header.Set("Authorization", api) 598 | client := &http.Client{} 599 | response, error := client.Do(request) 600 | if error != nil { 601 | fmt.Println(error) 602 | return false, "" 603 | } 604 | defer response.Body.Close() 605 | 606 | if response.StatusCode == http.StatusOK { 607 | 608 | error = writeToFile(response.Body, hash.Hash) 609 | if error != nil { 610 | fmt.Println(error) 611 | return false, "" 612 | } 613 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 614 | return true, hash.Hash 615 | } else if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { 616 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 617 | return false, "" 618 | } else { 619 | return false, "" 620 | } 621 | } 622 | 623 | func hybridAnlysis(uri string, api string, hash Hash, doNotExtract bool) (bool, string) { 624 | if api == "" { 625 | fmt.Println(" [!] !! Missing Key !!") 626 | return false, "" 627 | } 628 | 629 | if hash.HashType != sha256 { 630 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 631 | 632 | pData := []byte("hash=" + hash.Hash) 633 | request, error := http.NewRequest("POST", uri+"/search/hash", bytes.NewBuffer(pData)) 634 | 635 | if error != nil { 636 | fmt.Println(error) 637 | return false, "" 638 | } 639 | 640 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 641 | request.Header.Set("user-agent", "Falcon Sandbox") 642 | request.Header.Set("api-key", api) 643 | client := &http.Client{} 644 | response, error := client.Do(request) 645 | if error != nil { 646 | fmt.Println(error) 647 | return false, "" 648 | } 649 | defer response.Body.Close() 650 | 651 | if response.StatusCode == http.StatusForbidden { 652 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 653 | return false, "" 654 | } 655 | byteValue, _ := io.ReadAll(response.Body) 656 | 657 | var data = HybridAnalysisQuery{} 658 | error = json.Unmarshal(byteValue, &data) 659 | 660 | if error != nil { 661 | fmt.Println(error) 662 | return false, "" 663 | } 664 | 665 | if data.Sha256 == "" { 666 | return false, "" 667 | } 668 | hash.Hash = data.Sha256 669 | hash.HashType = sha256 670 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 671 | 672 | } 673 | 674 | if hash.HashType == sha256 { 675 | return hybridAnlysisDownload(uri, api, hash, doNotExtract) 676 | } 677 | return false, "" 678 | } 679 | 680 | func hybridAnlysisDownload(uri string, api string, hash Hash, extract bool) (bool, string) { 681 | request, error := http.NewRequest("GET", uri+"/overview/"+url.PathEscape(hash.Hash)+"/sample", nil) 682 | 683 | request.Header.Set("accept", "application/gzip") 684 | request.Header.Set("user-agent", "Falcon Sandbox") 685 | request.Header.Set("api-key", api) 686 | 687 | if error != nil { 688 | fmt.Println(error) 689 | return false, "" 690 | } 691 | client := &http.Client{} 692 | response, error := client.Do(request) 693 | if error != nil { 694 | fmt.Println(error) 695 | return false, "" 696 | } 697 | defer response.Body.Close() 698 | 699 | if response.StatusCode == http.StatusForbidden { 700 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\nCould also be that the sample is not allowed to be downloaded.\n") 701 | return false, "" 702 | } else if response.StatusCode != http.StatusOK { 703 | return false, "" 704 | } 705 | 706 | error = writeToFile(response.Body, hash.Hash+".gzip") 707 | if error != nil { 708 | fmt.Println(error) 709 | return false, "" 710 | } 711 | if doNotExtractFlag { 712 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".gzip") 713 | return true, hash.Hash + ".gzip" 714 | } else { 715 | fmt.Println(" [-] Extracting...") 716 | err := extractGzip(hash.Hash) 717 | if err != nil { 718 | fmt.Println(error) 719 | return false, "" 720 | } else { 721 | fmt.Printf(" [-] Extracted %s\n", hash.Hash) 722 | } 723 | os.Remove(hash.Hash + ".gzip") 724 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 725 | return true, hash.Hash 726 | 727 | } 728 | } 729 | 730 | func triage(uri string, api string, hash Hash) (bool, string) { 731 | if api == "" { 732 | fmt.Println(" [!] !! Missing Key !!") 733 | return false, "" 734 | } 735 | 736 | // Look up hash to get Sample ID 737 | query := "query=" + url.QueryEscape(hash.HashType.String()) + ":" + url.QueryEscape(hash.Hash) 738 | _, error := url.ParseQuery(query) 739 | if error != nil { 740 | fmt.Println(error) 741 | return false, "" 742 | } 743 | request, error := http.NewRequest("GET", uri+"/search?"+query, nil) 744 | if error != nil { 745 | fmt.Println(error) 746 | return false, "" 747 | } 748 | 749 | request.Header.Set("Authorization", "Bearer "+api) 750 | 751 | client := &http.Client{} 752 | response, error := client.Do(request) 753 | if error != nil { 754 | fmt.Println(error) 755 | return false, "" 756 | } 757 | defer response.Body.Close() 758 | 759 | if response.StatusCode == http.StatusOK { 760 | 761 | byteValue, error := io.ReadAll(response.Body) 762 | if error != nil { 763 | fmt.Println(error) 764 | return false, "" 765 | } 766 | 767 | var data = TriageQuery{} 768 | error = json.Unmarshal(byteValue, &data) 769 | 770 | if error != nil { 771 | fmt.Println(error) 772 | return false, "" 773 | } 774 | 775 | if len(data.Data) > 0 { 776 | sampleId := data.Data[0].Id 777 | fmt.Printf(" [-] Hash %s Sample ID: %s\n", hash.Hash, sampleId) 778 | 779 | // Download Sample using Sample ID 780 | return traigeDownload(uri, api, sampleId, hash) 781 | } else { 782 | return false, "" 783 | } 784 | } else if response.StatusCode == http.StatusForbidden { 785 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 786 | return false, "" 787 | } else { 788 | return false, "" 789 | } 790 | } 791 | 792 | func traigeDownload(uri string, api string, sampleId string, hash Hash) (bool, string) { 793 | request, error := http.NewRequest("GET", uri+"/samples/"+url.PathEscape(sampleId)+"/sample", nil) 794 | if error != nil { 795 | fmt.Println(error) 796 | return false, "" 797 | } 798 | 799 | request.Header.Set("Authorization", "Bearer "+api) 800 | 801 | client := &http.Client{} 802 | response, error := client.Do(request) 803 | if error != nil { 804 | fmt.Println(error) 805 | return false, "" 806 | } 807 | 808 | defer response.Body.Close() 809 | 810 | error = writeToFile(response.Body, hash.Hash) 811 | if error != nil { 812 | fmt.Println(error) 813 | return false, "" 814 | } 815 | // Triage will download an archive file that contians the hash in question sometimes versus the actual sample being requested 816 | hashMatch, _ := hash.ValidateFile(hash.Hash) 817 | if !hashMatch { 818 | files, err := extractPwdZip(hash.Hash, "", false, hash) 819 | if err != nil { 820 | fmt.Println(error) 821 | return false, "" 822 | } 823 | 824 | found := false 825 | 826 | fmt.Printf(" [-] The downloaded file appears to be a zip file in which the requested file should be located.\n") 827 | for _, f := range files { 828 | fmt.Printf(" [-] Checking file: %s\n", f.Name) 829 | hashMatch, _ = hash.ValidateFile(f.Name) 830 | if !hashMatch { 831 | err = os.Remove(f.Name) 832 | if err != nil { 833 | fmt.Println(" [!] Error when deleting file: ", f.Name) 834 | fmt.Println(err) 835 | } 836 | } else { 837 | fmt.Printf(" [+] %s is hash %s\n", f.Name, hash.Hash) 838 | err = os.Rename(f.Name, hash.Hash) 839 | if err != nil { 840 | fmt.Println(" [!] Error when renaming file: ", f.Name) 841 | fmt.Println(err) 842 | } else { 843 | found = true 844 | } 845 | } 846 | } 847 | if !found { 848 | fmt.Printf(" [!] Hash %s not found\n", hash.Hash) 849 | err = os.Remove(hash.Hash) 850 | if err != nil { 851 | fmt.Println(" [!] Error when deleting file: ", hash.Hash) 852 | fmt.Println(err) 853 | } 854 | return false, "" 855 | } else { 856 | fmt.Printf(" [+] Found %s\n", hash.Hash) 857 | return true, hash.Hash 858 | } 859 | } else { 860 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 861 | return true, hash.Hash 862 | } 863 | } 864 | 865 | func malshare(url string, api string, hash Hash) (bool, string) { 866 | if api == "" { 867 | fmt.Println(" [!] !! Missing Key !!") 868 | return false, "" 869 | } 870 | 871 | return malshareDownload(url, api, hash) 872 | } 873 | 874 | func malshareDownload(uri string, api string, hash Hash) (bool, string) { 875 | query := "api_key=" + url.QueryEscape(api) + "&action=getfile&hash=" + url.QueryEscape(hash.Hash) 876 | 877 | _, error := url.ParseQuery(query) 878 | if error != nil { 879 | fmt.Println(error) 880 | return false, "" 881 | } 882 | 883 | client := &http.Client{} 884 | response, error := client.Get(uri + "/api.php?" + query) 885 | if error != nil { 886 | fmt.Println(error) 887 | return false, "" 888 | } 889 | defer response.Body.Close() 890 | 891 | if response.StatusCode == http.StatusOK { 892 | 893 | error = writeToFile(response.Body, hash.Hash) 894 | if error != nil { 895 | fmt.Println(error) 896 | return false, "" 897 | } 898 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 899 | return true, hash.Hash 900 | } else if response.StatusCode == http.StatusForbidden { 901 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 902 | return false, "" 903 | } else { 904 | return false, "" 905 | } 906 | } 907 | 908 | func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (bool, string) { 909 | if hash.HashType != sha256 { 910 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 911 | 912 | query := "query=get_file&hash=" + hash.Hash 913 | values, error := url.ParseQuery(query) 914 | if error != nil { 915 | fmt.Println(error) 916 | return false, "" 917 | } 918 | 919 | client := &http.Client{} 920 | response, error := client.PostForm(uri, values) 921 | if error != nil { 922 | fmt.Println(error) 923 | return false, "" 924 | } 925 | defer response.Body.Close() 926 | 927 | if response.StatusCode == http.StatusForbidden { 928 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 929 | return false, "" 930 | } 931 | 932 | byteValue, _ := io.ReadAll(response.Body) 933 | 934 | var data = MalwareBazarQuery{} 935 | error = json.Unmarshal(byteValue, &data) 936 | 937 | if error != nil { 938 | fmt.Println(error) 939 | return false, "" 940 | } 941 | 942 | if data.Data == nil { 943 | return false, "" 944 | } 945 | hash.Hash = data.Data.Sha256_hash 946 | hash.HashType = sha256 947 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 948 | 949 | } 950 | 951 | if hash.HashType == sha256 { 952 | return malwareBazaarDownload(uri, hash, doNotExtract, password) 953 | } 954 | return false, "" 955 | } 956 | 957 | func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password string) (bool, string) { 958 | query := "query=get_file&sha256_hash=" + hash.Hash 959 | values, err := url.ParseQuery(query) 960 | if err != nil { 961 | fmt.Println(err) 962 | return false, "" 963 | } 964 | 965 | client := &http.Client{} 966 | 967 | response, err := client.PostForm(uri, values) 968 | if err != nil { 969 | fmt.Println(err) 970 | return false, "" 971 | } 972 | 973 | defer response.Body.Close() 974 | 975 | if response.Header["Content-Type"][0] == "application/json" { 976 | if response.StatusCode == http.StatusMethodNotAllowed { 977 | if !strings.HasSuffix(uri, "/") { 978 | fmt.Printf(" [!] Trying again with a trailing slash: %s/\n", uri) 979 | return malwareBazaarDownload(uri+"/", hash, doNotExtract, password) 980 | } else { 981 | fmt.Printf(" [!] Normally the response code: %s means that the provided URL %s needs a trailing slash (to avoid the redirect), but this already has a trailing slash.\nPlease file a bug report at https://github.com/xorhex/mlget/issues\n", response.Status, uri) 982 | } 983 | } else { 984 | fmt.Printf(" [!] %s\n", response.Status) 985 | } 986 | return false, "" 987 | } 988 | 989 | err = writeToFile(response.Body, hash.Hash+".zip") 990 | if err != nil { 991 | fmt.Println(err) 992 | return false, "" 993 | } 994 | 995 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") 996 | if doNotExtract { 997 | return true, hash.Hash + ".zip" 998 | } else { 999 | fmt.Println(" [-] Extracting...") 1000 | files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) 1001 | if err != nil { 1002 | fmt.Println(err) 1003 | return false, "" 1004 | } else { 1005 | for _, f := range files { 1006 | fmt.Printf(" [-] Extracted %s\n", f.Name) 1007 | } 1008 | } 1009 | os.Remove(hash.Hash + ".zip") 1010 | return true, hash.Hash 1011 | } 1012 | } 1013 | 1014 | func filescanio(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1015 | if api == "" { 1016 | fmt.Println(" [!] !! Missing Key !!") 1017 | return false, "" 1018 | } 1019 | return filescaniodownload(uri, api, hash, doNotExtract, password) 1020 | } 1021 | 1022 | func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1023 | query := "type=raw" 1024 | _, error := url.ParseQuery(query) 1025 | if error != nil { 1026 | fmt.Println(error) 1027 | return false, "" 1028 | } 1029 | 1030 | request, error := http.NewRequest("GET", uri+"/files/"+url.PathEscape(hash.Hash)+"?"+query, nil) 1031 | if error != nil { 1032 | fmt.Println(error) 1033 | return false, "" 1034 | } 1035 | 1036 | request.Header.Set("X-Api-Key", api) 1037 | 1038 | client := &http.Client{} 1039 | response, error := client.Do(request) 1040 | if error != nil { 1041 | fmt.Println(error) 1042 | return false, "" 1043 | } 1044 | 1045 | defer response.Body.Close() 1046 | 1047 | if response.StatusCode == 404 { 1048 | return false, "" 1049 | } else if response.StatusCode == 422 { 1050 | fmt.Printf(" [!] Validation Error.\n") 1051 | return false, "" 1052 | } else if response.StatusCode == http.StatusForbidden { 1053 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1054 | fmt.Printf(" [!] If you are sure this is correct, then test downloading a sample you've access to in their platform. It should work.\n") 1055 | fmt.Printf(" [!] Not sure why it does not just return a 404 instead; when the creds are correct but the file is not available.\n") 1056 | return false, "" 1057 | } 1058 | 1059 | error = writeToFile(response.Body, hash.Hash+".zip") 1060 | if error != nil { 1061 | fmt.Println(error) 1062 | return false, "" 1063 | } 1064 | 1065 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash+".zip") 1066 | if doNotExtract { 1067 | return true, hash.Hash + ".zip" 1068 | } else { 1069 | fmt.Println(" [-] Extracting...") 1070 | files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) 1071 | if err != nil { 1072 | fmt.Println(err) 1073 | return false, "" 1074 | } else { 1075 | for _, f := range files { 1076 | fmt.Printf(" [-] Extracted %s\n", f.Name) 1077 | } 1078 | } 1079 | os.Remove(hash.Hash + ".zip") 1080 | return true, hash.Hash 1081 | } 1082 | } 1083 | 1084 | func vxshare(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1085 | if api == "" { 1086 | fmt.Println(" [!] !! Missing Key !!") 1087 | return false, "" 1088 | } 1089 | return vxsharedownload(uri, api, hash, doNotExtract, password) 1090 | } 1091 | 1092 | func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, password string) (bool, string) { 1093 | query := "apikey=" + url.QueryEscape(api) + "&hash=" + url.QueryEscape(hash.Hash) 1094 | _, error := url.ParseQuery(query) 1095 | if error != nil { 1096 | fmt.Println(error) 1097 | return false, "" 1098 | } 1099 | 1100 | client := &http.Client{} 1101 | response, error := client.Get(uri + "/download?" + query) 1102 | if error != nil { 1103 | fmt.Println(error) 1104 | return false, "" 1105 | } 1106 | 1107 | defer response.Body.Close() 1108 | 1109 | if response.StatusCode == 404 { 1110 | return false, "" 1111 | } else if response.StatusCode == 204 { 1112 | fmt.Printf(" [!] Request rate limit exceeded. You are making more requests than are allowed or have exceeded your quota.\n") 1113 | return false, "" 1114 | } else if response.StatusCode == http.StatusForbidden { 1115 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1116 | return false, "" 1117 | } 1118 | 1119 | error = writeToFile(response.Body, hash.Hash+".zip") 1120 | if error != nil { 1121 | fmt.Println(error) 1122 | return false, "" 1123 | } 1124 | 1125 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1126 | if doNotExtract { 1127 | return true, hash.Hash + ".zip" 1128 | } else { 1129 | fmt.Println(" [-] Extracting...") 1130 | files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) 1131 | if err != nil { 1132 | fmt.Println(err) 1133 | return false, "" 1134 | } else { 1135 | for _, f := range files { 1136 | fmt.Printf(" [-] Extracted %s\n", f.Name) 1137 | } 1138 | } 1139 | os.Remove(hash.Hash + ".zip") 1140 | return true, hash.Hash 1141 | } 1142 | } 1143 | 1144 | func unpacme(uri string, api string, hash Hash) (bool, string) { 1145 | if api == "" { 1146 | fmt.Println(" [!] !! Missing Key !!") 1147 | return false, "" 1148 | } 1149 | 1150 | if hash.HashType != sha256 { 1151 | fmt.Printf(" [!] UnpacMe only supports SHA256\n Skipping\n") 1152 | return false, "" 1153 | } 1154 | 1155 | return unpacmeDownload(uri, api, hash) 1156 | } 1157 | 1158 | func unpacmeDownload(uri string, api string, hash Hash) (bool, string) { 1159 | request, error := http.NewRequest("GET", uri+"/private/download/"+url.PathEscape(hash.Hash), nil) 1160 | if error != nil { 1161 | fmt.Println(error) 1162 | return false, "" 1163 | } 1164 | 1165 | request.Header.Set("Authorization", "Key "+api) 1166 | 1167 | client := &http.Client{} 1168 | response, error := client.Do(request) 1169 | if error != nil { 1170 | fmt.Println(error) 1171 | return false, "" 1172 | } 1173 | 1174 | defer response.Body.Close() 1175 | 1176 | if response.StatusCode == 404 { 1177 | return false, "" 1178 | } else if response.StatusCode == http.StatusForbidden { 1179 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1180 | return false, "" 1181 | } else if response.StatusCode == http.StatusUnauthorized { 1182 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1183 | return false, "" 1184 | } 1185 | 1186 | error = writeToFile(response.Body, hash.Hash) 1187 | if error != nil { 1188 | fmt.Println(error) 1189 | return false, "" 1190 | } 1191 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1192 | return true, hash.Hash 1193 | } 1194 | 1195 | func urlscanio(uri string, api string, hash Hash) (bool, string) { 1196 | if api == "" { 1197 | fmt.Println(" [!] !! Missing Key !!") 1198 | return false, "" 1199 | } 1200 | 1201 | if hash.HashType != sha256 { 1202 | fmt.Printf(" [!] URLScanIO only supports SHA256\n Skipping\n") 1203 | } 1204 | 1205 | return urlscanioDownload(uri, api, hash) 1206 | } 1207 | 1208 | func urlscanioDownload(uri string, api string, hash Hash) (bool, string) { 1209 | //downloads/" 1210 | request, error := http.NewRequest("GET", uri+"/"+url.PathEscape(hash.Hash)+"/", nil) 1211 | if error != nil { 1212 | fmt.Println(error) 1213 | return false, "" 1214 | } 1215 | 1216 | request.Header.Set("API-Key", api) 1217 | 1218 | client := &http.Client{} 1219 | response, error := client.Do(request) 1220 | if error != nil { 1221 | fmt.Println(error) 1222 | return false, "" 1223 | } 1224 | 1225 | defer response.Body.Close() 1226 | 1227 | if response.StatusCode == 404 { 1228 | return false, "" 1229 | } else if response.StatusCode == http.StatusForbidden { 1230 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1231 | return false, "" 1232 | } 1233 | 1234 | error = writeToFile(response.Body, hash.Hash) 1235 | if error != nil { 1236 | fmt.Println(error) 1237 | return false, "" 1238 | } 1239 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1240 | return true, hash.Hash 1241 | } 1242 | 1243 | func malpedia(uri string, api string, hash Hash) (bool, string) { 1244 | if api == "" { 1245 | fmt.Println(" [!] !! Missing Key !!") 1246 | return false, "" 1247 | } 1248 | 1249 | if hash.HashType == sha1 { 1250 | fmt.Printf(" [!] Malpedia only supports MD5 and SHA256\n Skipping\n") 1251 | } 1252 | 1253 | return malpediaDownload(uri, api, hash) 1254 | 1255 | } 1256 | 1257 | func malpediaDownload(uri string, api string, hash Hash) (bool, string) { 1258 | ///get/sample//raw 1259 | // Malpedia returns a json file of the file and all of the related files (base64 encoded) 1260 | // No way to determine which file is which, so hashing each file found to identify the correct file 1261 | request, error := http.NewRequest("GET", uri+"/get/sample/"+url.PathEscape(hash.Hash)+"/raw", nil) 1262 | if error != nil { 1263 | fmt.Println(error) 1264 | return false, "" 1265 | } 1266 | 1267 | request.Header.Set("Authorization", "apitoken "+api) 1268 | 1269 | client := &http.Client{} 1270 | response, error := client.Do(request) 1271 | if error != nil { 1272 | fmt.Println(error) 1273 | return false, "" 1274 | } 1275 | 1276 | defer response.Body.Close() 1277 | 1278 | if response.StatusCode == 404 { 1279 | return false, "" 1280 | } else if response.StatusCode == http.StatusForbidden { 1281 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1282 | return false, "" 1283 | } 1284 | 1285 | byteValue, _ := io.ReadAll(response.Body) 1286 | jsonParseSuccesful, mpData := parseMalpediaJson(byteValue) 1287 | if jsonParseSuccesful { 1288 | for _, item := range mpData { 1289 | match, _ := hash.Validate(item.FileBytes) 1290 | if match { 1291 | error = writeBytesToFile(item.FileBytes, hash.Hash) 1292 | if error != nil { 1293 | fmt.Println(error) 1294 | return false, "" 1295 | } 1296 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1297 | return true, hash.Hash 1298 | } 1299 | } 1300 | } 1301 | return false, "" 1302 | } 1303 | 1304 | func parseMalpediaJson(byteValue []byte) (bool, []MalpediaData) { 1305 | // Code copied and modified fromm https://gist.github.com/mjohnsullivan/24647cae50928a34b5cc 1306 | // Unmarshal using a generic interface 1307 | var f interface{} 1308 | err := json.Unmarshal(byteValue, &f) 1309 | if err != nil { 1310 | fmt.Println("Error parsing JSON: ", err) 1311 | return false, nil 1312 | } 1313 | 1314 | // JSON object parses into a map with string keys 1315 | itemsMap := f.(map[string]interface{}) 1316 | var malpediaJsonItems []MalpediaData 1317 | 1318 | for key := range itemsMap { 1319 | var item MalpediaData 1320 | item.Name = key 1321 | rawDecodedValue, err := base64.StdEncoding.DecodeString(itemsMap[key].(string)) 1322 | if err != nil { 1323 | fmt.Println("Error base64 decoding the json value: ", err) 1324 | return false, nil 1325 | } 1326 | item.FileBytes = rawDecodedValue 1327 | malpediaJsonItems = append(malpediaJsonItems, item) 1328 | } 1329 | return true, malpediaJsonItems 1330 | } 1331 | 1332 | func assemblyline(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) { 1333 | if api == "" { 1334 | fmt.Println(" [!] !! Missing Key !!") 1335 | return false, "" 1336 | } 1337 | if user == "" { 1338 | fmt.Println(" [!] !! Missing User !!") 1339 | return false, "" 1340 | } 1341 | 1342 | if hash.HashType != sha256 { 1343 | fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) 1344 | 1345 | request, error := http.NewRequest("GET", uri+"/hash_search/"+url.PathEscape(hash.Hash)+"/", nil) 1346 | if error != nil { 1347 | fmt.Println(error) 1348 | return false, "" 1349 | } 1350 | 1351 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 1352 | request.Header.Set("x-user", user) 1353 | request.Header.Set("x-apikey", api) 1354 | 1355 | tr := &http.Transport{} 1356 | if ignoretlserrors { 1357 | fmt.Printf(" [!] Ignoring Certificate Errors.\n") 1358 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 1359 | } 1360 | 1361 | client := &http.Client{Transport: tr} 1362 | response, error := client.Do(request) 1363 | if error != nil { 1364 | fmt.Println(error) 1365 | return false, "" 1366 | } 1367 | defer response.Body.Close() 1368 | 1369 | if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { 1370 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 1371 | return false, "" 1372 | } 1373 | 1374 | byteValue, _ := io.ReadAll(response.Body) 1375 | 1376 | var data = AssemblyLineQuery{} 1377 | error = json.Unmarshal(byteValue, &data) 1378 | 1379 | if error != nil { 1380 | fmt.Println(error) 1381 | return false, "" 1382 | } 1383 | 1384 | if data.Response.AL == nil { 1385 | return false, "" 1386 | } 1387 | 1388 | if len(data.Response.AL.Items) > 0 { 1389 | hash.Hash = data.Response.AL.Items[0].Data.Sha256 1390 | hash.HashType = sha256 1391 | fmt.Printf(" [-] Using hash %s\n", hash.Hash) 1392 | } else { 1393 | return false, "" 1394 | } 1395 | } 1396 | 1397 | return assemblylineDownload(uri, user, api, ignoretlserrors, hash) 1398 | 1399 | } 1400 | 1401 | func assemblylineDownload(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) { 1402 | request, error := http.NewRequest("GET", uri+"/file/download/"+url.PathEscape(hash.Hash)+"/?encoding=raw", nil) 1403 | if error != nil { 1404 | fmt.Println(error) 1405 | return false, "" 1406 | } 1407 | 1408 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 1409 | request.Header.Set("x-user", user) 1410 | request.Header.Set("x-apikey", api) 1411 | 1412 | tr := &http.Transport{} 1413 | if ignoretlserrors { 1414 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 1415 | } 1416 | 1417 | client := &http.Client{Transport: tr} 1418 | response, error := client.Do(request) 1419 | if error != nil { 1420 | fmt.Println(error) 1421 | return false, "" 1422 | } 1423 | 1424 | defer response.Body.Close() 1425 | 1426 | if response.StatusCode == http.StatusNotFound { 1427 | return false, "" 1428 | } else if response.StatusCode == http.StatusForbidden { 1429 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 1430 | return false, "" 1431 | } else if response.StatusCode == http.StatusUnauthorized { 1432 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 1433 | return false, "" 1434 | } 1435 | 1436 | error = writeToFile(response.Body, hash.Hash) 1437 | if error != nil { 1438 | fmt.Println(error) 1439 | return false, "" 1440 | } 1441 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1442 | return true, hash.Hash 1443 | } 1444 | 1445 | func virusexchange(uri string, api string, hash Hash) (bool, string) { 1446 | if api == "" { 1447 | fmt.Println(" [!] !! Missing Key !!") 1448 | return false, "" 1449 | } 1450 | 1451 | if hash.HashType != sha256 { 1452 | fmt.Printf(" [!] VirusExchange only supports SHA256\n Skipping\n") 1453 | } 1454 | 1455 | request, error := http.NewRequest("GET", uri+"/samples/"+url.PathEscape(hash.Hash)+"/", nil) 1456 | if error != nil { 1457 | fmt.Println(error) 1458 | return false, "" 1459 | } 1460 | 1461 | request.Header.Set("Authorization", "Bearer "+api) 1462 | 1463 | client := &http.Client{} 1464 | response, error := client.Do(request) 1465 | if error != nil { 1466 | fmt.Println(error) 1467 | return false, "" 1468 | } 1469 | 1470 | defer response.Body.Close() 1471 | 1472 | if response.StatusCode == 404 { 1473 | return false, "" 1474 | } else if response.StatusCode == http.StatusForbidden { 1475 | fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") 1476 | return false, "" 1477 | } 1478 | 1479 | if response.StatusCode == http.StatusOK { 1480 | byteValue, _ := io.ReadAll(response.Body) 1481 | 1482 | var data = VirusExchangeData{} 1483 | error = json.Unmarshal(byteValue, &data) 1484 | 1485 | var unmarshalTypeError *json.UnmarshalTypeError 1486 | if errors.As(error, &unmarshalTypeError) { 1487 | fmt.Printf(" [!] Failed unmarshaling json. This could be due to the API changing or\n just no data inside the data array was returned\n") 1488 | fmt.Printf(" %s\n", byteValue) 1489 | 1490 | } else if error != nil { 1491 | fmt.Println(error) 1492 | return false, "" 1493 | } 1494 | 1495 | if data.Download_Link == "" { 1496 | return false, "" 1497 | } 1498 | return virusexchangeDownload(data.Download_Link, hash) 1499 | 1500 | } 1501 | // Sample not found 1502 | return false, "" 1503 | } 1504 | 1505 | func virusexchangeDownload(uri string, hash Hash) (bool, string) { 1506 | request, error := http.NewRequest("GET", uri, nil) 1507 | if error != nil { 1508 | fmt.Println(error) 1509 | return false, "" 1510 | } 1511 | 1512 | client := &http.Client{} 1513 | response, error := client.Do(request) 1514 | if error != nil { 1515 | fmt.Println(error) 1516 | return false, "" 1517 | } 1518 | 1519 | defer response.Body.Close() 1520 | 1521 | if response.StatusCode == http.StatusNotFound { 1522 | return false, "" 1523 | } else if response.StatusCode == http.StatusForbidden { 1524 | fmt.Printf(" [!] Not authorized for some reason.\n") 1525 | return false, "" 1526 | } else if response.StatusCode == http.StatusUnauthorized { 1527 | fmt.Printf(" [!] Not authorized for some reason\n") 1528 | return false, "" 1529 | } 1530 | 1531 | error = writeToFile(response.Body, hash.Hash) 1532 | if error != nil { 1533 | fmt.Println(error) 1534 | return false, "" 1535 | } 1536 | fmt.Printf(" [+] Downloaded %s\n", hash.Hash) 1537 | return true, hash.Hash 1538 | } 1539 | 1540 | func extractGzip(hash string) error { 1541 | r, err := os.Open(hash + ".gzip") 1542 | if err != nil { 1543 | log.Fatal(err) 1544 | } 1545 | defer r.Close() 1546 | 1547 | gzreader, e1 := gzip.NewReader(r) 1548 | if e1 != nil { 1549 | fmt.Println(e1) // Maybe panic here, depends on your error handling. 1550 | } 1551 | 1552 | err = writeToFile(io.NopCloser(gzreader), hash) 1553 | return err 1554 | } 1555 | 1556 | func extractPwdZip(file string, password string, renameFileAsHash bool, hash Hash) ([]*zip.File, error) { 1557 | 1558 | r, err := zip.OpenReader(file) 1559 | if err != nil { 1560 | log.Fatal(err) 1561 | } 1562 | defer r.Close() 1563 | 1564 | files := r.File 1565 | 1566 | for _, f := range r.File { 1567 | if f.IsEncrypted() { 1568 | f.SetPassword(password) 1569 | } 1570 | 1571 | r, err := f.Open() 1572 | if err != nil { 1573 | log.Fatal(err) 1574 | } 1575 | 1576 | var name string 1577 | 1578 | if !renameFileAsHash { 1579 | name = f.Name 1580 | } else { 1581 | name = hash.Hash 1582 | } 1583 | 1584 | out, error := os.Create(name) 1585 | if error != nil { 1586 | return nil, error 1587 | } 1588 | defer out.Close() 1589 | 1590 | _, err = io.Copy(out, r) 1591 | if err != nil { 1592 | return nil, err 1593 | } 1594 | } 1595 | return files, nil 1596 | } 1597 | 1598 | func findHashInObjectiveSeeList(list []ObjectiveSeeData, hash Hash) (ObjectiveSeeData, bool) { 1599 | for _, item := range list { 1600 | if item.Sha256 == hash.Hash { 1601 | return item, true 1602 | } 1603 | } 1604 | return ObjectiveSeeData{}, false 1605 | } 1606 | 1607 | func writeBytesToFile(bytes []byte, filename string) error { 1608 | // Create the file 1609 | out, err := os.Create(filename) 1610 | if err != nil { 1611 | return err 1612 | } 1613 | defer out.Close() 1614 | 1615 | _, err = out.Write(bytes) 1616 | return err 1617 | } 1618 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module xorhex.com/mlget 2 | 3 | go 1.21.3 4 | 5 | require ( 6 | github.com/kr/pretty v0.1.0 7 | github.com/spf13/pflag v1.0.5 8 | github.com/yeka/zip v0.0.0-20180914125537-d046722c6feb 9 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect 10 | golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 11 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 12 | gopkg.in/yaml.v2 v2.4.0 13 | ) 14 | -------------------------------------------------------------------------------- /hashes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "regexp" 11 | "strings" 12 | ) 13 | 14 | var alwaysDeleteInvalidFile = false 15 | 16 | type Hashes struct { 17 | Hashes []Hash 18 | } 19 | 20 | type Hash struct { 21 | Hash string 22 | HashType HashTypeOption 23 | Tags []string 24 | Comments []string 25 | Local bool // True if found locally on the filesystem (used with the -precheckdir flag). Default False 26 | LocalFile string // Full file name if file found on local system (used with the -precheckdir flag) 27 | } 28 | 29 | type HashTypeOption int64 30 | 31 | const ( 32 | NotAValidHashType HashTypeOption = iota 33 | md5 34 | sha1 35 | sha256 36 | ) 37 | 38 | func (hto HashTypeOption) String() string { 39 | switch hto { 40 | case md5: 41 | return "md5" 42 | case sha1: 43 | return "sha1" 44 | case sha256: 45 | return "sha256" 46 | } 47 | return "" 48 | } 49 | 50 | func addHash(hashes Hashes, hash Hash) (Hashes, error) { 51 | if hashes.hashExists(hash.Hash) { 52 | hsh, err := hashes.getByHash(hash.Hash) 53 | if err != nil { 54 | return hashes, err 55 | } 56 | for _, t := range hash.Tags { 57 | if !hsh.TagExists(t) { 58 | hsh.Tags = append(hsh.Tags, t) 59 | } 60 | } 61 | 62 | } else { 63 | hashes.Hashes = append(hashes.Hashes, hash) 64 | } 65 | return hashes, nil 66 | } 67 | 68 | func (hs Hashes) updateLocalFile(hash string, filename string) { 69 | for idx, h := range hs.Hashes { 70 | if h.Hash == hash { 71 | hs.Hashes[idx].Local = true 72 | hs.Hashes[idx].LocalFile = filename 73 | } 74 | } 75 | } 76 | 77 | func (hs Hashes) hashExists(hash string) bool { 78 | for _, h := range hs.Hashes { 79 | if h.Hash == hash { 80 | return true 81 | } 82 | } 83 | return false 84 | } 85 | 86 | func (hs Hashes) getByHash(hash string) (Hash, error) { 87 | for idx, h := range hs.Hashes { 88 | if h.Hash == hash { 89 | return hs.Hashes[idx], nil 90 | } 91 | } 92 | return Hash{}, fmt.Errorf("Hash not found") 93 | } 94 | 95 | func (h Hash) TagExists(tag string) bool { 96 | for _, t := range h.Tags { 97 | if t == tag { 98 | return true 99 | } 100 | } 101 | return false 102 | } 103 | 104 | func (h Hash) ValidateFile(filename string) (bool, string) { 105 | f, err := os.Open(filename) 106 | if err != nil { 107 | log.Fatal(err) 108 | } 109 | defer f.Close() 110 | 111 | var sum []byte 112 | 113 | if h.HashType == md5 { 114 | hasher := crypto.MD5.New() 115 | if _, err := io.Copy(hasher, f); err != nil { 116 | log.Fatal(err) 117 | } 118 | sum = hasher.Sum(nil) 119 | } else if h.HashType == sha1 { 120 | hasher := crypto.SHA1.New() 121 | if _, err := io.Copy(hasher, f); err != nil { 122 | log.Fatal(err) 123 | } 124 | sum = hasher.Sum(nil) 125 | } else if h.HashType == sha256 { 126 | hasher := crypto.SHA256.New() 127 | if _, err := io.Copy(hasher, f); err != nil { 128 | log.Fatal(err) 129 | } 130 | sum = hasher.Sum(nil) 131 | } 132 | if (fmt.Sprintf("%x", sum)) == strings.ToLower(h.Hash) { 133 | return true, fmt.Sprintf("%x", sum) 134 | } else { 135 | return false, fmt.Sprintf("%x", sum) 136 | } 137 | } 138 | 139 | func (h Hash) Validate(bytes []byte) (bool, string) { 140 | var sum []byte 141 | 142 | if h.HashType == md5 { 143 | hasher := crypto.MD5.New() 144 | hasher.Write(bytes) 145 | sum = hasher.Sum(nil) 146 | } else if h.HashType == sha1 { 147 | hasher := crypto.SHA1.New() 148 | hasher.Write(bytes) 149 | sum = hasher.Sum(nil) 150 | } else if h.HashType == sha256 { 151 | hasher := crypto.SHA256.New() 152 | hasher.Write(bytes) 153 | sum = hasher.Sum(nil) 154 | } 155 | if (fmt.Sprintf("%x", sum)) == strings.ToLower(h.Hash) { 156 | return true, fmt.Sprintf("%x", sum) 157 | } else { 158 | return false, fmt.Sprintf("%x", sum) 159 | } 160 | } 161 | 162 | func deleteInvalidFile(filename string) { 163 | if !alwaysDeleteInvalidFile { 164 | var delete_file string 165 | fmt.Printf(" [?] Delete invalid file? [A/Y/n] Always delete/Yes, this time/No, not this time\n") 166 | fmt.Scanln(&delete_file) 167 | if strings.ToUpper(delete_file) == "Y" || delete_file == "" || strings.ToUpper(delete_file) == "A" { 168 | os.Remove(filename) 169 | fmt.Printf(" [!] Deleted invalid file\n") 170 | if strings.ToUpper(delete_file) == "A" { 171 | alwaysDeleteInvalidFile = true 172 | } 173 | } else { 174 | fmt.Printf(" [!] Keeping invalid file\n") 175 | } 176 | } else { 177 | os.Remove(filename) 178 | fmt.Printf(" [!] Deleted invalid file\n") 179 | } 180 | } 181 | 182 | func hashType(hash string) (HashTypeOption, error) { 183 | match, _ := regexp.MatchString("^[A-Fa-f0-9]{64}$", hash) 184 | if match { 185 | return sha256, nil 186 | } 187 | match, _ = regexp.MatchString("^[A-Fa-f0-9]{40}$", hash) 188 | if match { 189 | return sha1, nil 190 | } 191 | match, _ = regexp.MatchString("^[A-Fa-f0-9]{32}$", hash) 192 | if match { 193 | return md5, nil 194 | } 195 | return NotAValidHashType, errors.New("not a valid hash") 196 | } 197 | 198 | func extractHashes(text string) ([]string, error) { 199 | hashes := make([]string, 0) 200 | 201 | re := regexp.MustCompile(`>\s*[A-Fa-f0-9]{64}\s*<`) 202 | matches := re.FindAllStringSubmatch(text, 100) 203 | for m := range matches { 204 | hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) 205 | } 206 | re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{40}\s*<`) 207 | matches = re.FindAllStringSubmatch(text, 100) 208 | for m := range matches { 209 | hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) 210 | } 211 | re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{32}\s*<`) 212 | matches = re.FindAllStringSubmatch(text, 100) 213 | for m := range matches { 214 | hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) 215 | } 216 | 217 | if len(hashes) > 0 { 218 | return hashes, fmt.Errorf("no hashes found") 219 | } 220 | 221 | return hashes, nil 222 | } 223 | -------------------------------------------------------------------------------- /history.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | func writeToFile(file io.ReadCloser, filename string) error { 15 | // Create the file 16 | out, err := os.Create(filename) 17 | if err != nil { 18 | return err 19 | } 20 | defer out.Close() 21 | 22 | // Write the body to file 23 | _, err = io.Copy(out, file) 24 | return err 25 | } 26 | 27 | // Code for this function came from - https://golangcode.com/how-to-check-if-a-string-is-a-url/ 28 | // isValidUrl tests a string to determine if it is a well-structured url or not. 29 | func isValidUrl(toTest string) bool { 30 | _, err := url.ParseRequestURI(toTest) 31 | if err != nil { 32 | return false 33 | } 34 | 35 | u, err := url.Parse(toTest) 36 | if err != nil || u.Scheme == "" || u.Host == "" { 37 | return false 38 | } 39 | 40 | return true 41 | } 42 | 43 | // https://golangcode.com/check-if-a-file-exists/ 44 | func fileExists(filename string) bool { 45 | info, err := os.Stat(filename) 46 | if os.IsNotExist(err) { 47 | return false 48 | } 49 | return !info.IsDir() 50 | } 51 | 52 | func downloadFromUrl(url string) (string, error) { 53 | 54 | filename := "mlget.download.data.tmp" 55 | 56 | r, err := http.Get(url) 57 | if err != nil { 58 | log.Println("Cannot get from URL", err) 59 | return "", err 60 | } 61 | 62 | defer r.Body.Close() 63 | 64 | if !fileExists(filename) { 65 | 66 | file, _ := os.Create(filename) 67 | defer file.Close() 68 | 69 | writer := bufio.NewWriter(file) 70 | io.Copy(writer, r.Body) 71 | writer.Flush() 72 | 73 | return filename, nil 74 | 75 | } else { 76 | return "", fmt.Errorf("file %s already exists - delete and try again", filename) 77 | } 78 | } 79 | 80 | func parseFileForHashEntries(filename string) ([]Hash, error) { 81 | hashes := []Hash{} 82 | var _filename string 83 | var err error 84 | 85 | fmt.Printf("Hashes Found in File:\n") 86 | 87 | if isValidUrl(filename) { 88 | _filename, err = downloadFromUrl(filename) 89 | if err != nil { 90 | return nil, err 91 | } 92 | } else { 93 | _filename = filename 94 | } 95 | 96 | file, err := os.Open(_filename) 97 | if err != nil { 98 | fmt.Println("Error reading file") 99 | fmt.Println(err) 100 | } 101 | 102 | defer func() ([]string, error) { 103 | if err = file.Close(); err != nil { 104 | fmt.Println(err) 105 | return nil, err 106 | } 107 | return nil, nil 108 | }() 109 | 110 | f := func(c rune) bool { 111 | return c == '|' 112 | } 113 | 114 | scanner := bufio.NewScanner(file) 115 | for scanner.Scan() { // internally, it advances token based on separator 116 | text := scanner.Text() 117 | if len(strings.TrimSpace(text)) > 0 { 118 | hash := strings.FieldsFunc(strings.TrimSpace(text), f)[0] 119 | tags := []string{} 120 | comments := []string{} 121 | if len(strings.FieldsFunc(text, f)) > 1 { 122 | fields := strings.FieldsFunc(text, f)[1:len(strings.FieldsFunc(text, f))] 123 | tagSection := false 124 | commentSection := false 125 | for _, f := range fields { 126 | if f == "TAGS" { 127 | tagSection = true 128 | commentSection = false 129 | } else if f == "COMMENTS" { 130 | tagSection = false 131 | commentSection = true 132 | } else if f != "TAGS" && f != "COMMENTS" && tagSection { 133 | tags = append(tags, f) 134 | } else if f != "TAGS" && f != "COMMENTS" && commentSection { 135 | comments = append(comments, f) 136 | } 137 | } 138 | } 139 | pHash := Hash{} 140 | pHash, err = parseFileHashEntry(hash, tags, comments) 141 | if err == nil { 142 | hashes = append(hashes, pHash) 143 | } else { 144 | // Try splitting on \t and check to see if any of the values match a hash 145 | // This is useful for reading files from the web that list sample hashes 146 | // This still assumes there is only one hash per line as it stops after the 147 | // first hash is found on that line 148 | s := func(c rune) bool { 149 | return c == '\t' 150 | } 151 | 152 | line := strings.FieldsFunc(strings.TrimSpace(text), s) 153 | if len(line) > 0 { 154 | for _, element := range line { 155 | lHash := Hash{} 156 | lHash, err := parseFileHashEntry(strings.TrimSpace(element), tags, comments) 157 | if err == nil { 158 | hashes = append(hashes, lHash) 159 | break 160 | } else { 161 | 162 | matches, err := extractHashes(strings.TrimSpace(element)) 163 | if err != nil { 164 | fmt.Println(err) 165 | } 166 | for m := range matches { 167 | tags := []string{} 168 | comments := []string{} 169 | lHash, err = parseFileHashEntry(matches[m], tags, comments) 170 | if err == nil { 171 | hashes = append(hashes, lHash) 172 | } 173 | } 174 | } 175 | } 176 | } 177 | } 178 | } 179 | } 180 | 181 | if _filename == "mlget.download.data.tmp" { 182 | os.Remove(_filename) 183 | } 184 | 185 | fmt.Println("") 186 | 187 | return hashes, nil 188 | } 189 | 190 | func writeUnfoundHashesToFile(filename string, hashes Hashes) error { 191 | f, err := os.Create(filename) 192 | if err != nil { 193 | return err 194 | } 195 | defer f.Close() 196 | 197 | w := bufio.NewWriter(f) 198 | defer w.Flush() 199 | 200 | for _, h := range hashes.Hashes { 201 | w.WriteString(h.Hash + "|TAGS|" + strings.Join(h.Tags, "|") + "|COMMENTS|" + strings.Join(h.Comments, "|") + "\n") 202 | } 203 | return nil 204 | } 205 | 206 | func parseFileHashEntry(hash string, tags []string, comments []string) (Hash, error) { 207 | ht, err := hashType(hash) 208 | if err != nil { 209 | fmt.Printf("\n Skipping %s because it's %s\n", hash, err) 210 | return Hash{}, err 211 | } 212 | fmt.Printf(" - %s\n", hash) // token in unicode-char 213 | hashS := Hash{Hash: hash, HashType: ht} 214 | if len(tags) > 0 { 215 | hashS.Tags = tags 216 | } 217 | if len(comments) > 0 { 218 | hashS.Comments = comments 219 | } 220 | return hashS, nil 221 | } 222 | -------------------------------------------------------------------------------- /mlget-test-config/samples.yaml: -------------------------------------------------------------------------------- 1 | test 1: 2 | name: TestJoeSandbox 3 | hash: 40541a03e910b21df681bec69cfe59678ebba86c 4 | test 2: 5 | name: TestObjectiveSee 6 | hash: 458a9ac086116fa011c1a7bd49ac15f386cd95e39eb6b7cd5c5125aef516c78c 7 | test 3: 8 | name: TestCapeSandbox 9 | hash: 28eefc36104bebb595fb38cae21a7d0a 10 | test 4: 11 | name: TestInquestLabsLookUp 12 | hash: b3f868fa1af24f270e3ecc0ecb79325e 13 | test 5: 14 | name: TestInquestLabsNoLookUp 15 | hash: 6b425804d43bb369211bbec59808807730a908804ca9b8c09081139179bbc868 16 | test 6: 17 | name: TestVirusTotal 18 | hash: 21cc9c0ae5f97b66d69f1ff99a4fed264551edfe0a5ce8d5449942bf8f0aefb2 19 | test 7: 20 | name: TestMWDB 21 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 22 | test 8: 23 | name: TestPolyswarm 24 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 25 | test 9: 26 | name: TestHybridAnalysis 27 | hash: ed2f501408a7a6e1a854c29c4b0bc5648a6aa8612432df829008931b3e34bf56 28 | test 10: 29 | name: TestTriage 30 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 31 | test 11: 32 | name: TestMalShare 33 | hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 34 | test 12: 35 | name: TestMalwareBazaar 36 | hash: 001bffcdd170c8328601006ad54a221d1073ba04fbdca556749cf1b041cfad97 37 | test 13: 38 | name: TestMalpedia 39 | hash: 78668c237097651d64c97b25fc86c74096bfe1ed53e1004445f118ea5feaa3ad 40 | test 14: 41 | name: TestUnpacme 42 | hash: 0219a79a2f47da42601568ee4a41392aa429f62a1fb01080cb68540074449c92 43 | test 15: 44 | name: TestVxShare 45 | hash: 1c11c963a417674e1414bac05fdbfa5cfa09f92c7b0d9882aeb55ce2a058d668 46 | test 16: 47 | name: TestFileScanIo 48 | hash: 2799af2efd698da215afc9c88da3b1e84b00137433d9444a5c11d69092b3f80d 49 | test 17: 50 | name: TestURLScanIo 51 | hash: 5b027ada26a610e97ab4ef9efb1118b377061712acec6db994d6aa1c78a332a8 52 | test 19: 53 | name: TestAssemblyLine 54 | hash: b78e786091f017510b44137961f3074fe7d5f950 55 | test 20: 56 | name: TestTriageV2 57 | hash: 5eaaf8ac2d358c2d7065884b7994638fee3987f02474e54467f14b010a18d028 58 | test 21: 59 | name: TestVirusExchange 60 | hash: ad5156e1e0285de9348d9b2c4649d9f6f39bac89567f833b1a8ba3d26468fc84 -------------------------------------------------------------------------------- /mlget.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "path" 10 | "sync" 11 | "time" 12 | 13 | flag "github.com/spf13/pflag" 14 | "golang.org/x/exp/slices" 15 | ) 16 | 17 | var apiFlag string 18 | var helpFlag bool 19 | var checkConfFlag bool 20 | var AddConfigEntryFlag bool 21 | var doNotExtractFlag bool 22 | var inputFileFlag string 23 | var outputFileFlag bool 24 | var uploadToMWDBAndDeleteFlag bool 25 | var downloadOnlyFlag bool 26 | var uploadToMWDBFlag bool 27 | var readFromFileAndUpdateWithNotFoundHashesFlag string 28 | var tagsFlag []string 29 | var commentsFlag []string 30 | var versionFlag bool 31 | var noValidationFlag bool 32 | var precheckdir bool 33 | var uploadToAssemblyLineFlag bool 34 | var uploadToAssemblyLineAndDeleteFlag bool 35 | var forceResubmission bool 36 | 37 | var version string = "3.4.1" 38 | 39 | func usage() { 40 | fmt.Println("mlget - A command line tool to download malware from a variety of sources") 41 | fmt.Println("") 42 | 43 | fmt.Printf("Usage: %s [OPTIONS] hash_arguments...\n", os.Args[0]) 44 | flag.PrintDefaults() 45 | 46 | fmt.Println("") 47 | fmt.Println("Example Usage: mlget ") 48 | fmt.Println("Example Usage: mlget --from mb ") 49 | fmt.Println("Example Usage: mlget --tag tag_one --tag tag_two --uploaddelete ") 50 | } 51 | 52 | func init() { 53 | flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - al (AssemblyLine)\n - cs (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tr (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n -ve (VExchange)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") 54 | flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)") 55 | flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (requires using --read)") 56 | flag.BoolVar(&helpFlag, "help", false, "Print the help message") 57 | flag.BoolVar(&checkConfFlag, "config", false, "Parse and print the config file") 58 | flag.BoolVar(&AddConfigEntryFlag, "addtoconfig", false, "Add entry to the config file") 59 | flag.BoolVar(&doNotExtractFlag, "noextraction", false, "Do not extract malware from archive file.\nCurrently this only effects MalwareBazaar and HybridAnalysis") 60 | flag.BoolVar(&uploadToMWDBFlag, "uploadmwdb", false, "Upload downloaded files to the Upload MWDB instances specified in the mlget.yml file.") 61 | flag.BoolVar(&uploadToAssemblyLineFlag, "uploadal", false, "Upload downloaded files to the Upload AssemblyLine instances specified in the mlget.yml file.") 62 | flag.StringVar(&readFromFileAndUpdateWithNotFoundHashesFlag, "readupdate", "", "Read hashes from file to download. Replace entries in the file with just the hashes that were not found (for next time).") 63 | flag.BoolVar(&uploadToMWDBAndDeleteFlag, "uploaddeletemwdb", false, "Upload downloaded files to the Upload MWDB instance specified in the mlget.yml file.\nDelete the files after successful upload") 64 | flag.BoolVar(&uploadToAssemblyLineAndDeleteFlag, "uploaddeleteal", false, "Upload downloaded files to the Upload AssemblyLine instances specified in the mlget.yml file.\nDelete the files after successful upload") 65 | flag.BoolVar(&forceResubmission, "f", false, "Force resubmission to AssemblyLine when the files already exists on the AssemblyLine instance.") 66 | flag.StringSliceVar(&tagsFlag, "tag", []string{}, "Tag the sample when uploading to your own instance of MWDB.") 67 | flag.StringSliceVar(&commentsFlag, "comment", []string{}, "Add comment to the sample when uploading to your own instance of MWDB.") 68 | flag.BoolVar(&downloadOnlyFlag, "downloadonly", false, "Download from any source, including your personal instance of MWDB.\nWhen this flag is set; it will NOT update any output file with the hashes not found.\nAnd it will not upload to any of the UploadMWDB.") 69 | flag.BoolVar(&versionFlag, "version", false, "Print the version number") 70 | flag.BoolVar(&noValidationFlag, "novalidation", false, "Turn off post download hash check verification") 71 | flag.BoolVar(&precheckdir, "precheckdir", false, "Search current dir for files matching the hashes provided, if found don't redownload") 72 | } 73 | 74 | func main() { 75 | 76 | doNotValidatehash := []MalwareRepoType{ObjectiveSee} 77 | 78 | homeDir, err := os.UserHomeDir() 79 | if err != nil { 80 | fmt.Println(err) 81 | return 82 | } 83 | 84 | configFileName := path.Join(homeDir, ".mlget.yml") 85 | 86 | cfg, err := LoadConfig(configFileName) 87 | if err != nil { 88 | fmt.Println(err) 89 | return 90 | } 91 | 92 | flag.Parse() 93 | 94 | if versionFlag { 95 | fmt.Printf("Mlget version: %s\n", version) 96 | return 97 | } 98 | 99 | if helpFlag { 100 | usage() 101 | return 102 | } 103 | 104 | if AddConfigEntryFlag { 105 | AddToConfig(configFileName) 106 | return 107 | } 108 | 109 | if checkConfFlag { 110 | fmt.Printf("%+v", cfg) 111 | return 112 | } 113 | 114 | args := flag.Args() 115 | /* 116 | if webserver { 117 | runWebServer(ip, port) 118 | } else {*/ 119 | downloadMalwareFromCLI(args, cfg, doNotValidatehash, precheckdir) 120 | // } 121 | 122 | } 123 | 124 | func parseArgHashes(hashes []string, tags []string, comments []string) Hashes { 125 | parsedHashes := Hashes{} 126 | fmt.Printf("Hashes Passed Via the Command Line:\n") 127 | for _, h := range hashes { 128 | ht, err := hashType(h) 129 | if err != nil { 130 | fmt.Printf("\n Skipping %s because it's %s\n", h, err) 131 | continue 132 | } 133 | fmt.Printf(" - %s\n", h) // token in unicode-char 134 | hash := Hash{Hash: h, HashType: ht, Local: false} 135 | if len(tags) > 0 { 136 | hash.Tags = tags 137 | } 138 | if len(comments) > 0 { 139 | hash.Comments = comments 140 | } 141 | parsedHashes, _ = addHash(parsedHashes, hash) 142 | } 143 | fmt.Println("") 144 | return parsedHashes 145 | } 146 | 147 | func downloadMalwareFromCLI(args []string, cfg []RepositoryConfigEntry, doNotValidatehash []MalwareRepoType, precheckdir bool) { 148 | if apiFlag != "" { 149 | flaggedRepo := getMalwareRepoByFlagName(apiFlag) 150 | if flaggedRepo == NotSupported { 151 | fmt.Printf("Invalid or unsupported malware repo type: %s\nCheck the help for the values to pass to the --from parameter\n", apiFlag) 152 | return 153 | } 154 | } 155 | 156 | if apiFlag != "" && downloadOnlyFlag { 157 | fmt.Printf(("Can't use both the --from flag and the --downloadonly flag together")) 158 | return 159 | } 160 | 161 | hashes := parseArgHashes(args, tagsFlag, commentsFlag) 162 | var err error 163 | 164 | if inputFileFlag != "" { 165 | hshs, err := parseFileForHashEntries(inputFileFlag) 166 | if err != nil { 167 | fmt.Printf("Error reading from %s\n", inputFileFlag) 168 | fmt.Println(err) 169 | } else { 170 | for idx, hsh := range hshs { 171 | if len(hshs) > 100 { 172 | fmt.Printf("Adding Hash %d of %d\n", idx, len(hshs)) 173 | } 174 | hashes, _ = addHash(hashes, hsh) 175 | } 176 | } 177 | } 178 | if readFromFileAndUpdateWithNotFoundHashesFlag != "" { 179 | hshs, err := parseFileForHashEntries(readFromFileAndUpdateWithNotFoundHashesFlag) 180 | if err != nil { 181 | fmt.Printf("Error reading from %s\n", readFromFileAndUpdateWithNotFoundHashesFlag) 182 | fmt.Println(err) 183 | } else { 184 | for idx, hsh := range hshs { 185 | if len(hshs) > 100 { 186 | fmt.Printf("Adding Hash %d of %d\n", idx, len(hshs)) 187 | } 188 | hashes, _ = addHash(hashes, hsh) 189 | } 190 | } 191 | } 192 | 193 | var notFoundHashes Hashes 194 | 195 | if len(hashes.Hashes) == 0 { 196 | fmt.Println("No hashes found; displaying Help") 197 | usage() 198 | return 199 | } 200 | 201 | var osq ObjectiveSeeQuery 202 | osConfigs := getConfigsByType(ObjectiveSee, cfg) 203 | // Can have multiple Objective-See configs but only the first one to load will be used 204 | for _, osc := range osConfigs { 205 | osq, err = loadObjectiveSeeJson(osc.Host) 206 | if err != nil { 207 | fmt.Println("Unable to load Objective-See json data. Skipping...") 208 | continue 209 | } 210 | fmt.Println("") 211 | break 212 | } 213 | 214 | if doNotExtractFlag { 215 | doNotValidatehash = append(doNotValidatehash, VxShare, MalwareBazaar, HybridAnalysis, FileScanIo) 216 | } 217 | 218 | if precheckdir { 219 | files, err := os.ReadDir(".") 220 | if err != nil { 221 | fmt.Println(err) 222 | } 223 | 224 | filesHashing := 0 225 | ch := make(chan Hash, len(files)*3) 226 | var wg sync.WaitGroup 227 | 228 | go func() { 229 | wg.Wait() 230 | close(ch) 231 | }() 232 | 233 | for _, file := range files { 234 | if file.IsDir() { 235 | continue 236 | } 237 | filesHashing++ 238 | wg.Add(1) 239 | 240 | name := file.Name() 241 | 242 | go func() { 243 | defer wg.Done() 244 | hashFileAndCheck(name, ch) 245 | }() 246 | } 247 | 248 | fmt.Println("\nFiles Matching Hashes Found Locally:") 249 | 250 | for i := range ch { 251 | if hashes.hashExists(i.Hash) { 252 | fmt.Printf(" - Found %s in current folder\n", i.Hash) 253 | fmt.Printf(" File Name: %s\n", i.LocalFile) 254 | hashes.updateLocalFile(i.Hash, i.LocalFile) 255 | 256 | h, _ := hashes.getByHash(i.Hash) 257 | 258 | if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { 259 | err := UploadSampleToMWDBs(cfg, h.LocalFile, h, uploadToMWDBAndDeleteFlag) 260 | if err != nil { 261 | fmt.Printf(" ! %s\n", err) 262 | } 263 | } 264 | if (uploadToAssemblyLineFlag || uploadToAssemblyLineAndDeleteFlag) && !downloadOnlyFlag { 265 | err := UploadSampleToAssemblyLine(cfg, h.LocalFile, h, uploadToAssemblyLineAndDeleteFlag, forceResubmission) 266 | if err != nil { 267 | fmt.Printf(" ! %s\n", err) 268 | } 269 | } 270 | } 271 | } 272 | } 273 | 274 | for idx, h := range hashes.Hashes { 275 | 276 | if h.Local { 277 | continue 278 | } 279 | 280 | fmt.Printf("\nLook up %s (%s) - (%d of %d)\n", h.Hash, h.HashType, idx+1, len(hashes.Hashes)) 281 | 282 | if apiFlag != "" { 283 | flaggedRepo := getMalwareRepoByFlagName(apiFlag) 284 | 285 | fmt.Printf("Looking on %s\n", getMalwareRepoByFlagName(apiFlag)) 286 | 287 | found, filename, checkedRepo := flaggedRepo.QueryAndDownload(cfg, h, doNotExtractFlag, osq) 288 | if !found { 289 | fmt.Println(" [!] Not Found") 290 | notFoundHashes, _ = addHash(notFoundHashes, h) 291 | } else if found { 292 | if !noValidationFlag { 293 | if slices.Contains(doNotValidatehash, checkedRepo) { 294 | if checkedRepo == ObjectiveSee { 295 | fmt.Printf(" [!] Not able to validate hash for repo %s\n", checkedRepo.String()) 296 | } else { 297 | fmt.Printf(" [!] Not able to validate hash for repo %s when noextraction flag is set to %t\n", checkedRepo.String(), doNotExtractFlag) 298 | } 299 | } else { 300 | valid, calculatedHash := h.ValidateFile(filename) 301 | if !valid { 302 | fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\n", calculatedHash, h.Hash) 303 | deleteInvalidFile(filename) 304 | notFoundHashes, _ = addHash(notFoundHashes, h) 305 | continue 306 | } else { 307 | fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", h.Hash) 308 | } 309 | } 310 | } 311 | if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { 312 | err := UploadSampleToMWDBs(cfg, filename, h, uploadToMWDBAndDeleteFlag) 313 | if err != nil { 314 | fmt.Printf(" ! %s", err) 315 | } 316 | } 317 | if (uploadToAssemblyLineFlag || uploadToAssemblyLineAndDeleteFlag) && !downloadOnlyFlag { 318 | err := UploadSampleToAssemblyLine(cfg, filename, h, uploadToAssemblyLineAndDeleteFlag, forceResubmission) 319 | if err != nil { 320 | fmt.Printf(" ! %s", err) 321 | } 322 | } 323 | } 324 | 325 | } else { 326 | fmt.Println("Querying all services") 327 | 328 | found, filename, _ := queryAndDownloadAll(cfg, h, doNotExtractFlag, !downloadOnlyFlag, osq, noValidationFlag, doNotValidatehash) 329 | if found { 330 | if (uploadToMWDBFlag || uploadToMWDBAndDeleteFlag) && !downloadOnlyFlag { 331 | err := UploadSampleToMWDBs(cfg, filename, h, uploadToMWDBAndDeleteFlag) 332 | if err != nil { 333 | fmt.Printf(" ! %s", err) 334 | } 335 | } 336 | if (uploadToAssemblyLineFlag || uploadToAssemblyLineAndDeleteFlag) && !downloadOnlyFlag { 337 | err := UploadSampleToAssemblyLine(cfg, filename, h, uploadToAssemblyLineAndDeleteFlag, forceResubmission) 338 | if err != nil { 339 | fmt.Printf(" ! %s", err) 340 | } 341 | } 342 | continue 343 | } 344 | 345 | notFoundHashes, _ = addHash(notFoundHashes, h) 346 | } 347 | } 348 | 349 | if len(notFoundHashes.Hashes) > 0 { 350 | fmt.Printf("\nHashes not found!\n") 351 | for i, s := range notFoundHashes.Hashes { 352 | fmt.Printf(" %d: %s\n", i, s.Hash) 353 | } 354 | } 355 | if !downloadOnlyFlag { 356 | if readFromFileAndUpdateWithNotFoundHashesFlag != "" && !isValidUrl(readFromFileAndUpdateWithNotFoundHashesFlag) { 357 | err := writeUnfoundHashesToFile(readFromFileAndUpdateWithNotFoundHashesFlag, notFoundHashes) 358 | if err != nil { 359 | fmt.Println("Error writing not found hashes to file") 360 | fmt.Println(err) 361 | } 362 | fmt.Printf("\n\n%s refreshed to show only the hashes not found.\n", readFromFileAndUpdateWithNotFoundHashesFlag) 363 | 364 | } else if outputFileFlag && len(notFoundHashes.Hashes) > 0 { 365 | var filename string 366 | if inputFileFlag != "" { 367 | filename = time.Now().Format("2006-01-02__3_4_5__pm__") + inputFileFlag 368 | } else { 369 | filename = time.Now().Format("2006-01-02__3_4_5__pm") + "_not_found_hashes.txt" 370 | } 371 | err := writeUnfoundHashesToFile(filename, notFoundHashes) 372 | if err != nil { 373 | fmt.Println("Error writing unfound hashes to file") 374 | fmt.Println(err) 375 | } 376 | fmt.Printf("\n\nUnfound hashes written to %s\n", filename) 377 | } else if isValidUrl(readFromFileAndUpdateWithNotFoundHashesFlag) { 378 | fmt.Println("File specified is a URL - can't update file. Please use --read and --output flags instead if you want to capture the hashes not found.") 379 | } 380 | } 381 | } 382 | 383 | func hashFileAndCheck(file string, c chan Hash) { 384 | 385 | f, err := os.Open(file) 386 | 387 | if err != nil { 388 | log.Fatal(err) 389 | } 390 | defer f.Close() 391 | 392 | hasherMD5 := crypto.MD5.New() 393 | if _, err := io.Copy(hasherMD5, f); err != nil { 394 | log.Fatal(err) 395 | } 396 | sumMD5 := hasherMD5.Sum(nil) 397 | 398 | c <- Hash{Hash: fmt.Sprintf("%x", sumMD5), HashType: md5, LocalFile: file} 399 | 400 | f.Seek(0, 0) 401 | hasherSHA1 := crypto.SHA1.New() 402 | if _, err := io.Copy(hasherSHA1, f); err != nil { 403 | log.Fatal(err) 404 | } 405 | sumSHA1 := hasherSHA1.Sum(nil) 406 | 407 | c <- Hash{Hash: fmt.Sprintf("%x", sumSHA1), HashType: sha1, LocalFile: file} 408 | 409 | f.Seek(0, 0) 410 | hasherSHA256 := crypto.SHA256.New() 411 | if _, err := io.Copy(hasherSHA256, f); err != nil { 412 | log.Fatal(err) 413 | } 414 | sumSHA256 := hasherSHA256.Sum(nil) 415 | 416 | c <- Hash{Hash: fmt.Sprintf("%x", sumSHA256), HashType: sha256, LocalFile: file} 417 | } 418 | -------------------------------------------------------------------------------- /mlget_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path" 9 | "testing" 10 | 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | type TestConfigEntry struct { 15 | Name string `yaml:"name"` 16 | Hash string `yaml:"hash"` 17 | } 18 | 19 | func parseTestConfig(path string, testName string) (TestConfigEntry, error) { 20 | var tce TestConfigEntry 21 | 22 | _, err := os.Stat(path) 23 | if os.IsNotExist(err) { 24 | return tce, err 25 | } 26 | 27 | f, err := os.ReadFile(path) 28 | if err != nil { 29 | fmt.Printf("%v", err) 30 | return tce, err 31 | } 32 | 33 | data := make(map[string]TestConfigEntry) 34 | 35 | err = yaml.Unmarshal(f, &data) 36 | if err != nil { 37 | fmt.Printf("%v", err) 38 | return tce, err 39 | } 40 | 41 | var filteredTestConfig []TestConfigEntry 42 | for _, v := range data { 43 | if v.Name == testName { 44 | filteredTestConfig = append(filteredTestConfig, v) 45 | } 46 | } 47 | 48 | if len(filteredTestConfig) != 1 { 49 | return tce, errors.New("No test config entry found") 50 | } 51 | return filteredTestConfig[0], nil 52 | } 53 | 54 | func TestJoeSandbox(t *testing.T) { 55 | home, _ := os.UserHomeDir() 56 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 57 | if err != nil { 58 | log.Fatal() 59 | t.Errorf("%v", err) 60 | } 61 | 62 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 63 | if err != nil { 64 | log.Fatal() 65 | t.Errorf("%v", err) 66 | } 67 | 68 | ht, _ := hashType(scfg.Hash) 69 | hash := Hash{HashType: ht, Hash: scfg.Hash} 70 | 71 | var osq ObjectiveSeeQuery 72 | result, filename, _ := JoeSandbox.QueryAndDownload(cfg, hash, false, osq) 73 | 74 | if !result { 75 | t.Errorf("JoeSandbox failed") 76 | } else { 77 | valid, errmsg := hash.ValidateFile(filename) 78 | 79 | if !valid { 80 | os.Remove(hash.Hash) 81 | t.Errorf(errmsg) 82 | } else { 83 | os.Remove(hash.Hash) 84 | } 85 | } 86 | } 87 | 88 | func TestObjectiveSee(t *testing.T) { 89 | home, _ := os.UserHomeDir() 90 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 91 | if err != nil { 92 | log.Fatal() 93 | t.Errorf("%v", err) 94 | } 95 | 96 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 97 | if err != nil { 98 | log.Fatal() 99 | t.Errorf("%v", err) 100 | } 101 | 102 | ht, _ := hashType(scfg.Hash) 103 | hash := Hash{HashType: ht, Hash: scfg.Hash} 104 | 105 | osq, _ := loadObjectiveSeeJson(getConfigsByType(ObjectiveSee, cfg)[0].Host) 106 | result, _, _ := ObjectiveSee.QueryAndDownload(cfg, hash, true, osq) 107 | 108 | if !result { 109 | t.Errorf("Objective-See failed") 110 | } else { 111 | os.Remove(hash.Hash) 112 | } 113 | } 114 | 115 | func TestCapeSandbox(t *testing.T) { 116 | home, _ := os.UserHomeDir() 117 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 118 | if err != nil { 119 | log.Fatal() 120 | t.Errorf("%v", err) 121 | } 122 | 123 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 124 | if err != nil { 125 | log.Fatal() 126 | t.Errorf("%v", err) 127 | } 128 | 129 | ht, _ := hashType(scfg.Hash) 130 | hash := Hash{HashType: ht, Hash: scfg.Hash} 131 | 132 | var osq ObjectiveSeeQuery 133 | result, filename, _ := CapeSandbox.QueryAndDownload(cfg, hash, false, osq) 134 | 135 | if !result { 136 | t.Errorf("CapeSandbox failed") 137 | } else { 138 | valid, errmsg := hash.ValidateFile(filename) 139 | 140 | if !valid { 141 | os.Remove(hash.Hash) 142 | t.Errorf(errmsg) 143 | } else { 144 | os.Remove(hash.Hash) 145 | } 146 | } 147 | } 148 | 149 | func TestInquestLabsLookUp(t *testing.T) { 150 | home, _ := os.UserHomeDir() 151 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 152 | if err != nil { 153 | log.Fatal() 154 | t.Errorf("%v", err) 155 | } 156 | 157 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 158 | if err != nil { 159 | log.Fatal() 160 | t.Errorf("%v", err) 161 | } 162 | 163 | ht, _ := hashType(scfg.Hash) 164 | hash := Hash{HashType: ht, Hash: scfg.Hash} 165 | 166 | var osq ObjectiveSeeQuery 167 | result, filename, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) 168 | 169 | if !result { 170 | t.Errorf("InquestLabs failed") 171 | } else { 172 | valid, errmsg := hash.ValidateFile(filename) 173 | 174 | if !valid { 175 | os.Remove(hash.Hash) 176 | t.Errorf(errmsg) 177 | } else { 178 | os.Remove(hash.Hash) 179 | } 180 | } 181 | } 182 | 183 | func TestInquestLabsNoLookUp(t *testing.T) { 184 | home, _ := os.UserHomeDir() 185 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 186 | if err != nil { 187 | log.Fatal() 188 | t.Errorf("%v", err) 189 | } 190 | 191 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 192 | if err != nil { 193 | log.Fatal() 194 | t.Errorf("%v", err) 195 | } 196 | 197 | ht, _ := hashType(scfg.Hash) 198 | hash := Hash{HashType: ht, Hash: scfg.Hash} 199 | 200 | var osq ObjectiveSeeQuery 201 | result, filename, _ := InQuest.QueryAndDownload(cfg, hash, false, osq) 202 | 203 | if !result { 204 | t.Errorf("InquestLabs failed") 205 | } else { 206 | valid, errmsg := hash.ValidateFile(filename) 207 | 208 | if !valid { 209 | os.Remove(hash.Hash) 210 | t.Errorf(errmsg) 211 | } else { 212 | os.Remove(hash.Hash) 213 | } 214 | } 215 | } 216 | 217 | func TestVirusTotal(t *testing.T) { 218 | home, _ := os.UserHomeDir() 219 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 220 | if err != nil { 221 | log.Fatal() 222 | t.Errorf("%v", err) 223 | } 224 | 225 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 226 | if err != nil { 227 | log.Fatal() 228 | t.Errorf("%v", err) 229 | } 230 | 231 | ht, _ := hashType(scfg.Hash) 232 | hash := Hash{HashType: ht, Hash: scfg.Hash} 233 | 234 | var osq ObjectiveSeeQuery 235 | result, filename, _ := VirusTotal.QueryAndDownload(cfg, hash, false, osq) 236 | 237 | if !result { 238 | t.Errorf("VirusTotal failed") 239 | } else { 240 | valid, errmsg := hash.ValidateFile(filename) 241 | 242 | if !valid { 243 | os.Remove(hash.Hash) 244 | t.Errorf(errmsg) 245 | } else { 246 | os.Remove(hash.Hash) 247 | } 248 | } 249 | } 250 | 251 | func TestMWDB(t *testing.T) { 252 | home, _ := os.UserHomeDir() 253 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 254 | if err != nil { 255 | log.Fatal() 256 | t.Errorf("%v", err) 257 | } 258 | 259 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 260 | if err != nil { 261 | log.Fatal() 262 | t.Errorf("%v", err) 263 | } 264 | 265 | ht, _ := hashType(scfg.Hash) 266 | hash := Hash{HashType: ht, Hash: scfg.Hash} 267 | 268 | var osq ObjectiveSeeQuery 269 | result, filename, _ := MWDB.QueryAndDownload(cfg, hash, false, osq) 270 | 271 | if !result { 272 | t.Errorf("MWDB failed") 273 | } else { 274 | valid, errmsg := hash.ValidateFile(filename) 275 | 276 | if !valid { 277 | os.Remove(hash.Hash) 278 | t.Errorf(errmsg) 279 | } else { 280 | os.Remove(hash.Hash) 281 | } 282 | } 283 | } 284 | 285 | func TestPolyswarm(t *testing.T) { 286 | home, _ := os.UserHomeDir() 287 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 288 | if err != nil { 289 | log.Fatal() 290 | t.Errorf("%v", err) 291 | } 292 | 293 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 294 | if err != nil { 295 | log.Fatal() 296 | t.Errorf("%v", err) 297 | } 298 | 299 | ht, _ := hashType(scfg.Hash) 300 | hash := Hash{HashType: ht, Hash: scfg.Hash} 301 | 302 | var osq ObjectiveSeeQuery 303 | result, filename, _ := Polyswarm.QueryAndDownload(cfg, hash, false, osq) 304 | 305 | if !result { 306 | t.Errorf("PolySwarm failed") 307 | } else { 308 | valid, errmsg := hash.ValidateFile(filename) 309 | 310 | if !valid { 311 | os.Remove(hash.Hash) 312 | t.Errorf(errmsg) 313 | } else { 314 | os.Remove(hash.Hash) 315 | } 316 | } 317 | } 318 | 319 | func TestHybridAnalysis(t *testing.T) { 320 | home, _ := os.UserHomeDir() 321 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 322 | if err != nil { 323 | log.Fatal() 324 | t.Errorf("%v", err) 325 | } 326 | 327 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 328 | if err != nil { 329 | log.Fatal() 330 | t.Errorf("%v", err) 331 | } 332 | 333 | ht, _ := hashType(scfg.Hash) 334 | hash := Hash{HashType: ht, Hash: scfg.Hash} 335 | 336 | var osq ObjectiveSeeQuery 337 | result, filename, _ := HybridAnalysis.QueryAndDownload(cfg, hash, false, osq) 338 | 339 | if !result { 340 | t.Errorf("HybridAnalysis failed") 341 | } else { 342 | valid, errmsg := hash.ValidateFile(filename) 343 | 344 | if !valid { 345 | os.Remove(hash.Hash) 346 | t.Errorf(errmsg) 347 | } else { 348 | os.Remove(hash.Hash) 349 | } 350 | } 351 | } 352 | 353 | func TestTriage(t *testing.T) { 354 | home, _ := os.UserHomeDir() 355 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 356 | if err != nil { 357 | log.Fatal() 358 | t.Errorf("%v", err) 359 | } 360 | 361 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 362 | if err != nil { 363 | log.Fatal() 364 | t.Errorf("%v", err) 365 | } 366 | 367 | ht, _ := hashType(scfg.Hash) 368 | hash := Hash{HashType: ht, Hash: scfg.Hash} 369 | 370 | var osq ObjectiveSeeQuery 371 | result, filename, _ := Triage.QueryAndDownload(cfg, hash, false, osq) 372 | 373 | if !result { 374 | t.Errorf("Triage failed") 375 | } else { 376 | valid, errmsg := hash.ValidateFile(filename) 377 | 378 | if !valid { 379 | os.Remove(hash.Hash) 380 | t.Errorf(errmsg) 381 | } else { 382 | os.Remove(hash.Hash) 383 | } 384 | } 385 | } 386 | 387 | func TestMalShare(t *testing.T) { 388 | home, _ := os.UserHomeDir() 389 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 390 | if err != nil { 391 | log.Fatal() 392 | t.Errorf("%v", err) 393 | } 394 | 395 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 396 | if err != nil { 397 | log.Fatal() 398 | t.Errorf("%v", err) 399 | } 400 | 401 | ht, _ := hashType(scfg.Hash) 402 | hash := Hash{HashType: ht, Hash: scfg.Hash} 403 | 404 | var osq ObjectiveSeeQuery 405 | result, filename, _ := Malshare.QueryAndDownload(cfg, hash, false, osq) 406 | 407 | if !result { 408 | t.Errorf("Malshare failed") 409 | } else { 410 | valid, errmsg := hash.ValidateFile(filename) 411 | 412 | if !valid { 413 | os.Remove(hash.Hash) 414 | t.Errorf(errmsg) 415 | } else { 416 | os.Remove(hash.Hash) 417 | } 418 | } 419 | } 420 | 421 | func TestMalwareBazaar(t *testing.T) { 422 | home, _ := os.UserHomeDir() 423 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 424 | if err != nil { 425 | log.Fatal() 426 | t.Errorf("%v", err) 427 | } 428 | 429 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 430 | if err != nil { 431 | log.Fatal() 432 | t.Errorf("%v", err) 433 | } 434 | 435 | ht, _ := hashType(scfg.Hash) 436 | hash := Hash{HashType: ht, Hash: scfg.Hash} 437 | 438 | var osq ObjectiveSeeQuery 439 | result, filename, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) 440 | 441 | if !result { 442 | t.Errorf("MalwareBazaar failed") 443 | } else { 444 | valid, errmsg := hash.ValidateFile(filename) 445 | 446 | if !valid { 447 | os.Remove(hash.Hash) 448 | t.Errorf(errmsg) 449 | } else { 450 | os.Remove(hash.Hash) 451 | } 452 | } 453 | } 454 | 455 | func TestMalpedia(t *testing.T) { 456 | home, _ := os.UserHomeDir() 457 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 458 | if err != nil { 459 | log.Fatal() 460 | t.Errorf("%v", err) 461 | } 462 | 463 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 464 | if err != nil { 465 | log.Fatal() 466 | t.Errorf("%v", err) 467 | } 468 | 469 | ht, _ := hashType(scfg.Hash) 470 | hash := Hash{HashType: ht, Hash: scfg.Hash} 471 | 472 | var osq ObjectiveSeeQuery 473 | result, filename, _ := Malpedia.QueryAndDownload(cfg, hash, false, osq) 474 | 475 | if !result { 476 | t.Errorf("Malpedia failed") 477 | } else { 478 | valid, errmsg := hash.ValidateFile(filename) 479 | 480 | if !valid { 481 | os.Remove(hash.Hash) 482 | t.Errorf(errmsg) 483 | } else { 484 | os.Remove(hash.Hash) 485 | } 486 | } 487 | } 488 | 489 | func TestUnpacme(t *testing.T) { 490 | home, _ := os.UserHomeDir() 491 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 492 | if err != nil { 493 | log.Fatal() 494 | t.Errorf("%v", err) 495 | } 496 | 497 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 498 | if err != nil { 499 | log.Fatal() 500 | t.Errorf("%v", err) 501 | } 502 | 503 | ht, _ := hashType(scfg.Hash) 504 | hash := Hash{HashType: ht, Hash: scfg.Hash} 505 | 506 | var osq ObjectiveSeeQuery 507 | result, filename, _ := UnpacMe.QueryAndDownload(cfg, hash, false, osq) 508 | 509 | if !result { 510 | t.Errorf("Unpacme failed") 511 | } else { 512 | valid, errmsg := hash.ValidateFile(filename) 513 | 514 | if !valid { 515 | os.Remove(hash.Hash) 516 | t.Errorf(errmsg) 517 | } else { 518 | os.Remove(hash.Hash) 519 | } 520 | } 521 | } 522 | 523 | func TestVxShare(t *testing.T) { 524 | home, _ := os.UserHomeDir() 525 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 526 | if err != nil { 527 | log.Fatal() 528 | t.Errorf("%v", err) 529 | } 530 | 531 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 532 | if err != nil { 533 | log.Fatal() 534 | t.Errorf("%v", err) 535 | } 536 | 537 | ht, _ := hashType(scfg.Hash) 538 | hash := Hash{HashType: ht, Hash: scfg.Hash} 539 | 540 | var osq ObjectiveSeeQuery 541 | result, filename, _ := VxShare.QueryAndDownload(cfg, hash, false, osq) 542 | 543 | if !result { 544 | t.Errorf("VxShare failed") 545 | } else { 546 | valid, errmsg := hash.ValidateFile(filename) 547 | 548 | if !valid { 549 | os.Remove(hash.Hash) 550 | t.Errorf(errmsg) 551 | } else { 552 | os.Remove(hash.Hash) 553 | } 554 | } 555 | } 556 | 557 | func TestFileScanIo(t *testing.T) { 558 | home, _ := os.UserHomeDir() 559 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 560 | if err != nil { 561 | log.Fatal() 562 | t.Errorf("%v", err) 563 | } 564 | 565 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 566 | if err != nil { 567 | log.Fatal() 568 | t.Errorf("%v", err) 569 | } 570 | 571 | ht, _ := hashType(scfg.Hash) 572 | hash := Hash{HashType: ht, Hash: scfg.Hash} 573 | 574 | var osq ObjectiveSeeQuery 575 | result, filename, _ := FileScanIo.QueryAndDownload(cfg, hash, false, osq) 576 | 577 | if !result { 578 | t.Errorf("FileScanIo failed") 579 | } else { 580 | valid, errmsg := hash.ValidateFile(filename) 581 | 582 | if !valid { 583 | os.Remove(hash.Hash) 584 | t.Errorf(errmsg) 585 | } else { 586 | os.Remove(hash.Hash) 587 | } 588 | } 589 | 590 | } 591 | 592 | func TestURLScanIo(t *testing.T) { 593 | home, _ := os.UserHomeDir() 594 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 595 | if err != nil { 596 | log.Fatal() 597 | t.Errorf("%v", err) 598 | } 599 | 600 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 601 | if err != nil { 602 | log.Fatal() 603 | t.Errorf("%v", err) 604 | } 605 | 606 | ht, _ := hashType(scfg.Hash) 607 | hash := Hash{HashType: ht, Hash: scfg.Hash} 608 | 609 | var osq ObjectiveSeeQuery 610 | result, filename, _ := URLScanIO.QueryAndDownload(cfg, hash, false, osq) 611 | 612 | if !result { 613 | t.Errorf("URLScanIO failed") 614 | } else { 615 | valid, errmsg := hash.ValidateFile(filename) 616 | 617 | if !valid { 618 | os.Remove(hash.Hash) 619 | t.Errorf(errmsg) 620 | } else { 621 | os.Remove(hash.Hash) 622 | } 623 | } 624 | 625 | } 626 | 627 | func TestAssemblyLine(t *testing.T) { 628 | home, _ := os.UserHomeDir() 629 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 630 | if err != nil { 631 | log.Fatal() 632 | t.Errorf("%v", err) 633 | } 634 | 635 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 636 | if err != nil { 637 | log.Fatal() 638 | t.Errorf("%v", err) 639 | } 640 | 641 | ht, _ := hashType(scfg.Hash) 642 | hash := Hash{HashType: ht, Hash: scfg.Hash} 643 | 644 | var osq ObjectiveSeeQuery 645 | result, filename, _ := AssemblyLine.QueryAndDownload(cfg, hash, false, osq) 646 | 647 | if !result { 648 | t.Errorf("Assemblyline failed") 649 | } else { 650 | valid, errmsg := hash.ValidateFile(filename) 651 | 652 | if !valid { 653 | os.Remove(filename) 654 | t.Errorf(errmsg) 655 | } else { 656 | os.Remove(filename) 657 | } 658 | } 659 | 660 | } 661 | 662 | func TestVirusExchange(t *testing.T) { 663 | home, _ := os.UserHomeDir() 664 | cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) 665 | if err != nil { 666 | log.Fatal() 667 | t.Errorf("%v", err) 668 | } 669 | 670 | scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) 671 | if err != nil { 672 | log.Fatal() 673 | t.Errorf("%v", err) 674 | } 675 | 676 | ht, _ := hashType(scfg.Hash) 677 | hash := Hash{HashType: ht, Hash: scfg.Hash} 678 | 679 | var osq ObjectiveSeeQuery 680 | result, filename, _ := VirusExchange.QueryAndDownload(cfg, hash, false, osq) 681 | 682 | if !result { 683 | t.Errorf("VirusExchange failed") 684 | } else { 685 | valid, errmsg := hash.ValidateFile(filename) 686 | 687 | if !valid { 688 | os.Remove(filename) 689 | t.Errorf(errmsg) 690 | } else { 691 | os.Remove(filename) 692 | } 693 | } 694 | 695 | } 696 | -------------------------------------------------------------------------------- /mlweb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "strings" 8 | "text/template" 9 | ) 10 | 11 | type Page struct { 12 | hashes []string 13 | tags []string 14 | comments []string 15 | } 16 | 17 | func indexHandler(w http.ResponseWriter, r *http.Request) { 18 | if r.Method == "POST" { 19 | values := strings.Split(r.PostFormValue("hashes"), "\r\n") 20 | tags := strings.Split(r.PostFormValue("tags"), "\r\n") 21 | comments := strings.Split(r.PostFormValue("comments"), "\r\n") 22 | go processMalwareDownloadRequest(values, tags, comments) 23 | } 24 | t, _ := template.ParseFiles("./web/templates/index.html") 25 | t.Execute(w, nil) 26 | } 27 | 28 | func processMalwareDownloadRequest(values []string, tags []string, comments []string) { 29 | //hashes := parseArgHashes(values, tags, comments) 30 | //downloadMalwareFromWebServer(hashes) 31 | } 32 | 33 | func runWebServer(bind string, port int) { 34 | 35 | http.HandleFunc("/styles/style.css", func(response http.ResponseWriter, request *http.Request) { 36 | http.ServeFile(response, request, "./web/styles/style.css") 37 | }) 38 | 39 | http.HandleFunc("/scripts/script.js", func(response http.ResponseWriter, request *http.Request) { 40 | http.ServeFile(response, request, "./web/scripts/script.js") 41 | }) 42 | 43 | http.HandleFunc("/", indexHandler) 44 | 45 | //http.HandleFunc("/download", postDataHandler) 46 | 47 | log.Fatal(http.ListenAndServe(fmt.Sprint(bind, ":", port), nil)) 48 | } 49 | -------------------------------------------------------------------------------- /upload.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "mime/multipart" 11 | "net/http" 12 | "net/url" 13 | "os" 14 | "strings" 15 | ) 16 | 17 | func doesSampleExistInAssemblyLine(uri string, api string, user string, hash Hash, ignoreTLSErrors bool) bool { 18 | if api == "" { 19 | fmt.Println(" [!] !! Missing Key !!") 20 | return false 21 | } 22 | if user == "" { 23 | fmt.Println(" [!] !! Missing User !!") 24 | return false 25 | } 26 | 27 | request, error := http.NewRequest("GET", uri+"/hash_search/"+url.PathEscape(hash.Hash)+"/", nil) 28 | if error != nil { 29 | fmt.Println(error) 30 | return false 31 | } 32 | 33 | request.Header.Set("Content-Type", "application/json; charset=UTF-8") 34 | request.Header.Set("x-user", user) 35 | request.Header.Set("x-apikey", api) 36 | 37 | tr := &http.Transport{} 38 | if ignoreTLSErrors { 39 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 40 | } 41 | 42 | client := &http.Client{Transport: tr} 43 | response, error := client.Do(request) 44 | if error != nil { 45 | fmt.Printf(" [!] Error with querying AssemblyLine for hash : %s\n", error) 46 | return false 47 | } 48 | defer response.Body.Close() 49 | 50 | if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { 51 | fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") 52 | return false 53 | } 54 | 55 | byteValue, _ := io.ReadAll(response.Body) 56 | 57 | var data = AssemblyLineQuery{} 58 | error = json.Unmarshal(byteValue, &data) 59 | 60 | if error != nil { 61 | fmt.Println(error) 62 | return false 63 | } 64 | 65 | if data.Response.AL == nil { 66 | return false 67 | } 68 | 69 | if len(data.Response.AL.Items) > 0 { 70 | return true 71 | } 72 | return false 73 | } 74 | 75 | func UploadSampleToAssemblyLine(repos []RepositoryConfigEntry, filename string, hash Hash, deleteFromDisk bool, forceResubmission bool) error { 76 | matchingConfigRepos := getConfigsByType(UploadAssemblyLine, repos) 77 | if len(matchingConfigRepos) == 0 { 78 | return fmt.Errorf(" upload to assemblyline config entry not found") 79 | } 80 | for _, mcr := range matchingConfigRepos { 81 | if !forceResubmission && doesSampleExistInAssemblyLine(mcr.Host, mcr.Api, mcr.User, hash, mcr.IgnoreTLSErrors) { 82 | fmt.Println(" Sample Already Exist in AssemblyLine. Not Reuploading.") 83 | continue 84 | } 85 | 86 | bodyBuf := &bytes.Buffer{} 87 | bodyWriter := multipart.NewWriter(bodyBuf) 88 | 89 | // this step is very important 90 | fileWriter, err := bodyWriter.CreateFormFile("bin", filename) 91 | if err != nil { 92 | return fmt.Errorf("error writing to buffer") 93 | } 94 | 95 | // open file handle 96 | fh, err := os.Open(filename) 97 | if err != nil { 98 | return fmt.Errorf("error opening file") 99 | } 100 | defer fh.Close() 101 | 102 | //iocopy 103 | _, err = io.Copy(fileWriter, fh) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | contentType := bodyWriter.FormDataContentType() 109 | bodyWriter.Close() 110 | 111 | request, error := http.NewRequest("POST", mcr.Host+"/submit/", bodyBuf) 112 | if error != nil { 113 | return error 114 | } 115 | 116 | request.Header.Set("accept", "application/json") 117 | request.Header.Set("x-user", mcr.User) 118 | request.Header.Set("x-apikey", mcr.Api) 119 | request.Header.Set("Content-Type", contentType) 120 | 121 | tr := &http.Transport{} 122 | if mcr.IgnoreTLSErrors { 123 | tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 124 | } 125 | client := &http.Client{Transport: tr} 126 | resp, err := client.Do(request) 127 | if err != nil { 128 | return err 129 | } 130 | 131 | defer resp.Body.Close() 132 | 133 | if resp.StatusCode != http.StatusOK { 134 | return fmt.Errorf("error uploading file - status code %d returned", resp.StatusCode) 135 | 136 | } else { 137 | fmt.Printf(" %s uploaded to AssemblyLine (%s)\n", filename, mcr.Host) 138 | if deleteFromDisk { 139 | os.Remove(filename) 140 | fmt.Printf(" %s deleted from disk\n", filename) 141 | } 142 | } 143 | } 144 | return nil 145 | } 146 | 147 | type CommentItemResponse struct { 148 | Author string `json:"author"` 149 | Comment string `json:"comment"` 150 | Id int32 `json:"id"` 151 | Timestamp string `json:"timestamp"` 152 | } 153 | 154 | func doesSampleExistInMWDB(uri string, api string, hash string) bool { 155 | query := uri + "/file/" + hash 156 | 157 | request, err := http.NewRequest("GET", query, nil) 158 | if err != nil { 159 | fmt.Println(err) 160 | return false 161 | } 162 | 163 | request.Header.Set("Authorization", "Bearer "+api) 164 | client := &http.Client{} 165 | response, error := client.Do(request) 166 | if error != nil { 167 | fmt.Println(error) 168 | return false 169 | } 170 | defer response.Body.Close() 171 | 172 | if response.StatusCode == http.StatusOK { 173 | fmt.Printf(" [!] File %s already exists in MWDB: %s \n", hash, uri) 174 | return true 175 | } else { 176 | fmt.Println("") 177 | return false 178 | } 179 | } 180 | 181 | func UploadSampleToMWDBs(repos []RepositoryConfigEntry, filename string, hash Hash, deleteFromDisk bool) error { 182 | matchingConfigRepos := getConfigsByType(UploadMWDB, repos) 183 | for _, mcr := range matchingConfigRepos { 184 | if doesSampleExistInMWDB(mcr.Host, mcr.Api, hash.Hash) { 185 | continue 186 | } 187 | 188 | bodyBuf := &bytes.Buffer{} 189 | bodyWriter := multipart.NewWriter(bodyBuf) 190 | 191 | // this step is very important 192 | fileWriter, err := bodyWriter.CreateFormFile("file", filename) 193 | if err != nil { 194 | fmt.Println("error writing to buffer") 195 | return err 196 | } 197 | 198 | // open file handle 199 | fh, err := os.Open(filename) 200 | if err != nil { 201 | fmt.Println("error opening file") 202 | return err 203 | } 204 | defer fh.Close() 205 | 206 | //iocopy 207 | _, err = io.Copy(fileWriter, fh) 208 | if err != nil { 209 | return err 210 | } 211 | 212 | contentType := bodyWriter.FormDataContentType() 213 | bodyWriter.Close() 214 | 215 | request, error := http.NewRequest("POST", mcr.Host+"/file", bodyBuf) 216 | if error != nil { 217 | fmt.Println(error) 218 | return error 219 | } 220 | 221 | request.Header.Set("Authorization", "Bearer "+mcr.Api) 222 | request.Header.Set("Content-Type", contentType) 223 | 224 | client := &http.Client{} 225 | resp, err := client.Do(request) 226 | if err != nil { 227 | return err 228 | } 229 | 230 | defer resp.Body.Close() 231 | 232 | if resp.StatusCode != http.StatusOK { 233 | return fmt.Errorf("error uploading file - status code %d returned", resp.StatusCode) 234 | 235 | } else { 236 | fmt.Printf(" [-] %s uploaded to MWDB (%s)\n", filename, mcr.Host) 237 | if deleteFromDisk { 238 | os.Remove(filename) 239 | fmt.Printf(" [-] %s deleted from disk\n", filename) 240 | } 241 | } 242 | 243 | err = addTagsToSampleInMWDB(hash, mcr.Host, mcr.Api) 244 | if err != nil { 245 | return err 246 | } 247 | 248 | err = addCommentsToSampleInMWDB(hash, mcr.Host, mcr.Api) 249 | if err != nil { 250 | return err 251 | } 252 | } 253 | return nil 254 | } 255 | 256 | func AddTagsToSamplesAcrossMWDBs(repos []RepositoryConfigEntry, hash Hash) { 257 | matchingConfigRepos := getConfigsByType(UploadMWDB, repos) 258 | for _, mcr := range matchingConfigRepos { 259 | if !doesSampleExistInMWDB(mcr.Host, mcr.Api, hash.Hash) { 260 | continue 261 | } 262 | err := addTagsToSampleInMWDB(hash, mcr.Host, mcr.Api) 263 | if err != nil { 264 | fmt.Printf("Error occurred while tagging %s on %s (%s)\n", hash.Hash, mcr.Type, mcr.Host) 265 | } 266 | } 267 | } 268 | 269 | func addTagsToSampleInMWDB(hash Hash, mwdbServer string, auth string) error { 270 | for _, t := range hash.Tags { 271 | query := mwdbServer + "/file/" + hash.Hash + "/tag" 272 | 273 | _, error := url.ParseQuery(query) 274 | if error != nil { 275 | fmt.Println(error) 276 | return error 277 | } 278 | 279 | value_json := "{\"tag\":\"" + t + "\"}" 280 | request, error := http.NewRequest("PUT", query, strings.NewReader(value_json)) 281 | if error != nil { 282 | fmt.Println(error) 283 | return error 284 | } 285 | 286 | request.Header.Set("Authorization", "Bearer "+auth) 287 | 288 | client := &http.Client{} 289 | respTag, err := client.Do(request) 290 | if err != nil { 291 | return err 292 | } 293 | 294 | if respTag.StatusCode == http.StatusOK { 295 | fmt.Printf(" [-] %s tagged as %s\n", hash.Hash, t) 296 | } else { 297 | fmt.Printf(" [!] Failed to tag %s as %s\n", hash.Hash, t) 298 | } 299 | } 300 | return nil 301 | } 302 | 303 | func AddCommentsToSamplesAcrossMWDBs(repos []RepositoryConfigEntry, hash Hash) { 304 | matchingConfigRepos := getConfigsByType(UploadMWDB, repos) 305 | for _, mcr := range matchingConfigRepos { 306 | if !doesSampleExistInMWDB(mcr.Host, mcr.Api, hash.Hash) { 307 | continue 308 | } 309 | err := addCommentsToSampleInMWDB(hash, mcr.Host, mcr.Api) 310 | if err != nil { 311 | fmt.Printf("Error occurred while adding comments to %s on %s (%s)\n", hash.Hash, mcr.Type, mcr.Host) 312 | } 313 | } 314 | } 315 | 316 | func addCommentsToSampleInMWDB(hash Hash, mwdbServer string, auth string) error { 317 | 318 | // Get existing comments 319 | getQuery := mwdbServer + "/file/" + hash.Hash + "/comment" 320 | _, error := url.ParseQuery(getQuery) 321 | if error != nil { 322 | fmt.Println(error) 323 | return error 324 | } 325 | getRequest, error := http.NewRequest("GET", getQuery, nil) 326 | if error != nil { 327 | fmt.Println(error) 328 | return error 329 | } 330 | getRequest.Header.Set("Authorization", "Bearer "+auth) 331 | 332 | getClient := &http.Client{} 333 | getResponse, error := getClient.Do(getRequest) 334 | if error != nil { 335 | fmt.Println(error) 336 | return error 337 | } 338 | defer getResponse.Body.Close() 339 | 340 | var getData []CommentItemResponse 341 | if getResponse.StatusCode == http.StatusOK { 342 | 343 | byteValue, error := ioutil.ReadAll(getResponse.Body) 344 | if error != nil { 345 | fmt.Println(error) 346 | return error 347 | } 348 | 349 | error = json.Unmarshal(byteValue, &getData) 350 | 351 | if error != nil { 352 | fmt.Println(error) 353 | return error 354 | } 355 | } 356 | 357 | for _, c := range hash.Comments { 358 | 359 | // Check to make sure the comment does not already exists before added it, if it does exist continue on to the next comment 360 | commentExists := false 361 | if len(getData) > 0 { 362 | for _, existingComment := range getData { 363 | if c == existingComment.Comment { 364 | commentExists = true 365 | break 366 | } 367 | } 368 | } 369 | if commentExists { 370 | fmt.Printf(" [!] %s comment already exists for %s\n", c, hash.Hash) 371 | continue 372 | } 373 | 374 | // Add net comment to sample 375 | query := mwdbServer + "/file/" + hash.Hash + "/comment" 376 | 377 | _, error := url.ParseQuery(query) 378 | if error != nil { 379 | fmt.Println(error) 380 | return error 381 | } 382 | 383 | value_json := "{\"comment\":\"" + c + "\"}" 384 | request, error := http.NewRequest("POST", query, strings.NewReader(value_json)) 385 | if error != nil { 386 | fmt.Println(error) 387 | return error 388 | } 389 | 390 | request.Header.Set("Authorization", "Bearer "+auth) 391 | 392 | client := &http.Client{} 393 | respTag, err := client.Do(request) 394 | if err != nil { 395 | return err 396 | } 397 | 398 | if respTag.StatusCode == http.StatusOK { 399 | fmt.Printf(" [-] %s comment added for %s\n", c, hash.Hash) 400 | } else { 401 | fmt.Printf(" [!] Failed to comment %s for %s\n", c, hash.Hash) 402 | } 403 | } 404 | return nil 405 | } 406 | -------------------------------------------------------------------------------- /web/scripts/script.js: -------------------------------------------------------------------------------- 1 | var table = $('#hashes').DataTable( { 2 | serverSide: true, 3 | ajax: '/data-source' 4 | } ); 5 | 6 | // Attach a submit handler to the form 7 | $( "#download" ).submit(function( event ) { 8 | 9 | // Stop form from submitting normally 10 | event.preventDefault(); 11 | 12 | // Get some values from elements on the page: 13 | var $form = $( this ), 14 | term = $form.find( "input[name='hashes']" ).val(), 15 | url = $form.attr( "action" ); 16 | 17 | // Send the data using post 18 | var posting = $.post( url, { hashes: term } ); 19 | 20 | // Put the results in a div 21 | posting.done(function( data ) { 22 | table.ajax.reload( null, false ); // user paging is not reset on reload 23 | }); 24 | }); 25 | 26 | 27 | setInterval( function () { 28 | table.ajax.reload( null, false ); // user paging is not reset on reload 29 | }, 30000 ); -------------------------------------------------------------------------------- /web/styles/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xorhex/mlget/02c69e58f65a59e356c33ec59fcbc239f275ebfc/web/styles/style.css -------------------------------------------------------------------------------- /web/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 10 | 11 | 13 | 14 | 27 | 28 | 29 |
30 |
31 |
32 |

MLGET - Download Malware

33 |
34 |
35 | 36 |
37 |
38 | 39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 | 47 |
48 |
49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 |
HashHash TypeFound On
60 |
61 |
--------------------------------------------------------------------------------