├── .dockerignore ├── .gitignore ├── Dockerfile ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── README.md ├── main.go ├── masscan.list └── nmap.csv /.dockerignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | .idea/ 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | vendor/ 15 | results.csv 16 | .idea/ 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build Container 2 | FROM golang:1.9.4-alpine3.7 AS build-env 3 | RUN apk add --no-cache --upgrade git openssh-client ca-certificates 4 | RUN go get -u github.com/golang/dep/cmd/dep 5 | 6 | WORKDIR /go/src/github.com/anshumanbh/merge-nmap-masscan 7 | 8 | # Cache the dependencies early 9 | COPY Gopkg.toml Gopkg.lock ./ 10 | RUN dep ensure -vendor-only -v 11 | 12 | COPY main.go ./ 13 | 14 | RUN go install 15 | 16 | # Final Container 17 | FROM alpine:3.7 18 | LABEL maintainer="Anshuman Bhartiya" 19 | COPY --from=build-env /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 20 | COPY --from=build-env /go/bin/merge-nmap-masscan /usr/bin/merge-nmap-masscan 21 | 22 | COPY masscan.list / 23 | COPY nmap.csv / 24 | 25 | ENTRYPOINT ["/usr/bin/merge-nmap-masscan"] 26 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | branch = "master" 6 | name = "github.com/gocarina/gocsv" 7 | packages = ["."] 8 | revision = "a5c9099e2484f1551abb9433885e158610a25f4b" 9 | 10 | [solve-meta] 11 | analyzer-name = "dep" 12 | analyzer-version = 1 13 | inputs-digest = "c63b60246416084d16accd1df90ec1f303f31d14e650ad67f0c895ed8ee6ce13" 14 | solver-name = "gps-cdcl" 15 | solver-version = 1 16 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | branch = "master" 30 | name = "github.com/gocarina/gocsv" 31 | 32 | [prune] 33 | go-tests = true 34 | unused-packages = true 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Anshuman Bhartiya 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # merge-nmap-masscan 2 | 3 | A quick and ditry utility that can be used for merging NMAP's CSV file (Use [this](https://github.com/anshumanbh/nmapxmltocsv) to convert NMAP XML to CSV) and Masscan's List file (use the -oL flag to generate a list file for masscan output) into one consolidated CSV file to be used in other automation and workflows. 4 | 5 | NMAP file looks like this: 6 | ``` 7 | Hostname,IPAddress,Port,Protocol,Servicename,Servicestate 8 | anshumanbhartiya.com,104.198.14.52,80,tcp,tcpwrapped,open 9 | anshumanbhartiya.com,104.198.14.52,443,tcp,tcpwrapped,open 10 | anshumanbhartiya.com,104.198.14.52,4443,tcp,tcpwrapped,open 11 | github.anshumanbhartiya.com,185.199.110.153,80,tcp,tcpwrapped,open 12 | github.anshumanbhartiya.com,185.199.110.153,443,tcp,tcpwrapped,open 13 | yolo.anshumanbhartiya.com,1.2.3.4,999,tcp,tcpwrapped,open 14 | ``` 15 | 16 | Masscan file looks like this: 17 | ``` 18 | #masscan 19 | open tcp 43 104.198.14.52 1529229065 20 | open tcp 443 185.199.110.153 1529229065 21 | open tcp 4443 104.198.14.52 1529229065 22 | open tcp 80 104.198.14.52 1529229065 23 | open tcp 80 185.199.110.153 1529229065 24 | open tcp 8000 185.199.110.153 1529229065 25 | open tcp 8080 1.2.3.4 1529229065 26 | open tcp 8080 104.198.14.52 1529229065 27 | open tcp 8800 185.199.110.153 1529229065 28 | # end 29 | ``` 30 | 31 | The final merged file will look like this: 32 | ``` 33 | Hostname,IPAddress,Port,Protocol,Servicename,Servicestate 34 | anshumanbhartiya.com,104.198.14.52,80,tcp,tcpwrapped,open 35 | anshumanbhartiya.com,104.198.14.52,443,tcp,tcpwrapped,open 36 | anshumanbhartiya.com,104.198.14.52,4443,tcp,tcpwrapped,open 37 | github.anshumanbhartiya.com,185.199.110.153,80,tcp,tcpwrapped,open 38 | github.anshumanbhartiya.com,185.199.110.153,443,tcp,tcpwrapped,open 39 | yolo.anshumanbhartiya.com,1.2.3.4,999,tcp,tcpwrapped,open 40 | anshumanbhartiya.com,104.198.14.52,43,tcp,NA,open 41 | github.anshumanbhartiya.com,185.199.110.153,8000,tcp,NA,open 42 | yolo.anshumanbhartiya.com,1.2.3.4,8080,tcp,NA,open 43 | anshumanbhartiya.com,104.198.14.52,8080,tcp,NA,open 44 | github.anshumanbhartiya.com,185.199.110.153,8800,tcp,NA,open 45 | ``` 46 | 47 | The above masscan and nmap files are provided with the repo and the Docker image as well!! 48 | 49 | ## Running 50 | Either run with defaults - `go run main.go` 51 | 52 | OR 53 | 54 | Specify the files - `go run main.go -masscanFile <> -nmapFile <> -outFile <>` 55 | 56 | ### Don't have GO? Use Docker instead 57 | `docker run -it abhartiya/tools_mergenmapmasscan` 58 | 59 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "os/exec" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/gocarina/gocsv" 15 | ) 16 | 17 | type config struct { 18 | nmapFile string 19 | masscanFile string 20 | outFile string 21 | } 22 | 23 | type portscan struct { 24 | Hostname string 25 | IPAddress string 26 | Port int 27 | Protocol string 28 | Servicename string 29 | Servicestate string 30 | } 31 | 32 | var ( 33 | cfg config 34 | ps []portscan 35 | found bool 36 | m map[string]string 37 | ) 38 | 39 | func loadConfig() { 40 | nmapFile := flag.String("nmapFile", "nmap.csv", "Nmap scan file in CSV") 41 | masscanFile := flag.String("masscanFile", "masscan.list", "Masscan scan file in List format") 42 | outFile := flag.String("outFile", "results.csv", "Final output merged file") 43 | 44 | flag.Parse() 45 | 46 | cfg = config{ 47 | nmapFile: *nmapFile, 48 | masscanFile: *masscanFile, 49 | outFile: *outFile, 50 | } 51 | 52 | } 53 | 54 | func exists(path string) (bool, int64, error) { 55 | fi, err := os.Stat(path) 56 | if err == nil { 57 | return true, fi.Size(), nil 58 | } 59 | if os.IsNotExist(err) { 60 | return false, int64(0), nil 61 | } 62 | return false, int64(0), err 63 | } 64 | 65 | func ensureFilePathExists(filepath string) error { 66 | value := false 67 | fsize := int64(0) 68 | 69 | for (value == false) || (fsize == int64(0)) { 70 | i, s, err := exists(filepath) 71 | if err != nil { 72 | log.Println("Failed to determine if the file exists or not..") 73 | } 74 | value = i 75 | fsize = s 76 | } 77 | 78 | log.Println(filepath+" File exists:", value) 79 | log.Println(filepath+" File size:", fsize) 80 | 81 | return nil 82 | } 83 | 84 | func loopnmapfile(nmapfile string) error { 85 | 86 | m = make(map[string]string) 87 | 88 | ns, err := os.Open(nmapfile) 89 | if err != nil { 90 | log.Printf("Couldn't open the NMAP Scan file: %v", err) 91 | return err 92 | } 93 | defer ns.Close() 94 | 95 | nsfScanner := bufio.NewScanner(ns) 96 | nsfScanner.Split(bufio.ScanLines) // splitting at each line 97 | nsfcount := 0 98 | 99 | fmt.Println("\nCreating a map of IP=Host from the NMAP scan file\n") 100 | 101 | for nsfScanner.Scan() { 102 | if nsfcount == 0 { 103 | nsfcount++ //don't need the first line, increment the counter 104 | } else { 105 | nsline := strings.Split(nsfScanner.Text(), ",") 106 | tp, err := strconv.Atoi(nsline[2]) 107 | if err != nil { 108 | log.Printf("Couldn't convert port string to int: %v", err) 109 | return err 110 | } 111 | 112 | _, ok := m[nsline[1]] 113 | if ok { 114 | // If an IP already exists, it prolly has a host as well 115 | } else { 116 | m[nsline[1]] = nsline[0] // If not, set the ip=host key/value pair 117 | } 118 | 119 | p := portscan{ 120 | Hostname: nsline[0], 121 | IPAddress: nsline[1], 122 | Port: tp, 123 | Protocol: nsline[3], 124 | Servicename: nsline[4], 125 | Servicestate: nsline[5], 126 | } 127 | ps = append(ps, p) 128 | } 129 | } 130 | fmt.Printf("Map: \n%s\n", m) 131 | fmt.Printf("\nPortscan struct formed from the NMAP scan file only: \n%v\n", ps) 132 | 133 | return nil 134 | } 135 | 136 | func writeResultsToCsv(scanResults []portscan, outputFilePath string) error { 137 | outputFile, err := os.Create(outputFilePath) 138 | if err != nil { 139 | fmt.Printf("Couldn't create the output file: %v", err) 140 | return err 141 | } 142 | defer outputFile.Close() 143 | 144 | err = gocsv.MarshalFile(&scanResults, outputFile) 145 | if err != nil { 146 | fmt.Printf("Couldn't marshal the output file: %v", err) 147 | return err 148 | } 149 | return nil 150 | } 151 | 152 | func main() { 153 | 154 | loadConfig() 155 | 156 | err := ensureFilePathExists(cfg.nmapFile) 157 | if err != nil { 158 | log.Fatalf("Couldn't ensure whether the NMAP file exists or not: %v", err) 159 | } 160 | 161 | err = ensureFilePathExists(cfg.masscanFile) 162 | if err != nil { 163 | log.Fatalf("Couldn't ensure whether the Masscan file exists or not: %v", err) 164 | } 165 | 166 | err = loopnmapfile(cfg.nmapFile) // need to loop through the nmap scan file and create a new array of portscans from that so that we can just keep appending anything new to that list 167 | if err != nil { 168 | log.Fatalf("Couldn't loop the nmap scan file: %v", err) 169 | } 170 | 171 | // need to do some sed magic on the masscan file to get a CSV from the Masscan generated list file. stored at /tmp/masscantemp.txt 172 | masscanCmd := exec.Command("sh", "-c", "sed '1d;$d' "+cfg.masscanFile+" | cut -d' ' -f1,2,3,4 | sed 's/ /,/g' | sort -u > /tmp/masscantemp.txt") 173 | var masscanOut, masscanStderr bytes.Buffer 174 | masscanCmd.Stdout = &masscanOut 175 | masscanCmd.Stderr = &masscanStderr 176 | err = masscanCmd.Run() 177 | if err != nil { 178 | log.Fatalf("Couldn't run the sed command for Masscan Scan file: %v", err) 179 | } 180 | 181 | // now scanning the temp masscan file per line and splitting on , just like the nmap file. no need to skip the first line though 182 | ms, err := os.Open("/tmp/masscantemp.txt") 183 | if err != nil { 184 | log.Fatalf("Couldn't open the masscan temp file: %v", err) 185 | } 186 | defer ms.Close() 187 | 188 | msfScanner := bufio.NewScanner(ms) 189 | msfScanner.Split(bufio.ScanLines) // splitting at each line 190 | 191 | found = false 192 | 193 | // For each line in the masscan file, we need to loop over the entire nmap file and see if an entry already exists from before for that masscan line 194 | // If it does, we have found a duplicate and need to move onto the next line in the masscan file 195 | // If it does not, we have found a new record found by masscan, which we need to merge with the existing nmap file. 196 | 197 | for msfScanner.Scan() { //looping over each line of the masscan file 198 | fmt.Println("=======================================") 199 | msline := strings.Split(msfScanner.Text(), ",") 200 | fmt.Printf("\nLine of the sorted Masscan file: %v\n", msline) 201 | fmt.Println("Now comparing this line with each line of the NMAP Scan file..\n") 202 | 203 | mip := msline[3] 204 | mstate := msline[0] 205 | mport := msline[2] 206 | mprotocol := msline[1] 207 | 208 | nmapscanfile, err := os.Open(cfg.nmapFile) 209 | if err != nil { 210 | log.Fatalf("Couldn't open the NMAP Scan file: %v", err) 211 | } 212 | defer nmapscanfile.Close() 213 | 214 | nmapScanner := bufio.NewScanner(nmapscanfile) 215 | nmapScanner.Split(bufio.ScanLines) // splitting at each line 216 | ncount := 0 217 | 218 | for nmapScanner.Scan() { // for each line from the masscan file, looping over all lines of the nmap file to find a possible match 219 | 220 | if ncount == 0 { 221 | ncount++ //don't need the first line, increment the counter 222 | } else { 223 | 224 | nsline := strings.Split(nmapScanner.Text(), ",") 225 | fmt.Println(nsline) 226 | 227 | nip := nsline[1] 228 | nport := nsline[2] 229 | nstate := nsline[5] 230 | nprotocol := nsline[3] 231 | 232 | if mip == nip && mstate == nstate && mport == nport && mprotocol == nprotocol { 233 | fmt.Println("exact match found..no need to add anything!") 234 | found = true 235 | break // need to stop scanning any more lines from the nmap file 236 | } else { 237 | fmt.Println("match not found") 238 | found = false 239 | continue // match not found so need to continue iterating through the rest of the lines in the nmap file 240 | } 241 | 242 | } 243 | } 244 | 245 | if found { 246 | continue // exact match found so need to move onto the next line in the masscan file 247 | } else { 248 | // not found in any of the rows of the nmap scan file so need to create an entry 249 | p, err := strconv.Atoi(mport) 250 | if err != nil { 251 | log.Fatalf("Couldn't convert masscan port from string to int: %v", err) 252 | } 253 | 254 | foo := portscan{ 255 | Hostname: m[mip], 256 | IPAddress: mip, 257 | Port: p, 258 | Protocol: mprotocol, 259 | Servicename: "NA", //masscan doesn't have the service name unfortunately :( 260 | Servicestate: mstate, 261 | } 262 | ps = append(ps, foo) 263 | } 264 | 265 | fmt.Println("Since no match found, adding the masscan entry into the portscan struct..\n") 266 | fmt.Println(ps) 267 | 268 | // setting back the defaults 269 | found = false 270 | 271 | } 272 | 273 | // writing the ps to the outfile once everything is done 274 | err = writeResultsToCsv(ps, cfg.outFile) 275 | if err != nil { 276 | log.Fatalf("Couldn't write to the out file: %v", err) 277 | } 278 | 279 | fmt.Println("=======================================") 280 | fmt.Println("Results saved to: " + cfg.outFile) 281 | } 282 | -------------------------------------------------------------------------------- /masscan.list: -------------------------------------------------------------------------------- 1 | #masscan 2 | open tcp 43 104.198.14.52 1529229065 3 | open tcp 443 185.199.110.153 1529229065 4 | open tcp 4443 104.198.14.52 1529229065 5 | open tcp 80 104.198.14.52 1529229065 6 | open tcp 80 185.199.110.153 1529229065 7 | open tcp 8000 185.199.110.153 1529229065 8 | open tcp 8080 1.2.3.4 1529229065 9 | open tcp 8080 104.198.14.52 1529229065 10 | open tcp 8800 185.199.110.153 1529229065 11 | # end 12 | -------------------------------------------------------------------------------- /nmap.csv: -------------------------------------------------------------------------------- 1 | Hostname,IPAddress,Port,Protocol,Servicename,Servicestate 2 | anshumanbhartiya.com,104.198.14.52,80,tcp,tcpwrapped,open 3 | anshumanbhartiya.com,104.198.14.52,443,tcp,tcpwrapped,open 4 | anshumanbhartiya.com,104.198.14.52,4443,tcp,tcpwrapped,open 5 | github.anshumanbhartiya.com,185.199.110.153,80,tcp,tcpwrapped,open 6 | github.anshumanbhartiya.com,185.199.110.153,443,tcp,tcpwrapped,open 7 | yolo.anshumanbhartiya.com,1.2.3.4,999,tcp,tcpwrapped,open 8 | --------------------------------------------------------------------------------