├── FlowPlotterD3 ├── data │ ├── data.csv │ ├── data.tsv │ ├── ms.csv │ └── sensordata.csv ├── output.file ├── notes ├── README ├── templates │ ├── stylesheet.css │ ├── jquery.tipsy.js │ ├── linechart.html │ ├── mslinechart.html │ └── linechart.html.bak ├── mslinechart.html └── flowplotter.sh ├── googlechart ├── geomap.html ├── piechart.html ├── barchart.html ├── histogram.html ├── columnchart.html ├── linechart.html ├── orgchart.html ├── timeline.html ├── bubblechart.html ├── treemap.html └── tablechart.html ├── LICENSE.md ├── AssetDiscoveryTree.sh ├── scripts └── AssetDiscoveryTree.sh ├── d3chart ├── treeasset.html └── forceopacity.html ├── README.md └── flowplotter.sh /FlowPlotterD3/data/data.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /FlowPlotterD3/data/data.tsv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /FlowPlotterD3/data/ms.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /FlowPlotterD3/output.file: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /FlowPlotterD3/data/sensordata.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /FlowPlotterD3/notes: -------------------------------------------------------------------------------- 1 | seriesselect is for the "select sensor" or whatever it might be 2 | titleselect is for the title of the page 3 | optionlist is for the list of items in the series 4 | 5 | -------------------------------------------------------------------------------- /FlowPlotterD3/README: -------------------------------------------------------------------------------- 1 | This is just a test directory for converting all of the old FlowPlotter to newer D3 versions with more capability. 2 | Old FlowPlotter will remain as it is, but this will serve as the newer "better" FlowPlotter once it is complete. 3 | -------------------------------------------------------------------------------- /googlechart/geomap.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 20 | 21 |

22 | 23 | 24 | -------------------------------------------------------------------------------- /googlechart/piechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 21 | 22 | 23 |

24 | 25 | 26 | -------------------------------------------------------------------------------- /googlechart/barchart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 21 | 22 | 23 |

24 | 25 | 26 | -------------------------------------------------------------------------------- /googlechart/histogram.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 20 | 21 |

22 | 23 | 24 | -------------------------------------------------------------------------------- /googlechart/columnchart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 21 | 22 | 23 |

24 | 25 | 26 | -------------------------------------------------------------------------------- /googlechart/linechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 22 | 23 | 24 |

25 | 26 | 27 | -------------------------------------------------------------------------------- /googlechart/orgchart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 20 | 21 | 22 |

23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /googlechart/timeline.html: -------------------------------------------------------------------------------- 1 | 3 | 23 | 24 |

25 | -------------------------------------------------------------------------------- /googlechart/bubblechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 23 | 24 | 25 |

26 | 27 | 28 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Jason Smith 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /googlechart/treemap.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /googlechart/tablechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 30 | 31 | 32 | 33 |

34 | 35 | 36 | -------------------------------------------------------------------------------- /FlowPlotterD3/templates/stylesheet.css: -------------------------------------------------------------------------------- 1 | 2 | .plot{ 3 | fill: #fff; 4 | } 5 | .axis path, 6 | .axis line { 7 | fill: none; 8 | stroke: #AFBABF; 9 | shape-rendering: crispEdges; 10 | } 11 | .axis text { 12 | font-family: SourceSansPro-Bold; 13 | font-size: 14px; 14 | fill: #AFBABF; 15 | } 16 | 17 | .x.axis path { 18 | display: none; 19 | } 20 | .y.axis path{ 21 | display: none; 22 | } 23 | 24 | .line { 25 | fill: none; 26 | stroke: steelblue; 27 | stroke-linejoin: round; 28 | stroke-linecap:round; 29 | } 30 | 31 | .hed { 32 | font-family: SourceSansPro-Bold; 33 | font-size: 24pt; 34 | margin-bottom:5px; 35 | margin-top:0px; 36 | } 37 | 38 | .dek { 39 | font-family: SourceSansPro-Light; 40 | font-size: 6pt; 41 | margin-bottom:0px; 42 | margin-top:0px; 43 | } 44 | .menuchoice { 45 | font-family: SourceSansPro-Light; 46 | font-size: 10pt; 47 | margin-bottom:0px; 48 | fill: #231F20; 49 | } 50 | .legend { 51 | font-family: SourceSansPro-Light; 52 | font-size: 10pt; 53 | margin-bottom:0px; 54 | fill: #231F20; 55 | } 56 | .legend-off { 57 | font-family: SourceSansPro-Light; 58 | font-size: 10pt; 59 | margin-bottom:0px; 60 | fill: #231F20; 61 | } 62 | .legend-select { 63 | font-family: SourceSansPro-Bold; 64 | font-size: 10pt; 65 | margin-bottom:0px; 66 | fill: #231F20; 67 | } 68 | .tip1 { 69 | font-family: SourceSansPro-Bold; 70 | font-size: 14pt; 71 | text-align: center; 72 | margin-bottom:-12px; 73 | margin-top:0px; 74 | } 75 | 76 | .tip2 { 77 | font-family: SourceSansPro-ExtraLight; 78 | font-size: 10pt; 79 | text-align: center; 80 | margin-bottom:-12px; 81 | } 82 | .tip3 { 83 | font-family: SourceSansPro-ExtraLight; 84 | font-size: 10pt; 85 | text-align: center; 86 | margin-bottom:0px; 87 | } 88 | 89 | @font-face { 90 | font-family: 'SourceSansPro-ExtraLight'; 91 | font-style: normal; 92 | font-weight: 200; 93 | src: local('Source Sans Pro ExtraLight'), local('SourceSansPro-ExtraLight'), url(http://themes.googleusercontent.com/static/fonts/sourcesanspro/v5/toadOcfmlt9b38dHJxOBGC-IiwWNzQT_LXAtr-Ee9Pw.woff) format('woff'); 94 | } 95 | @font-face { 96 | font-family: 'SourceSansPro-Light'; 97 | font-style: normal; 98 | font-weight: 300; 99 | src: local('Source Sans Pro Light'), local('SourceSansPro-Light'), url(http://themes.googleusercontent.com/static/fonts/sourcesanspro/v5/toadOcfmlt9b38dHJxOBGMVNtom4QlEDNJaqqqzqdSs.woff) format('woff'); 100 | } 101 | @font-face { 102 | font-family: 'SourceSansPro-Bold'; 103 | font-style: normal; 104 | font-weight: 700; 105 | src: local('Source Sans Pro Bold'), local('SourceSansPro-Bold'), url(http://themes.googleusercontent.com/static/fonts/sourcesanspro/v5/toadOcfmlt9b38dHJxOBGIqjGYJUyOXcBwUQbRaNH6c.woff) format('woff'); 106 | } 107 | 108 | .tipsy { font-family:"Source Sans Pro"; font-size: 10px; position: absolute; padding: 5px; z-index: 100000; } 109 | .tipsy-inner { background-color: #000; color: #FFF; max-width: 200px; padding: 0; text-align: left; padding:10px;} 110 | /* Rounded corners */ 111 | .tipsy-inner { border-radius: 3px; -moz-border-radius: 3px; -webkit-border-radius: 3px; } 112 | 113 | /* Uncomment for shadow */ 114 | /*.tipsy-inner { box-shadow: 0 0 5px #000000; -webkit-box-shadow: 0 0 5px #000000; -moz-box-shadow: 0 0 5px #000000; }*/ 115 | 116 | .tipsy-arrow { position: absolute; width: 0; height: 0; line-height: 0; border: 5px dashed #000; } 117 | 118 | /* Rules to colour arrows */ 119 | .tipsy-arrow-n { border-bottom-color: #000; } 120 | .tipsy-arrow-s { border-top-color: #000; } 121 | .tipsy-arrow-e { border-left-color: #FFF; } 122 | .tipsy-arrow-w { border-right-color: #000; } 123 | 124 | .tipsy-n .tipsy-arrow { top: 0px; left: 50%; margin-left: -5px; border-bottom-style: solid; border-top: none; border-left-color: transparent; border-right-color: transparent; } 125 | .tipsy-nw .tipsy-arrow { top: 0; left: 10px; border-bottom-style: solid; border-top: none; border-left-color: transparent; border-right-color: transparent;} 126 | .tipsy-ne .tipsy-arrow { top: 0; right: 10px; border-bottom-style: solid; border-top: none; border-left-color: transparent; border-right-color: transparent;} 127 | .tipsy-s .tipsy-arrow { bottom: 0; left: 50%; margin-left: -5px; border-top-style: solid; border-bottom: none; border-left-color: transparent; border-right-color: transparent; } 128 | .tipsy-sw .tipsy-arrow { bottom: 0; left: 10px; border-top-style: solid; border-bottom: none; border-left-color: transparent; border-right-color: transparent; } 129 | .tipsy-se .tipsy-arrow { bottom: 0; right: 10px; border-top-style: solid; border-bottom: none; border-left-color: transparent; border-right-color: transparent; } 130 | .tipsy-e .tipsy-arrow { right: 0; top: 50%; margin-top: -5px; border-left-style: solid; border-right: none; border-top-color: transparent; border-bottom-color: transparent; } 131 | .tipsy-w .tipsy-arrow { left: 0; top: 50%; margin-top: -5px; border-right-style: solid; border-left: none; border-top-color: transparent; border-bottom-color: transparent; } 132 | 133 | 134 | .gravity { width: 100%; margin: 5px 0; border-spacing: 5px; } 135 | .gravity td { text-align: center; vertical-align: middle; padding: 5px 0; background-color: #d0d0d0; width: 33%; } 136 | .gravity a { } 137 | .gravity a:hover { color: #505050; background: none; } 138 | -------------------------------------------------------------------------------- /AssetDiscoveryTree.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Generates an Asset List using a pre-existing filter file. 4 | #Based on a combination of works from Applied Network Security Monitoring and http://www.sei.cmu.edu/reports/12tr006.pdf 5 | 6 | #Usage 7 | # Obtain a filter file with a large amount of network data, representative of all hosts on your network 8 | # $ rwfilter --start-date=2014/05/13 --proto=0- --type=all --pass=sample.rw 9 | 10 | # Run AssetDiscoveryTree.sh against the file and send to an output html file 11 | # $ ./AssetDiscoveryTree.sh /home/jason/sample.rw > /home/jason/assetlist.html 12 | 13 | # By default, you will be generating data for servers making up greater than 1 percent of all "server" traffic for a given service. 14 | # If instead you want a static number threshold, you can specify --count=50 to display the top 50 talkers for a service. 15 | # $ ./AssetDiscoveryTree.sh /home/jason/sample.rw --count=50 > /home/jason/assetlistTOP50.html 16 | 17 | # If instead you want to threshold by byte size, you can specify --threshold=75000000 to display the services with greater than 75000000 bytes outbound. 18 | # $ ./AssetDiscoveryTree.sh /home/jason/sample.rw --threshold=75000000 > /home/jason/assetlistBYTETHRESHOLD.html 19 | 20 | limitvalue=$2 21 | if [ -z "${limitvalue}" ]; then 22 | limitvalue="--percentage=1" 23 | fi 24 | 25 | echo '{ "name": "Assets","children": [ ' > flare.temp.json 26 | 27 | rwfilter $1 --type=outweb --sport=80,443,8080 --protocol=6 --packets=4- --ack-flag=1 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild > web_servers.set 28 | rwfilter $1 --type=outweb --sport=80,443,8080 --protocol=6 --packets=4- --ack-flag=1 --sipset=web_servers.set --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "Web Servers",\n"children": [/' | sed '$s/$/\n\]},/' >> flare.temp.json 29 | 30 | rwfilter $1 --type=out --sport=25,465,110,995,143,993 --protocol=6 --packets=4- --ack-flag=1 --pass=stdout|rwset --sip-file=smtpservers.set 31 | rwfilter $1 --type=out --sport=25,465,110,995,143,993 --sipset=smtpservers.set --protocol=6 --packets=4- --ack-flag=1 --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "SMTP Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 32 | 33 | rwfilter $1 --type=out --sport=53 --protocol=17 --pass=stdout|rwstats --fields=sip $limitvalue --packets --no-titles|cut -f 1 -d "|"| rwsetbuild > dnsservers.set 34 | rwfilter $1 --type=out --sport=53 --protocol=17 --sipset=dnsservers.set --pass=stdout | rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "DNS Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 35 | 36 | rwfilter $1 --type=out --protocol=47,50,51 --pass=stdout|rwuniq --fields=sip --no-titles|cut -f 1 -d "|" |rwsetbuild > vpn.set 37 | rwfilter $1 --type=out --sipset=vpn.set --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "VPN Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 38 | 39 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sport=21 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild > ftpservers.set 40 | rwfilter $1 --type=out --sipset=ftpservers.set --sport=20 --flags-initial=S/SAFR --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "FTP Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 41 | 42 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sport=22 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild>sshservers.set 43 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sipset=sshservers.set --sport=22 --pass=stdout | rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "SSH Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 44 | 45 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sport=23 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild > telnetservers.set 46 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sipset=telnetservers.set --sport=23 --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "Telnet Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 47 | 48 | cat flare.temp.json | sed '$s/,$/\n/' | sed '$s/$/\]\}/' | tr '\n' ' ' | sed "s/^/var myjson = '/"| sed "s/$/';/" > assets.json 49 | 50 | sed '/dataplaceholder/{ 51 | s/dataplaceholder//g 52 | r assets.json 53 | }' d3chart/treeasset.html 54 | 55 | rm *.set 56 | rm flare.temp.json 57 | rm assets.json 58 | -------------------------------------------------------------------------------- /scripts/AssetDiscoveryTree.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Generates an Asset List using a pre-existing filter file. 4 | #Based on a combination of works from Applied Network Security Monitoring and http://www.sei.cmu.edu/reports/12tr006.pdf 5 | 6 | #Usage 7 | # Obtain a filter file with a large amount of network data, representative of all hosts on your network 8 | # $ rwfilter --start-date=2014/05/13 --proto=0- --type=all --pass=sample.rw 9 | 10 | # Run AssetDiscoveryTree.sh against the file and send to an output html file 11 | # $ ./AssetDiscoveryTree.sh /home/jason/sample.rw > /home/jason/assetlist.html 12 | 13 | # By default, you will be generating data for servers making up greater than 1 percent of all "server" traffic for a given service. 14 | # If instead you want a static number threshold, you can specify --count=50 to display the top 50 talkers for a service. 15 | # $ ./AssetDiscoveryTree.sh /home/jason/sample.rw --count=50 > /home/jason/assetlistTOP50.html 16 | 17 | # If instead you want to threshold by byte size, you can specify --threshold=75000000 to display the services with greater than 75000000 bytes outbound. 18 | # $ ./AssetDiscoveryTree.sh /home/jason/sample.rw --threshold=75000000 > /home/jason/assetlistBYTETHRESHOLD.html 19 | 20 | limitvalue=$2 21 | if [ -z "${limitvalue}" ]; then 22 | limitvalue="--percentage=1" 23 | fi 24 | 25 | echo '{ "name": "Assets","children": [ ' > flare.temp.json 26 | 27 | rwfilter $1 --type=outweb --sport=80,443,8080 --protocol=6 --packets=4- --ack-flag=1 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild > web_servers.set 28 | rwfilter $1 --type=outweb --sport=80,443,8080 --protocol=6 --packets=4- --ack-flag=1 --sipset=web_servers.set --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "Web Servers",\n"children": [/' | sed '$s/$/\n\]},/' >> flare.temp.json 29 | 30 | rwfilter $1 --type=out --sport=25,465,110,995,143,993 --protocol=6 --packets=4- --ack-flag=1 --pass=stdout|rwset --sip-file=smtpservers.set 31 | rwfilter $1 --type=out --sport=25,465,110,995,143,993 --sipset=smtpservers.set --protocol=6 --packets=4- --ack-flag=1 --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "SMTP Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 32 | 33 | rwfilter $1 --type=out --sport=53 --protocol=17 --pass=stdout|rwstats --fields=sip $limitvalue --packets --no-titles|cut -f 1 -d "|"| rwsetbuild > dnsservers.set 34 | rwfilter $1 --type=out --sport=53 --protocol=17 --sipset=dnsservers.set --pass=stdout | rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "DNS Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 35 | 36 | rwfilter $1 --type=out --protocol=47,50,51 --pass=stdout|rwuniq --fields=sip --no-titles|cut -f 1 -d "|" |rwsetbuild > vpn.set 37 | rwfilter $1 --type=out --sipset=vpn.set --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "VPN Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 38 | 39 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sport=21 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild > ftpservers.set 40 | rwfilter $1 --type=out --sipset=ftpservers.set --sport=20 --flags-initial=S/SAFR --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "FTP Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 41 | 42 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sport=22 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild>sshservers.set 43 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sipset=sshservers.set --sport=22 --pass=stdout | rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "SSH Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 44 | 45 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sport=23 --pass=stdout|rwstats --fields=sip $limitvalue --bytes --no-titles|cut -f 1 -d "|"|rwsetbuild > telnetservers.set 46 | rwfilter $1 --type=out --protocol=6 --packets=4- --ack-flag=1 --sipset=telnetservers.set --sport=23 --pass=stdout|rwuniq --fields=sip --bytes --sort-output --no-titles --delimited=,|sed 's/^$[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$,$[0-9]\{1,\}$$/\t{"name": "\1", "size": \2},/g' |sed '$s/,$//g' | sed '1s/^/\{"name": "Telnet Servers",\n"children": [/' | sed '$s/$/\n\]},/' >>flare.temp.json 47 | 48 | cat flare.temp.json | sed '$s/,$/\n/' | sed '$s/$/\]\}/' | tr '\n' ' ' | sed "s/^/var myjson = '/"| sed "s/$/';/" > assets.json 49 | 50 | sed '/dataplaceholder/{ 51 | s/dataplaceholder//g 52 | r assets.json 53 | }' d3chart/treeasset.html 54 | 55 | rm *.set 56 | rm flare.temp.json 57 | rm assets.json 58 | -------------------------------------------------------------------------------- /d3chart/treeasset.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 125 | 126 | 127 |

128 | 132 |

133 | 275 | 276 | 277 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FlowPlotter 2 | =========== 3 | Jason Smith 4 | 5 | 6 | Generates visualizations from the output of flow tools such as SiLK. 7 | 8 | --Prerequisites-- 9 | 10 | Installed SiLK tools with access to a SiLK data set (https://tools.netsa.cert.org/silk/). 11 | 12 | --Usage-- 13 | 14 | rwfilter [filter] | flowplotter.sh [charttype] [independent variable] [dependent variable] 15 | 16 | Currently you must run a SiLK rwfilter command and pipe it to flowplotter.sh and specify various options as arguments. 17 | The following chart types are currently functional 18 | 19 | 20 | #Google Charts 21 | ================== 22 | geomap 23 | - independent variable = Must specify an rwstats compatible field for country type (scc or dcc). 24 | - dependent variable = Must specify an rwstats compatible value (Records, Packets, Bytes, sIP-Distinct, dIP-Distinct, or Distinct:[field]) 25 | 26 | linechart 27 | - independent variable = Must specify a bin-size that the dependent variable will be calculated by. For example, if you want "Records per Minute", this variable will be 60. 28 | - dependent variable = Must specify an rwuniq compatible value (Records, Packets, Bytes, sIP-Distinct, dIP-Distinct, or Distinct:[field]). 29 | 30 | treemap 31 | - independent variable = Must specify an rwstats compatible field. 32 | - dependent variable = Must specify an rwstats compatible value (Records, Packets, Bytes, sIP-Distinct, dIP-Distinct, or Distinct:[field]) 33 | 34 | timeline 35 | - independent variable = Must specify an rwcut compatible field. 36 | - dependent variable = Must specify an rwcut compatible field. 37 | 38 | piechart 39 | - independent variable = Must specify an rwstats compatible field. 40 | - dependent variable = Must specify an rwstats compatible value (Records, Packets, Bytes, sIP-Distinct, dIP-Distinct, or Distinct:[field]) 41 | 42 | barchart 43 | - independent variable = Must specify an rwstats compatible field. 44 | - dependent variable = Must specify an rwstats compatible value (Records, Packets, Bytes, sIP-Distinct, dIP-Distinct, or Distinct:[field]) 45 | 46 | columnchart 47 | - independent variable = Must specify an rwstats compatible field. 48 | - dependent variable = Must specify an rwstats compatible value (Records, Packets, Bytes, sIP-Distinct, dIP-Distinct, or Distinct:[field]) 49 | 50 | bubblechart 51 | - Displays a bytes:packets:records ratio bubblechart for the top 20 [independent variable] for a given filter. 52 | - independent variable = Must specify an rwstats compatible field. 53 | 54 | =========================================================================================================================================== 55 | 56 | 57 | D3 Charts 58 | ================================================================================= 59 | forceopacity 60 | - Displays a force directed link graph based in d3. Reads from an autogenerated CSV file. 61 | - Requires 4 variables {source target value nodelimit}. 62 | - source = anything from --fields in rwstats 63 | - target = anything from --fields in rwstats 64 | - value = anything from --value in rwstats 65 | - nodelimit = anything from --count in rwstats 66 | - Click and drag a node to "stick it" 67 | - Hold shiftKey and click a node to turn it red 68 | - Hold altKey and click a node to turn it green 69 | 70 | =========================================== 71 | AssetDiscovery 72 | 73 | - Creates and Asset List based on SiLK data 74 | 75 | Usage 76 | Use large data sets instead of focused data. 77 | Due to the size of the datasets, you might be better of generating a sample file first. 78 | 79 | Obtain a filter file with a large amount of network data, representative of all hosts on your network 80 | $ rwfilter --start-date=2014/02/06 --end-date=2014/02/08 --proto=0- --type=all --pass=sample.rw 81 | $ cat sample.rw | ./flowplotter.sh assetdiscovery > assetlist.html 82 | 83 | Alternatively you can pipe directly to flowplotter as usual. 84 | $ rwfilter --start-date=2014/02/06 --end-date=2014/02/08 --proto=0- --type=all --pass=stdout | ./flowplotter.sh assetdiscovery > assetlist.html 85 | 86 | Also allows for custom thresholding using --count=50 and --threshold=3450012. 87 | Defaults to --percentage=1 if no option is given. See rwstats for more detail on those options. 88 | $ rwfilter --start-date=2014/02/06 --end-date=2014/02/08 --proto=0- --type=all --pass=stdout | ./flowplotter.sh assetdiscovery --count=50 > assetlist.html 89 | 90 | inspired by http://mbostock.github.io/d3/talk/20111018/tree.html 91 | 92 | 93 | --Google Chart Examples-- 94 | ========================= 95 | 96 | Generate a geomap of bytes to from all traffic to destination country codes - 97 | rwfilter --start-date=2013/12/27 --proto=0- --type=all --pass=stdout | ./flowplotter.sh geomap dcc bytes > geomap.html 98 | 99 | Generate a linechart for all traffic displaying the amount of bytes,records,packets, and distinct sip,dip, and dcc values. per 60 second bins - 100 | rwfilter --start-date=2013/12/27 --proto=0- --type=all --pass=stdout | ./flowplotter.sh linechart 60 distinct:sip,distinct:dip,distinct:dcc,records,bytes,packets > linechart.html 101 | 102 | Generate a treemap that shows the destination IP addresses (NOT in the 192.168.1.0/24 range) that exhibited the most records consisting of highport-highport communications - 103 | rwfilter --start-date=2013/12/27 --sport=1025- --dport=1025- --not-daddress=192.168.1.0/24 --proto=0- --type=all --pass=stdout | ./flowplotter.sh treemap dip records > treemap.html 104 | 105 | Generate a timeline showing devices communicating with non-local and non-US hosts 106 | rwfilter --start-date=2013/12/27 --proto=0- --type=out,outweb --dcc=us,-- --fail=stdout | ./flowplotter.sh timeline sip dip > timeline.html 107 | 108 | Generate a piechart that shows the destination IP addresses (NOT in the 192.168.1.0/24 range) that exhibited the most records consisting of highport-highport communications - 109 | rwfilter --start-date=2013/12/27 --sport=1025- --dport=1025- --not-daddress=192.168.1.0/24 --proto=0- --type=all --pass=stdout | ./flowplotter.sh piechart dip bytes > piechart.html 110 | 111 | Generate a barchart that shows the destination IP addresses (NOT in the 192.168.1.0/24 range) that exhibited the most records consisting of highport-highport communications - 112 | rwfilter --start-date=2013/12/27 --sport=1025- --dport=1025- --not-daddress=192.168.1.0/24 --proto=0- --type=all --pass=stdout | ./flowplotter.sh barchart dip bytes > barchart.html 113 | 114 | Generate a columnchart that shows the destination IP addresses (NOT in the 192.168.1.0/24 range) that exhibited the most records consisting of highport-highport communications - 115 | rwfilter --start-date=2013/12/27 --sport=1025- --dport=1025- --not-daddress=192.168.1.0/24 --proto=0- --type=all --pass=stdout | ./flowplotter.sh columnchart dip bytes > columnchart.html 116 | 117 | Generate a bubblechart that shows the the top 20 destination country codes sorted by a bytes:records:packet ratio - 118 | rwfilter --start=2014/02/01 --end-date=2014/02/05 --proto=0- --type=all --pass=stdout | ./flowplotter.sh bubblechart dcc > test.html 119 | 120 | 121 | --D3 Chart Examples-- 122 | ========================= 123 | 124 | Generate a force-directed graph showing two way relationships between IP addresses from rwstats, showing the top 100 sip,dip pairs sorted by the highest distinct dport numbers to each. 125 | rwfilter --start-date=2014/02/06 --proto=0- --type=all --pass=stdout | ./flowplotter.sh forceopacity sip dip distinct:dport 100 > forcetest.html 126 | 127 | Generate an asset tree based only on data provided. Best to provide well rounded data as seen below. Defaults to assets exhibiting at least 1 percent of the total service traffic. 128 | rwfilter --start-date=2014/02/06 --end-date=2014/02/08 --proto=0- --type=all --pass=stdout | ./flowplotter.sh assetdiscovery > assetlist.html 129 | -------------------------------------------------------------------------------- /d3chart/forceopacity.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

dataplaceholder
  5 |

6 | 94 | 95 | 96 |

97 | 101 | 110 |

111 |

112 | 113 | 311 | 312 | 313 | 314 | 315 | -------------------------------------------------------------------------------- /FlowPlotterD3/templates/jquery.tipsy.js: -------------------------------------------------------------------------------- 1 | // tipsy, facebook style tooltips for jquery 2 | // version 1.0.0a 3 | // (c) 2008-2010 jason frame [jason@onehackoranother.com] 4 | // released under the MIT license 5 | 6 | (function($) { 7 | 8 | function maybeCall(thing, ctx) { 9 | return (typeof thing == 'function') ? (thing.call(ctx)) : thing; 10 | } 11 | 12 | // CAUTION the current implementation does not allow for tipsied elements to stay out of DOM (in between events) 13 | // i.e. don't remove, store, then re-insert tipsied elements (and why would you want to do that anyway?) 14 | var garbageCollect = (function() { 15 | var currentInterval; 16 | var to = null; 17 | var tipsies = []; 18 | 19 | function _do() { 20 | for (var i = 0; i < tipsies.length;) { 21 | var t = tipsies[i]; 22 | // FIXME? the 2nd (non-paranoid) check is from the link below, it should be replaced if a better way is found 23 | // http://stackoverflow.com/questions/4040715/check-if-cached-jquery-object-is-still-in-dom 24 | if (t.options.gcInterval === 0 || t.$element.closest('body').length === 0) { 25 | t.hoverState = 'out'; 26 | t.hide(); 27 | tipsies.splice(i,1); 28 | } else { 29 | i++; 30 | } 31 | } 32 | } 33 | function _loop() { 34 | to = setTimeout(function() { _do(); _loop(); }, currentInterval); 35 | } 36 | 37 | return function(t) { 38 | if (t.options.gcInterval === 0) return; 39 | 40 | if (to && t.options.gcInterval < currentInterval) { 41 | clearTimeout(to); to = null; 42 | currentInterval = t.options.gcInterval; 43 | } 44 | tipsies.push(t); 45 | if (!to) _loop(); 46 | }; 47 | })(); 48 | 49 | function Tipsy(element, options) { 50 | this.$element = $(element); 51 | this.options = options; 52 | this.enabled = true; 53 | this.fixTitle(); 54 | garbageCollect(this); 55 | } 56 | 57 | 58 | Tipsy.prototype = { 59 | show: function() { 60 | var title = this.getTitle(); 61 | if (title && this.enabled) { 62 | var $tip = this.tip(); 63 | 64 | $tip.find('.tipsy-inner')[this.options.html ? 'html' : 'text'](title); 65 | $tip[0].className = 'tipsy'; // reset classname in case of dynamic gravity 66 | $tip.remove().css({top: 0, left: 0, visibility: 'hidden', display: 'block'}).prependTo(document.body); 67 | 68 | var pos = $.extend({}, this.$element.offset(), { 69 | width: this.$element[0].offsetWidth || 0, 70 | height: this.$element[0].offsetHeight || 0 71 | }); 72 | 73 | if (typeof this.$element[0].nearestViewportElement == 'object') { 74 | // SVG 75 | var el = this.$element[0]; 76 | var rect = el.getBoundingClientRect(); 77 | pos.width = rect.width; 78 | pos.height = rect.height; 79 | } 80 | 81 | 82 | var actualWidth = $tip[0].offsetWidth, 83 | actualHeight = $tip[0].offsetHeight, 84 | gravity = maybeCall(this.options.gravity, this.$element[0]); 85 | 86 | var tp; 87 | switch (gravity.charAt(0)) { 88 | case 'n': 89 | tp = {top: pos.top + pos.height + this.options.offset, left: pos.left + pos.width / 2 - actualWidth / 2}; 90 | break; 91 | case 's': 92 | tp = {top: pos.top - actualHeight - this.options.offset, left: pos.left + pos.width / 2 - actualWidth / 2}; 93 | break; 94 | case 'e': 95 | tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth - this.options.offset}; 96 | break; 97 | case 'w': 98 | tp = {top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width + this.options.offset}; 99 | break; 100 | } 101 | 102 | if (gravity.length == 2) { 103 | if (gravity.charAt(1) == 'w') { 104 | tp.left = pos.left + pos.width / 2 - 15; 105 | } else { 106 | tp.left = pos.left + pos.width / 2 - actualWidth + 15; 107 | } 108 | } 109 | 110 | $tip.css(tp).addClass('tipsy-' + gravity); 111 | $tip.find('.tipsy-arrow')[0].className = 'tipsy-arrow tipsy-arrow-' + gravity.charAt(0); 112 | if (this.options.className) { 113 | $tip.addClass(maybeCall(this.options.className, this.$element[0])); 114 | } 115 | 116 | if (this.options.fade) { 117 | $tip.stop().css({opacity: 0, display: 'block', visibility: 'visible'}).animate({opacity: this.options.opacity}); 118 | } else { 119 | $tip.css({visibility: 'visible', opacity: this.options.opacity}); 120 | } 121 | 122 | var t = this; 123 | var set_hovered = function(set_hover){ 124 | return function(){ 125 | t.$tip.stop(); 126 | t.tipHovered = set_hover; 127 | if (!set_hover){ 128 | if (t.options.delayOut === 0 && t.options.trigger != 'manual') { 129 | t.hide(); 130 | } else { 131 | setTimeout(function() { 132 | if (t.hoverState == 'out') t.hide(); }, t.options.delayOut); 133 | } 134 | } 135 | }; 136 | }; 137 | $tip.hover(set_hovered(true), set_hovered(false)); 138 | } 139 | }, 140 | 141 | hide: function() { 142 | if (this.options.fade) { 143 | this.tip().stop().fadeOut(function() { $(this).remove(); }); 144 | } else { 145 | this.tip().remove(); 146 | } 147 | }, 148 | 149 | fixTitle: function() { 150 | var $e = this.$element; 151 | 152 | if ($e.attr('title') || typeof($e.attr('original-title')) != 'string') { 153 | $e.attr('original-title', $e.attr('title') || '').removeAttr('title'); 154 | } 155 | if (typeof $e.context.nearestViewportElement == 'object'){ 156 | if ($e.children('title').length){ 157 | $e.append('' + ($e.children('title').text() || '') + '') 158 | .children('title').remove(); 159 | } 160 | } 161 | }, 162 | 163 | getTitle: function() { 164 | 165 | var title, $e = this.$element, o = this.options; 166 | this.fixTitle(); 167 | 168 | if (typeof o.title == 'string') { 169 | var title_name = o.title == 'title' ? 'original-title' : o.title; 170 | if ($e.children(title_name).length){ 171 | title = $e.children(title_name).html(); 172 | } else{ 173 | title = $e.attr(title_name); 174 | if (typeof title == 'undefined') title = '' 175 | } 176 | 177 | } else if (typeof o.title == 'function') { 178 | title = o.title.call($e[0]); 179 | } 180 | title = ('' + title).replace(/(^\s*|\s*$)/, ""); 181 | return title || o.fallback; 182 | }, 183 | 184 | tip: function() { 185 | if (!this.$tip) { 186 | this.$tip = $('

').html('

'); 187 | } 188 | return this.$tip; 189 | }, 190 | 191 | validate: function() { 192 | if (!this.$element[0].parentNode) { 193 | this.hide(); 194 | this.$element = null; 195 | this.options = null; 196 | } 197 | }, 198 | 199 | enable: function() { this.enabled = true; }, 200 | disable: function() { this.enabled = false; }, 201 | toggleEnabled: function() { this.enabled = !this.enabled; } 202 | }; 203 | 204 | $.fn.tipsy = function(options) { 205 | 206 | if (options === true) { 207 | return this.data('tipsy'); 208 | } else if (typeof options == 'string') { 209 | $(this).each(function(i,el){ 210 | if ($(el).data('tipsy')) { 211 | tipsy = $(el).data('tipsy') 212 | tipsy[options](); 213 | } 214 | }); 215 | return this; 216 | } 217 | 218 | options = $.extend({}, $.fn.tipsy.defaults, options); 219 | 220 | if (options.hoverlock && options.delayOut === 0) { 221 | options.delayOut = 100; 222 | } 223 | 224 | function get(ele) { 225 | var tipsy = $.data(ele, 'tipsy'); 226 | if (!tipsy) { 227 | tipsy = new Tipsy(ele, $.fn.tipsy.elementOptions(ele, options)); 228 | $.data(ele, 'tipsy', tipsy); 229 | } 230 | return tipsy; 231 | } 232 | 233 | function enter() { 234 | var tipsy = get(this); 235 | tipsy.hoverState = 'in'; 236 | if (options.delayIn === 0) { 237 | tipsy.show(); 238 | } else { 239 | tipsy.fixTitle(); 240 | setTimeout(function() { if (tipsy.hoverState == 'in') tipsy.show(); }, options.delayIn); 241 | } 242 | } 243 | 244 | function leave() { 245 | var tipsy = get(this); 246 | tipsy.hoverState = 'out'; 247 | if (options.delayOut === 0) { 248 | tipsy.hide(); 249 | } else { 250 | var to = function() { 251 | if (!tipsy.tipHovered || !options.hoverlock){ 252 | if (tipsy.hoverState == 'out') tipsy.hide(); 253 | } 254 | }; 255 | setTimeout(to, options.delayOut); 256 | } 257 | } 258 | 259 | if (!options.live) this.each(function() { get(this); }); 260 | 261 | if (options.trigger != 'manual') { 262 | var binder = options.live ? 'live' : 'bind', 263 | eventIn = options.trigger == 'hover' ? 'mouseenter' : 'focus', 264 | eventOut = options.trigger == 'hover' ? 'mouseleave' : 'blur'; 265 | this[binder](eventIn, enter)[binder](eventOut, leave); 266 | } 267 | 268 | return this; 269 | 270 | }; 271 | 272 | $.fn.tipsy.defaults = { 273 | className: null, 274 | delayIn: 0, 275 | delayOut: 0, 276 | fade: false, 277 | fallback: '', 278 | gcInterval: 0, 279 | gravity: 'n', 280 | html: false, 281 | live: false, 282 | offset: 0, 283 | opacity: 0.8, 284 | title: 'title', 285 | trigger: 'hover', 286 | hoverlock: false 287 | }; 288 | 289 | // Overwrite this method to provide options on a per-element basis. 290 | // For example, you could store the gravity in a 'tipsy-gravity' attribute: 291 | // return $.extend({}, options, {gravity: $(ele).attr('tipsy-gravity') || 'n' }); 292 | // (remember - do not modify 'options' in place!) 293 | $.fn.tipsy.elementOptions = function(ele, options) { 294 | return $.metadata ? $.extend({}, options, $(ele).metadata()) : options; 295 | }; 296 | 297 | $.fn.tipsy.autoNS = function() { 298 | return $(this).offset().top > ($(document).scrollTop() + $(window).height() / 2) ? 's' : 'n'; 299 | }; 300 | 301 | $.fn.tipsy.autoWE = function() { 302 | return $(this).offset().left > ($(document).scrollLeft() + $(window).width() / 2) ? 'e' : 'w'; 303 | }; 304 | 305 | /** 306 | * yields a closure of the supplied parameters, producing a function that takes 307 | * no arguments and is suitable for use as an autogravity function like so: 308 | * 309 | * @param margin (int) - distance from the viewable region edge that an 310 | * element should be before setting its tooltip's gravity to be away 311 | * from that edge. 312 | * @param prefer (string, e.g. 'n', 'sw', 'w') - the direction to prefer 313 | * if there are no viewable region edges effecting the tooltip's 314 | * gravity. It will try to vary from this minimally, for example, 315 | * if 'sw' is preferred and an element is near the right viewable 316 | * region edge, but not the top edge, it will set the gravity for 317 | * that element's tooltip to be 'se', preserving the southern 318 | * component. 319 | */ 320 | $.fn.tipsy.autoBounds = function(margin, prefer) { 321 | return function() { 322 | var dir = {ns: prefer[0], ew: (prefer.length > 1 ? prefer[1] : false)}, 323 | boundTop = $(document).scrollTop() + margin, 324 | boundLeft = $(document).scrollLeft() + margin, 325 | $this = $(this); 326 | 327 | if ($this.offset().top < boundTop) dir.ns = 'n'; 328 | if ($this.offset().left < boundLeft) dir.ew = 'w'; 329 | if ($(window).width() + $(document).scrollLeft() - $this.offset().left < margin) dir.ew = 'e'; 330 | if ($(window).height() + $(document).scrollTop() - $this.offset().top < margin) dir.ns = 's'; 331 | 332 | return dir.ns + (dir.ew ? dir.ew : ''); 333 | }; 334 | }; 335 | })(jQuery); 336 | -------------------------------------------------------------------------------- /flowplotter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #FlowPlotter is a script that allows for the "easy" integration of SiLK results into various Google Visualization Chart APIs. 4 | 5 | #GEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAP 6 | geomap () { 7 | #Variable Creation 8 | #independent variable is a string 9 | #value is a string 10 | title="$1 by $2" 11 | independent="$1" 12 | value="$2" 13 | graphtitle="$independent by $value" 14 | ###################### 15 | rwstats --top --count=275 --fields=$independent --value=$value --delimited=, |\ 16 | grep ","| grep -v -- "--"| grep -v "a1" | grep -v "a2" | grep -v "us"| grep -v "o1" | cut -d "," -f1,2 |\ 17 | sed "1 s/$[A-Za-z]\{1,20\}$,$[A-Za-z]\{1,20\}$/['\1', '\2'],/g"|sed "s/$[a-z]\{2\}$,$[0-9]\{1,50\}$/['\1', \2],/g"|sed '$s/,$//' > temp.test 18 | 19 | sed '/dataplaceholder/{ 20 | s/dataplaceholder//g 21 | r temp.test 22 | }' googlechart/geomap.html 23 | rm temp.test 24 | } 25 | #GEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAP 26 | 27 | 28 | 29 | #linechartlinechartlinechartlinechartlinechartlinechartlinechartlinechartlinechartlinechartlinechart 30 | linechart () { 31 | #Variable Creation 32 | #independent variable is a string 33 | #value is an string 34 | title="$1 by $2" 35 | resolution="$1" 36 | value="$2" 37 | number_of_values=$(echo $(( $(grep -o ',' <<<"$value" | grep -c .) + 2))) 38 | valuelist=$(seq $number_of_values | tr '\n' ',' | sed 's/,$//g') 39 | 40 | if [ "$1" -eq "$1" ] 2>/dev/null; then 41 | resolution="$1" 42 | else 43 | echo "The resolution you selected is invalid. Your first argument to linechart should be an integer value in seconds" 44 | exit 45 | fi 46 | 47 | graphtitle="$valuename per $resolution second bins" 48 | 49 | ########################## 50 | #rwcount --bin-size=$resolution --delimited=, | cut -d "," -f1,$value|\ 51 | rwuniq --fields=stime --bin-time=$resolution --values=$value --delimited=, --no-titles --sort-output | cut -d "," -f$valuelist |\ 52 | sed "s/^$.\{10\}T.\{8\}$\,$.*$/['\1',\2],/g" | sed '$s/,$//' | sed "1 i\[stime,$value]" | sed "1 s/\[/\['/" | sed "1 s/,/','/g" | sed "1 s/\]/'\],/" > temp.test 53 | 54 | 55 | #sed "s/$.*$,$.*$/['\1', \2],/g"|sed '$s/,$//'| sed "s/, $[A-Za-z].*$],/, '\1'],/g" > temp.test 56 | 57 | sed '/dataplaceholder/{ 58 | s/dataplaceholder//g 59 | r temp.test 60 | }' googlechart/linechart.html | sed "s/titleplaceholder/${graphtitle}/g" 61 | 62 | rm temp.test 63 | } 64 | #linechartlinechartlinechartlinechartlinechartlinechartlinechartlinechartlinechartlinechartlinechart 65 | 66 | 67 | 68 | #treemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemap 69 | treemap () { 70 | #Variable Creation 71 | #independent variable is a string 72 | #value is a string 73 | title="$1 by $2" 74 | independent="$1" 75 | value="$2" 76 | graphtitle="$independent by $value" 77 | ##################### 78 | echo "['$independent', 'parent', '$value']," > temp.test 79 | echo "['$independent', null, 0]," >> temp.test 80 | rwstats --fields=$independent --top --count=20 --no-titles --delimited=, --value=$value |\ 81 | cut -d "," -f1,2 |\ 82 | grep ,| sed "s/$.*$,$.*$/['\1', '$independent', \2],/g"|sed '$s/,$//'| sed "s/, $[A-Za-z].*$],/, '\1'],/g" | grep "," >> temp.test 83 | 84 | sed '/dataplaceholder/{ 85 | s/dataplaceholder//g 86 | r temp.test 87 | }' googlechart/treemap.html | sed "s/titleplaceholder/${graphtitle}/g" 88 | 89 | rm temp.test 90 | } 91 | #treemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemaptreemap 92 | 93 | 94 | #timelinetimelinetimelinetimelinetimelinetimelinetimelinetimelinetimelinetimelinetimeline 95 | timeline () { 96 | #Variable Creation 97 | #independent variable is a string 98 | #dependent variable is a string 99 | title='Suspicious Server Traffic' 100 | title="$1 by $2" 101 | independent="$1" 102 | dependent="$2" 103 | if [ -z "${independent}" ]; then 104 | independent="sip" 105 | fi 106 | if [ -z "${dependent}" ]; then 107 | dependent="dip" 108 | fi 109 | graphtitle="$independent by $dependent over time" 110 | ############################## 111 | rwsort --fields=stime | rwgroup --delta-field=stime --delta-value=2 --summarize | rwsort --fields=sip | rwcut --fields=$independent,$dependent,stime,etime --delimited=, --no-titles |\ 112 | sed "s/$.*$,$.*$,$.*$,$.*$/['\1', '\2', new Date(\3), new Date (\4)],/g"|sed '$s/,$//'| \ 113 | sed "s/$[0-9]\{4\}$\/$[0-9]\{2\}$\/$[0-9]\{2\}$T$[0-9]\{2\}$:$[0-9]\{2\}$:$[0-9]\{2\}$\.[0-9]\{3\}/\1, \2, \3, \4, \5, \6/g" > temp.test 114 | 115 | sed '/dataplaceholder/{ 116 | s/dataplaceholder//g 117 | r temp.test 118 | }' googlechart/timeline.html | sed "s/titleplaceholder/${graphtitle}/g" 119 | 120 | rm temp.test 121 | } 122 | #timelinetimelinetimelinetimelinetimelinetimelinetimelinetimelinetimelinetimelinetimeline 123 | 124 | 125 | #piechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechart 126 | piechart () { 127 | #Variable Creation 128 | #independent variable is a string 129 | #value is a string 130 | title="$1 by $2" 131 | independent="$1" 132 | value="$2" 133 | graphtitle="$independent by $value" 134 | ##################### 135 | echo "['$independent', '$value']," > temp.test 136 | rwstats --fields=$independent --top --count=12 --no-titles --delimited=, --value=$value |\ 137 | cut -d "," -f1,2 |\ 138 | grep ,| sed "s/$.*$,$.*$/['\1', \2],/g"|sed '$s/,$//'| sed "s/, $[A-Za-z].*$],/, '\1'],/g" | grep "," >> temp.test 139 | 140 | sed '/dataplaceholder/{ 141 | s/dataplaceholder//g 142 | r temp.test 143 | }' googlechart/piechart.html | sed "s/titleplaceholder/${graphtitle}/g" 144 | 145 | rm temp.test 146 | } 147 | #piechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechartpiechart 148 | 149 | 150 | #barchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchart 151 | barchart () { 152 | #Variable Creation 153 | #independent variable is a string 154 | #value is an string 155 | title="$1 by $2" 156 | independent="$1" 157 | value="$2" 158 | graphtitle="$independent by $value" 159 | ##################### 160 | echo "['$independent', '$value']," > temp.test 161 | rwstats --fields=$independent --top --count=12 --no-titles --delimited=, --value=$value |\ 162 | cut -d "," -f1,2 |\ 163 | grep ,| sed "s/$.*$,$.*$/['\1', \2],/g"|sed '$s/,$//'| sed "s/, $[A-Za-z].*$],/, '\1'],/g" | grep "," >> temp.test 164 | 165 | sed '/dataplaceholder/{ 166 | s/dataplaceholder//g 167 | r temp.test 168 | }' googlechart/barchart.html | sed "s/titleplaceholder/${graphtitle}/g" 169 | 170 | rm temp.test 171 | } 172 | #barchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchartbarchart 173 | 174 | 175 | #columnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchart 176 | columnchart () { 177 | #Variable Creation 178 | #independent variable is a string 179 | #value is an string 180 | title="$1 by $2" 181 | independent="$1" 182 | value="$2" 183 | graphtitle="$independent by $value" 184 | ##################### 185 | echo "['$independent', '$value']," > temp.test 186 | rwstats --fields=$independent --top --count=12 --no-titles --delimited=, --value=$value |\ 187 | cut -d "," -f1,2 |\ 188 | grep ,| sed "s/$.*$,$.*$/['\1', \2],/g"|sed '$s/,$//'| sed "s/, $[A-Za-z].*$],/, '\1'],/g" | grep "," >> temp.test 189 | 190 | sed '/dataplaceholder/{ 191 | s/dataplaceholder//g 192 | r temp.test 193 | }' googlechart/columnchart.html | sed "s/titleplaceholder/${graphtitle}/g" 194 | 195 | rm temp.test 196 | } 197 | #columnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchartcolumnchart 198 | 199 | 200 | 201 | #tablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablechart 202 | tablechart () { 203 | #echo "['sTime', 'sIP', 'sPort', 'scc', 'dIP', 'dPort', 'dcc', 'protocol', 'initialFlags', 'flags', 'type', 'sensor', 'eTime']," > temp.test 204 | rwcut --fields=stime,sip,sport,scc,dip,dport,dcc,pro,initialF,flags,type,sen,etime --delimited=, --no-titles| head -1000 |\ 205 | perl -p -i -e "s/(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)\,(.*?)$/['\$1', '\$2', '\$3', '\$4', '\$5', '\$6', \$7, '\$8', '\$9', '\$10', '\$11', '\$12', '\$13'],/" |\ 206 | sed 's/\-\-//g'| sed '$s/,$//' >>temp.test 207 | 208 | sed '/dataplaceholder/{ 209 | s/dataplaceholder//g 210 | r temp.test 211 | }' googlechart/tablechart.html | sed "s/titleplaceholder/${graphtitle}/g" 212 | 213 | rm temp.test 214 | } 215 | #tablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablecharttablechart 216 | 217 | 218 | 219 | #bubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechart 220 | bubblechart () { 221 | #Variable Creation 222 | #independent variable is a string 223 | #value is an string 224 | title="$1 by $2" 225 | independent="$1" 226 | graphtitle="Byte:Packet:Record for top 20 $independent" 227 | ##################### 228 | #echo "['$independent', '$value']," > temp.test 229 | 230 | rwstats --fields=$independent --value=bytes,records,packets --count=20 --delimited=" " | awk '{print $1,$4,$2,$1,$3}'| grep -v ":"|sed 's/ /,/g' |\ 231 | sed "s/$[a-zA-Z]\{1,20\}$,$[a-zA-Z]\{1,20\}$,$[a-zA-Z]\{1,20\}$,$[a-zA-Z]\{1,20\}$,$[a-zA-Z]\{1,20\}$/['\1', '\2', '\3', '\4', '\5'],/g" |\ 232 | sed "s/$.\{1,20\}$,$[0-9]\{1,20\}$,$[0-9]\{1,20\}$,$.\{1,20\}$,$.\{1,20\}$/['\1', \2, \3, '\4', \5],/g" | sed '$s/,$//' >> temp.test 233 | 234 | sed '/dataplaceholder/{ 235 | s/dataplaceholder//g 236 | r temp.test 237 | }' googlechart/bubblechart.html | sed "s/titleplaceholder/${graphtitle}/g" 238 | 239 | rm temp.test 240 | } 241 | #bubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechartbubblechart 242 | 243 | 244 | 245 | #forceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacity 246 | forceopacity () { 247 | #Variable Creation 248 | #source variable is a string 249 | #target variable is a string 250 | #value is an string 251 | title="$1 by $2" 252 | source="$1" 253 | if [ -z "$source" ]; then 254 | source="sip" 255 | echo "No source variable provided. Defaulting to sip" 256 | fi 257 | target="$2" 258 | if [ -z "$target" ]; then 259 | target="dip" 260 | echo "No target variable provided. Defaulting to dip" 261 | fi 262 | value="$3" 263 | if [ -z "$value" ]; then 264 | value="bytes" 265 | echo "No value variable provided. Defaulting to bytes" 266 | fi 267 | count="$4" 268 | if [ -z "$count" ]; then 269 | count="50" 270 | echo "No count variable provided. Defaulting to 50" 271 | fi 272 | graphtitle="source-target-value" 273 | ##################### 274 | rwstats --top --count=$count --fields=$source,$target --value=$value --delimited=, --no-titles | cut -d ',' -f1,2,3 | sed '1 s/^/source,target,value\n/' >> temp.test 275 | 276 | sed '/dataplaceholder/{ 277 | s/dataplaceholder//g 278 | r temp.test 279 | }' d3chart/forceopacity.html | sed "s/titleplaceholder/${graphtitle}/g" 280 | 281 | rm temp.test 282 | } 283 | #forceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacityforceopacity 284 | 285 | 286 | 287 | #assetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscovery 288 | assetdiscovery () { 289 | #Variable Creation 290 | #value is a string 291 | 292 | rwfilter stdin --proto=0- --pass=stdout > test.rw 293 | 294 | scripts/./AssetDiscoveryTree.sh test.rw $1 295 | rm test.rw 296 | } 297 | #assetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscoveryassetdiscovery 298 | 299 | #Function Initiation 300 | 301 | if [ "$1" == "-h" ]; then 302 | cat README.md 303 | echo -e "\n" 304 | exit 0 305 | fi 306 | 307 | if [ "$1" != "geomap" -a "$1" != "linechart" -a "$1" != "treemap" -a "$1" != "timeline" -a "$1" != "piechart" -a "$1" != "barchart" -a "$1" != "columnchart" -a "$1" != "bubblechart" -a "$1" != "forceopacity" -a "$1" != "assetdiscovery" ]; then 308 | echo "" 309 | echo "ERROR: You must specify a supported chart type. " 310 | echo "" 311 | echo "The only supported chart types for flowplotter are:" 312 | echo "./flowplotter.sh geomap [field] [value]" 313 | echo "./flowplotter.sh linechart [bin size in seconds] [value]" 314 | echo "./flowplotter.sh treemap [field] [value]" 315 | echo "./flowplotter.sh timeline [field] [field]" 316 | echo "./flowplotter.sh piechart [field] [value]" 317 | echo "./flowplotter.sh barchart [field] [value]" 318 | echo "./flowplotter.sh columnchart [field] [value]" 319 | echo "./flowplotter.sh bubblechart [value]" 320 | echo "./flowplotter.sh forceopacity [field] [field] [value]" 321 | echo "./flowplotter.sh assetdiscovery [threshold]" 322 | echo "" 323 | echo "flowplotter --help" 324 | echo "./flowplotter -h" 325 | echo "" 326 | exit 0 327 | fi 328 | 329 | if [ "$1" = "geomap" ]; then 330 | geomap $2 $3 331 | fi 332 | if [ "$1" = "linechart" ]; then 333 | linechart $2 $3 334 | fi 335 | if [ "$1" = "treemap" ]; then 336 | treemap $2 $3 337 | fi 338 | if [ "$1" = "timeline" ]; then 339 | timeline $2 $3 340 | fi 341 | if [ "$1" = "piechart" ]; then 342 | piechart $2 $3 343 | fi 344 | if [ "$1" = "barchart" ]; then 345 | barchart $2 $3 346 | fi 347 | if [ "$1" = "columnchart" ]; then 348 | columnchart $2 $3 349 | fi 350 | if [ "$1" = "bubblechart" ]; then 351 | bubblechart $2 352 | fi 353 | if [ "$1" = "forceopacity" ]; then 354 | forceopacity $2 $3 $4 $5 355 | fi 356 | if [ "$1" = "assetdiscovery" ]; then 357 | assetdiscovery $2 358 | fi 359 | #if [ "$1" = "tablechart" ]; then 360 | #tablechart $2 361 | #fi 362 | -------------------------------------------------------------------------------- /FlowPlotterD3/mslinechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |

10 |

dport status

11 | 13 |

14 | 15 |

20 | 21 | 22 | 393 | 394 | 395 | -------------------------------------------------------------------------------- /FlowPlotterD3/templates/linechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |

10 |

Sensor Status

11 | 13 |

14 | 15 |

20 | 21 | 22 | 393 | 394 | 395 | -------------------------------------------------------------------------------- /FlowPlotterD3/templates/mslinechart.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |

10 |

seriesselect status

11 | 13 |

14 | 15 |

20 | 21 | 22 | 393 | 394 | 395 | -------------------------------------------------------------------------------- /FlowPlotterD3/templates/linechart.html.bak: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |

10 |

Sensor Status

11 | 13 |

14 | 15 |

21 | 22 | 23 | 394 | 395 | 396 | -------------------------------------------------------------------------------- /FlowPlotterD3/flowplotter.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #FlowPlotter is a script that allows for the "easy" integration of SiLK results into various Google Visualization Chart APIs. 4 | 5 | #GEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAP 6 | geomap () { 7 | #Variable Creation 8 | #independent variable is a string 9 | #value is a string 10 | title="$1 by $2" 11 | independent="$1" 12 | value="$2" 13 | graphtitle="$independent by $value" 14 | ###################### 15 | rwstats --top --count=275 --fields=$independent --value=$value --delimited=, |\ 16 | grep ","| grep -v -- "--"| grep -v "a1" | grep -v "a2" | grep -v "us"| grep -v "o1" | cut -d "," -f1,2 |\ 17 | sed "1 s/$[A-Za-z]\{1,20\}$,$[A-Za-z]\{1,20\}$/['\1', '\2'],/g"|sed "s/$[a-z]\{2\}$,$[0-9]\{1,50\}$/['\1', \2],/g"|sed '$s/,$//' > temp.test 18 | 19 | sed '/dataplaceholder/{ 20 | s/dataplaceholder//g 21 | r temp.test 22 | }' googlechart/geomap.html 23 | rm temp.test 24 | } 25 | #GEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAPGEOMAP 26 | 27 | 28 | #mslinechartmslinechartmslinechartmslinechartmslinechartmslinechartmslinechartmslinechartmslinechartmslinechartmslinechart 29 | mslinechart () { 30 | #Variable Creation 31 | #independent variable is a string 32 | #value is an string 33 | title="$1 in $2 second bins per $3" 34 | resolution="$1" 35 | value="$2" 36 | series="$3" 37 | number_of_values=$(echo $(( $(grep -o ',' <<<"$value" | grep -c .) + 2))) 38 | valuelist=$(seq $number_of_values | tr '\n' ',' | sed 's/,$//g') 39 | 40 | if [ "$1" -eq "$1" ] 2>/dev/null; then 41 | resolution="$1" 42 | else 43 | echo "The resolution you selected is invalid. Your first argument to mslinechart should be an integer value in seconds" 44 | exit 45 | fi 46 | 47 | graphtitle="$valuename in $resolution second bins per $series" 48 | #awk '!/date/' 49 | rwuniq --bin-time=$resolution --fields=$series,stime --values=$value --delimited=, | sed 's/sTime/date/' | sort -k2n > data/test.csv 50 | cat <(grep 'date' data/test.csv) <(grep -v 'date' data/test.csv) | sed -e '1s/$.*$/\L\1/' > data/ms.csv 51 | rm data/test.csv 52 | #optionlist=$(cat data/ms.csv | awk -F, '{print $1}' | tail -n +2 | uniq | sed 's/^/