├── .gitignore ├── .mailmap ├── Design.mkdn ├── Ideas.mkdn ├── README.mkdn ├── bin ├── dc-analyze.pl ├── dc-bit-whois.pl ├── dc-bitflip-dns-server.pl ├── dc-component-analyze.pl ├── dc-component-parse.pl ├── domain-jumble.pl └── dreamcatcher.pl ├── cpanfile ├── dist.ini ├── dreamcatcher.yml.default ├── examples ├── bitflip-server-config.yaml ├── screenshots │ ├── 0-main.png │ ├── 1-server.png │ └── 3-questions.png └── test-feathers.pl ├── lib ├── DreamCatcher.pm └── DreamCatcher │ ├── Controller.pm │ ├── Controller │ ├── Client.pm │ ├── Conversation.pm │ ├── List.pm │ ├── Main.pm │ ├── Questions.pm │ ├── Server.pm │ └── Utility.pm │ ├── Feather │ ├── anomaly │ │ ├── query.pm │ │ └── question.pm │ ├── conversation.pm │ ├── list │ │ ├── meta.pm │ │ ├── refresh.pm │ │ └── tracking.pm │ ├── query │ │ └── response.pm │ ├── store.pm │ └── zone │ │ └── discovery.pm │ ├── Feathers.pm │ ├── Helpers.pm │ ├── Packet.pm │ ├── Role │ ├── Anomaly.pm │ ├── Anomaly │ │ └── Query.pm │ ├── Cache.pm │ ├── DBH.pm │ ├── Feather.pm │ ├── Feather │ │ ├── Analysis.pm │ │ ├── Anomaly.pm │ │ └── Sniffer.pm │ ├── Logger.pm │ └── RRData.pm │ └── Types.pm ├── logging.conf ├── public ├── css │ ├── bootstrap-responsive.min.css │ ├── bootstrap.min.css │ ├── jquery.dataTables.css │ └── main.css ├── img │ ├── back_disabled.png │ ├── back_enabled.png │ ├── back_enabled_hover.png │ ├── forward_disabled.png │ ├── forward_enabled.png │ ├── forward_enabled_hover.png │ ├── glyphicons-halflings-white.png │ ├── glyphicons-halflings.png │ ├── sort_asc.png │ ├── sort_asc_disabled.png │ ├── sort_both.png │ ├── sort_desc.png │ └── sort_desc_disabled.png └── js │ ├── bootstrap-datatables.js │ ├── bootstrap.min.js │ ├── d3.v2.min.js │ ├── date.js │ ├── dreamcatcher.js │ ├── jquery.bootstrap-growl.js │ ├── jquery.dataTables.min.js │ ├── jquery.min.js │ └── sigma.js ├── script └── dream_catcher ├── sql ├── deploy_database_schema.pl └── schema │ ├── deploy.yml │ ├── install │ ├── base │ │ ├── client.sql │ │ ├── conversation.sql │ │ ├── find_or_create_conversation.sql │ │ └── server.sql │ └── plugins │ │ ├── anomaly │ │ ├── anomaly.sql │ │ └── create_anomaly_table.sql │ │ ├── list │ │ ├── list.sql │ │ ├── list_entry.sql │ │ ├── list_meta_answer.sql │ │ ├── list_meta_question.sql │ │ ├── list_tracking_client.sql │ │ ├── list_type.sql │ │ └── refresh_list_entry.sql │ │ ├── packet_store │ │ ├── add_query.sql │ │ ├── add_response.sql │ │ ├── answer.sql │ │ ├── find_or_create_answer.sql │ │ ├── find_or_create_question.sql │ │ ├── link_query_response.sql │ │ ├── link_question_answer.sql │ │ ├── meta_answer.sql │ │ ├── meta_query_response.sql │ │ ├── meta_question.sql │ │ ├── meta_question_answer.sql │ │ ├── query.sql │ │ ├── question.sql │ │ ├── response.sql │ │ └── store_cleanup.sql │ │ └── zone_discovery │ │ ├── get_zone_id.sql │ │ ├── link_zone_answer.sql │ │ ├── link_zone_question.sql │ │ ├── zone.sql │ │ ├── zone_answer.sql │ │ └── zone_question.sql │ └── upgrade │ └── 20150330 │ └── plugins │ ├── anomaly_query │ └── anomaly_query.sql │ └── anomaly_question │ └── anomaly_question.sql ├── t ├── feather-00-basic.t └── web-00-basic.t ├── templates ├── conversation │ └── view.html.ep ├── graph │ └── sigma.html.ep ├── layouts │ ├── blank.html.ep │ └── bootstrap.html.ep ├── list │ ├── index.html.ep │ └── view.html.ep ├── main │ └── index.html.ep ├── questions │ └── index.html.ep ├── server │ ├── index.html.ep │ ├── server_responses.html.ep │ ├── top_servers.html.ep │ └── view.html.ep ├── utility │ ├── client_server_map.html.ep │ ├── clients_asking.html.ep │ ├── form_clients_asking.html.ep │ ├── index.html.ep │ ├── reverse.html.ep │ └── reverse_form.html.ep └── zone │ └── top_zones.html.ep └── weaver.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.bak 2 | .svn 3 | inc 4 | blib 5 | log 6 | dreamcatcher.yml 7 | DreamCatcher-* 8 | MANIFEST 9 | Makefile 10 | Makefile.old 11 | .DS_Store 12 | .cvsignore 13 | pm_to_blib 14 | MYMETA.* 15 | .build 16 | *.log 17 | *.log.* 18 | *.pid 19 | *.LCK 20 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Brad Lhotsky Brad Lhotsky 2 | Brad Lhotsky Brad Lhotsky 3 | Brad Lhotsky Brad Lhotsky 4 | -------------------------------------------------------------------------------- /Design.mkdn: -------------------------------------------------------------------------------- 1 | # Overview of the DreamCatcher Architecture # 2 | 3 | DreamCatcher is a rewrite of an old DNS Monitoring project. It is based on 4 | that work and redesigned around the idea of a Native American Dream Catcher. 5 | 6 | ## Basic Components ## 7 | 8 | The DreamCatcher daemon consists of two primary object types, the first is 9 | the net, the second are feathers. The net catches packets and performs 10 | basic processing for handling by the feathers. 11 | 12 | The feathers receive the "good packets" from the net, either directly or 13 | through a trickle down mechanism from other feathers. A feather may tie 14 | itself to another feather. If that feather has not been loaded, then that 15 | feather will be shutdown. 16 | 17 | ### The Net ### 18 | 19 | The net is the network capture session. It implements a packet sniffer and 20 | packet processor which classifies and passes the packets to the first level 21 | feathers. 22 | 23 | Information passed to feathers from the net include: 24 | 25 | * *server_ip* 26 | * *server_port* 27 | * *client_ip* 28 | * *client_port* 29 | * *time* - hires time packet was captured) 30 | * *dns* - Net::DNS::Packet object 31 | 32 | ### Feathers ### 33 | 34 | Feathers serve to deliver the good packets to the user. This means that the 35 | feathers provide context, sort, filter, and store the good packets in 36 | some fashion. Feathers may provide additional information about the 37 | packets. This information is then used to generate a dependency graph which 38 | allows feathers to receive packets in the correct order. 39 | 40 | Information that feathers may implement include: 41 | 42 | * conversation 43 | * *server_id* - integer id for quick reference in data store 44 | * *client_id* - integer id for quick reference in data store 45 | * *conversation_id* - integer id denoting the unique transacation 46 | -------------------------------------------------------------------------------- /Ideas.mkdn: -------------------------------------------------------------------------------- 1 | # Passive Collection 2 | 3 | * Add a whois collector 4 | * whois when a new domain is seen 5 | * record AUTH NS's, re-whois when they change or when the DB expires 6 | 7 | * Bit squatting DNS server needs to send the info for both bitflipped *AND* 8 | squatted domain. 9 | 10 | # TODO 11 | 12 | * Port dns-monitor functions 13 | * statistics 14 | * analyzers 15 | * client acting as server 16 | * server authorized 17 | * list tracking 18 | * Front end improvements 19 | * List tracking 20 | * Search functions 21 | * Statistics 22 | * CLI 23 | * Need to provide CLI tools which implement similar functionality as Web 24 | 25 | -------------------------------------------------------------------------------- /README.mkdn: -------------------------------------------------------------------------------- 1 | # NAME 2 | 3 | DreamCatcher - DreamCatcher is a DNS Monitoring Suite 4 | 5 | # VERSION 6 | 7 | version 0.001 8 | 9 | # SYNOPSIS 10 | 11 | This is a complete DNS Monitoring Suite. It is currently in **alpha** status. 12 | 13 | A libpcap based sniffer daemon listens to DNS traffic on your network. The 14 | conversations are recorded and analyzed to provide insight. 15 | 16 | # PREREQUISISTES 17 | 18 | - **Perl** 19 | 20 | 5.14.2 or better 21 | 22 | - **PostgreSQL** 23 | 24 | 9.4 or better with the **ltree** extension 25 | 26 | # INSTALLATION 27 | 28 | Installation in the works, for now try: 29 | 30 | perl Makefile.PL 31 | make 32 | 33 | Then install the schema: 34 | 35 | cd sql 36 | ./deploy_database_schema.pl install 37 | 38 | Configure the instance: 39 | 40 | cp dreamcatcher.yml.default dreamcatcher.yml 41 | $EDITOR dreamcatcher.yml 42 | 43 | Configure logging: 44 | 45 | $EDITOR logging.conf 46 | 47 | # USAGE 48 | 49 | Once you have the database schema and the dreamcatcher.yaml configured, run the collector: 50 | 51 | sudo ./bin/dreamcatcher.pl start 52 | 53 | Now start the web application for viewing the data: 54 | 55 | morbo -v script/dream_catcher 56 | 57 | # SCREENSHOTS 58 | 59 | - [Overview Page](https://github.com/reyjrar/DreamCatcher/raw/master/examples/screenshots/0-main.png) 60 | - [Viewing a Server](https://github.com/reyjrar/DreamCatcher/raw/master/examples/screenshots/1-server.png) 61 | - [Recently Asked Questions](https://github.com/reyjrar/DreamCatcher/raw/master/examples/screenshots/3-questions.png) 62 | 63 | # AUTHOR 64 | 65 | Brad Lhotsky 66 | 67 | # COPYRIGHT AND LICENSE 68 | 69 | This software is Copyright (c) 2017 by Brad Lhotsky. 70 | 71 | This is free software, licensed under: 72 | 73 | The (three-clause) BSD License 74 | 75 | # SUPPORT 76 | 77 | ## Websites 78 | 79 | The following websites have more information about this module, and may be of help to you. As always, 80 | in addition to those websites please use your favorite search engine to discover more resources. 81 | 82 | - MetaCPAN 83 | 84 | A modern, open-source CPAN search engine, useful to view POD in HTML format. 85 | 86 | [http://metacpan.org/release/DreamCatcher](http://metacpan.org/release/DreamCatcher) 87 | 88 | - RT: CPAN's Bug Tracker 89 | 90 | The RT ( Request Tracker ) website is the default bug/issue tracking system for CPAN. 91 | 92 | [https://rt.cpan.org/Public/Dist/Display.html?Name=DreamCatcher](https://rt.cpan.org/Public/Dist/Display.html?Name=DreamCatcher) 93 | 94 | ## Source Code 95 | 96 | This module's source code is available by visiting: 97 | [https://github.com/reyjrar/DreamCatcher](https://github.com/reyjrar/DreamCatcher) 98 | -------------------------------------------------------------------------------- /bin/dc-analyze.pl: -------------------------------------------------------------------------------- 1 | #!perl 2 | # PODNAME: dc-analyze.pl 3 | use strict; 4 | use warnings; 5 | use feature 'say'; 6 | 7 | use CLI::Helpers qw(:output); 8 | use DreamCatcher::Feathers; 9 | use FindBin; 10 | use Getopt::Long::Descriptive; 11 | use Pod::Usage; 12 | use YAML (); 13 | 14 | #------------------------------------------------------------------------# 15 | # Argument Parsing 16 | my ($opt,$usage) = describe_options( 17 | "%c %o", 18 | [], 19 | [ 'feather|f=s@', "Specify a feather to run, multiple options accepted." ], 20 | [ 'period|p=i', "Number of seconds to check, default: 7200", {default=>7200} ], 21 | [ 'max|m=i', "Maximum number of records to process default: 5,000", {default=>5_000} ], 22 | [], 23 | [ 'config|c=s', "DreamCatcher Config File", { 24 | default => '/etc/dreamcatcher/main.yaml', 25 | callbacks => { exists => sub { -f shift } } 26 | }], 27 | [ 'help|h', 'print this menu and exit'], 28 | [ 'manual|m', 'print the manual'], 29 | ); 30 | #------------------------------------------------------------------------# 31 | # Display Documentation 32 | pod2usage(-exit=>0,-verbose=>2) if $opt->manual; 33 | say($usage->text) if $opt->help; 34 | 35 | #------------------------------------------------------------------------# 36 | # Configure the Feathers 37 | my $CFG = YAML::LoadFile( $opt->config ); 38 | my $Plumage = DreamCatcher::Feathers->new( 39 | Config => $CFG, 40 | Log => \&logger, 41 | ); 42 | 43 | my %A = (); 44 | foreach my $f (@{ $Plumage->chain('analysis') }) { 45 | # Configure the feather 46 | $f->check_period($opt->period); 47 | $f->batch_max($opt->max); 48 | 49 | # Make it accessible 50 | $A{$f->name} = $f; 51 | } 52 | 53 | foreach my $feather (@{ $opt->feather }) { 54 | if( exists $A{$feather} ) { 55 | debug("Running $feather."); 56 | $A{$feather}->analyze(); 57 | } 58 | else { 59 | output({color=>'yellow'}, "Unknown analysis feather '$feather'"); 60 | } 61 | } 62 | 63 | 64 | # Logging Closure 65 | { 66 | my %colors = ( 67 | debug => 'white', 68 | info => 'cyan', 69 | notice => 'cyan', 70 | warn => 'yellow', 71 | warning => 'yellow', 72 | err => 'red', 73 | error => 'red', 74 | crit => 'red', 75 | emerg => 'red', 76 | ); 77 | my %cb = ( 78 | debug => \&debug, 79 | info => sub { 80 | my $opts = ref $_[0] eq 'HASH' ? shift : {}; 81 | $opts->{level} = 2; 82 | verbose($opts,@_); 83 | }, 84 | notice => \&verbose, 85 | ); 86 | sub logger { 87 | my ($level,@msgs) = @_; 88 | 89 | my %opts = ( color => $colors{$level} ); 90 | 91 | exists $cb{$level} ? $cb{$level}->(\%opts,@msgs) 92 | : output(\%opts, @msgs); 93 | } 94 | } 95 | 96 | __END__ 97 | 98 | =head1 SYNOPSIS 99 | 100 | dc-analyze.pl 101 | 102 | Run one or more analysis feathers manually. 103 | 104 | Options: 105 | 106 | --help print help 107 | --manual print full manual 108 | --config Location of the Config file, see: L 109 | 110 | 111 | =head1 OPTIONS 112 | 113 | =over 8 114 | 115 | =item B 116 | 117 | Location of the config file 118 | 119 | =back 120 | 121 | =head1 DESCRIPTION 122 | 123 | This script is used to run one or more of the analysis feathers manually from the 124 | command line. 125 | 126 | =head1 CONFIGURATION 127 | 128 | The DreamCatcher config is stored in L format. The defaults look like this: 129 | 130 | --- 131 | time_zone: America/New_York 132 | db: 133 | dsn: dbi:Pg:host=localhost;database=dreamcatcher 134 | user: admin 135 | pass: 136 | 137 | network: 138 | nameservers: &GLOBALnameservers 139 | - 8.8.8.8 140 | - 8.8.4.4 141 | clients: &GLOBALclients 142 | - 192.168.1.0/24 143 | 144 | pcap: 145 | dev: any 146 | snaplen: 1518 147 | timeout: 100 148 | filter: (tcp or udp) and port 53 149 | promisc: 0 150 | 151 | sniffer: 152 | workers: 4 153 | 154 | analysis: 155 | disabled: 0 156 | 157 | feather: 158 | conversation: 159 | disabled: 0 160 | 161 | =cut 162 | -------------------------------------------------------------------------------- /bin/dc-bit-whois.pl: -------------------------------------------------------------------------------- 1 | #!perl 2 | # PODNAME: dc-bit-whois.pl 3 | use strict; 4 | use warnings; 5 | 6 | use CLI::Helpers qw(:output); 7 | use Getopt::Long::Descriptive; 8 | use Net::Whois::Raw; 9 | use Net::Whois::Parser; 10 | 11 | my %DEFAULT = ( 12 | cache_dir => "$ENV{HOME}/tmp", 13 | ); 14 | my ($opt,$usage) = describe_options("%c %o domain", 15 | ["This utility will find all bitflipped variations on a domain and check their availability."], 16 | ['cache-dir|cache=s', "Directory to cache whois responses, default is $DEFAULT{cache_dir}", 17 | {default=>$DEFAULT{cache_dir}}], 18 | [], 19 | ['help', "Display this help.", {shortcircuit => 1}] 20 | ); 21 | if( $opt->help || !@ARGV ) { 22 | print $usage->text; 23 | exit; 24 | } 25 | # Configure Net::Whois::Raw 26 | $Net::Whois::Raw::OMIT_MSG = 1; 27 | $Net::Whois::Raw::CHECK_FAIL = 0; 28 | $Net::Whois::Raw::CACHE_DIR = $opt->cache_dir; 29 | $Net::Whois::Raw::TIMEOUT = 2; 30 | 31 | my $domain = shift @ARGV; 32 | my @parts = split /\./, lc $domain; 33 | my %VALID = map { $_ => 1 } split '', q{ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-}; 34 | 35 | my $word = shift @parts; 36 | my %variations = map { $_ => 1 } ( lc $domain ); 37 | 38 | for my $place ( 0 .. length($word)-1 ) { 39 | my $letter = substr($word,$place,1); 40 | 41 | my %valid = %VALID; 42 | delete $valid{'-'} if $place == 0 || $place == length($word)-1; 43 | 44 | my $base = ord($letter); 45 | for my $os ( 0..7 ) { 46 | my $xor = 2 ** $os; 47 | my $new = chr($base ^ $xor); 48 | next unless exists $valid{$new}; 49 | my $copy = $word; 50 | substr($copy,$place,1,$new); 51 | my $variation = lc join('.', $copy, @parts); 52 | verbose({color=>'yellow'}, "New bit variation on $domain: $variation") 53 | unless exists $variations{$variation}; 54 | $variations{$variation} = 1; 55 | } 56 | } 57 | debug("Found variations: "); 58 | debug_var([sort keys %variations]); 59 | 60 | my @NotFound = split /\n/, <1,color=>'red'}, "WHOIS ERROR: $error"); 81 | }; 82 | } while ( $raw =~ /LIMIT EXCEEDED/ ); 83 | 84 | if( defined $raw && $raw =~ /^$NotFound/o ) { 85 | $error = undef; 86 | $info = undef; 87 | } 88 | elsif( defined $raw && $raw =~ /WHOIS LIMIT EXCEEDED/ ) { 89 | undef($info); 90 | $error = $raw; 91 | } 92 | else { 93 | eval { 94 | my $result = parse_whois( raw => $raw, domain => $variation ); 95 | die "parse error" unless defined $result && ref $result eq 'HASH'; 96 | 97 | if( $result->{nameservers} && grep { defined } @{ $result->{nameservers} } ) { 98 | $info = join (',', sort map { exists $_->{domain} ? $_->{domain} : $_->{ip} } @{ $result->{nameservers} } ); 99 | } 100 | elsif( $result->{emails} && grep { defined } @{ $result->{emails} } ) { 101 | $info = join( ',', sort grep { defined } @{ $result->{emails} } ); 102 | } 103 | else { 104 | foreach my $f (qw(admin_email tech_email billing_email)) { 105 | last if length $info; 106 | $info = $result->{$f} if exists $result->{$f}; 107 | } 108 | } 109 | }; 110 | $error .= "\n$@" if $@; 111 | } 112 | 113 | if( $domain eq $variation ) { 114 | # Reference: 115 | output({color=>'cyan',sticky=>1}, defined $info ? 116 | "Reference $domain found with $info" : "Reference $domain not found." 117 | ); 118 | } 119 | else { 120 | verbose({color=>defined $error ? 'red' : defined $info ? 'yellow' : 'green'}, 121 | sprintf "%s variation %s is %s", 122 | $domain, 123 | $variation, 124 | defined $info ? "taken ($info)" : defined $error ? '!! ERROR !!' : '** AVAILABLE **', 125 | ); 126 | output({stderr=>1,color=>'red'}, sprintf "(error) %s - %s", $variation, $error) if defined $error && length $error; 127 | push @available, $variation if !defined $info && !defined $error; 128 | } 129 | } 130 | if( @available ) { 131 | output({color=>'cyan',clear=>1}, sprintf "# [%s] Available Variations %d of %d, %0.2f%%", 132 | $domain, 133 | scalar(@available), 134 | scalar(keys %variations), 135 | 100*(scalar(@available) / scalar(keys %variations)) 136 | ); 137 | output({indent=>1,data=>1}, @available); 138 | } 139 | -------------------------------------------------------------------------------- /bin/dc-bitflip-dns-server.pl: -------------------------------------------------------------------------------- 1 | #!perl 2 | # PODNAME: dc-bitflip-dns-server.pl 3 | use strict; 4 | use warnings; 5 | 6 | use CLI::Helpers qw(:output); 7 | use Getopt::Long::Descriptive; 8 | use JSON::MaybeXS; 9 | use Net::DNS; 10 | use Net::DNS::Nameserver; 11 | use YAML; 12 | 13 | my %DEFAULT = ( 14 | port => 53, 15 | addr => "0.0.0.0", 16 | ); 17 | my ($opt,$usage) = describe_options("%c %o", 18 | ['config=s', "Config file", { required => 1 } ], 19 | [], 20 | ['addr=s', "DNS Listening Address, default $DEFAULT{addr}", 21 | { default => $DEFAULT{addr} }], 22 | ['port=i', "DNS Listening Port, default $DEFAULT{port}", 23 | { default => $DEFAULT{port} }], 24 | [], 25 | ['help', "Display this help and exit", { shortcircuit => 1 }], 26 | ); 27 | if( $opt->help ) { 28 | print $usage->text; 29 | exit; 30 | } 31 | 32 | =head2 CONFIG File Format 33 | 34 | The YAML config file looks like this: 35 | 36 | --- 37 | domains: 38 | booking.com: 39 | records: 40 | A: 1.2.3.4 41 | MX: 10 mx.example.com 42 | flipped: 43 | - bnoking.com 44 | - bookiog.com 45 | 46 | =cut 47 | 48 | my $CFG = YAML::LoadFile($opt->config); 49 | 50 | my %DomainMap = (); 51 | foreach my $domain ( sort keys %{ $CFG->{domains} } ) { 52 | foreach my $flipped ( @{ $CFG->{domains}{$domain}{flipped} } ) { 53 | $DomainMap{$flipped} = $domain; 54 | } 55 | } 56 | 57 | my $MATCH = join( '|', map { quotemeta } sort keys %DomainMap ); 58 | my $JSON = JSON->new->canonical->utf8; 59 | 60 | sub handle_request { 61 | my @incoming = @_; 62 | my @names = qw(qname qclass qtype peerhost query conn); 63 | my %q = map { shift(@names) => $_ } @incoming; 64 | 65 | my $log = { 66 | src_ip => $q{peerhost}, 67 | query => { 68 | class => $q{qclass}, 69 | type => $q{qtype}, 70 | name => $q{qname}, 71 | }, 72 | }; 73 | 74 | my $rcode = $log->{status} = "NXDOMAIN"; 75 | my (@ans,@auth,@add); 76 | 77 | my $name = lc $q{qname}; 78 | if( my($flipped) = ($name =~ /($MATCH)$/) ) { 79 | $log->{src_domain} = $flipped; 80 | if( my $domain = $DomainMap{$flipped} ) { 81 | $log->{dst_domain} = $domain; 82 | if( exists $CFG->{domains}{$domain}{records}{$q{qtype}} ) { 83 | $rcode = $log->{status} = "NOERROR"; 84 | my $answer = $CFG->{domains}{$domain}{records}{$q{qtype}}; 85 | my $faked = $q{qname} =~ s/$flipped/$domain/ri; 86 | # Faked Response 87 | push @ans, Net::DNS::RR->new("$faked $q{qclass} $q{qtype} $answer"); 88 | # Original 89 | push @add, Net::DNS::RR->new("$q{qname} $q{qclass} $q{qtype} $answer"); 90 | } 91 | } 92 | } 93 | output($log->{status} eq 'NOERROR' ? {color=>"green"} :{stderr=>1,color=>'red'}, 94 | sprintf "QUERY %s [%s] from %s for '%s %s %s': %s", 95 | $log->{status}, 96 | $log->{src_domain} || 'UNKNOWN', 97 | $log->{src_ip}, 98 | $log->{query}{class}, 99 | $log->{query}{type}, 100 | $log->{query}{name}, 101 | $JSON->encode($log) 102 | ); 103 | 104 | return ( $rcode, \@ans, \@auth, \@add, { aa => 1 } ); 105 | } 106 | 107 | my $ns = Net::DNS::Nameserver->new( 108 | LocalAddr => $opt->addr, 109 | LocalPort => $opt->port, 110 | ReplyHandler => \&handle_request, 111 | Verbose => (CLI::Helpers::def('VERBOSE') || CLI::Helpers::def('DEBUG')) ? 1 : 0, 112 | ); 113 | output({color=>'cyan'}, "bitflip dns server started"); 114 | $ns->main_loop; 115 | -------------------------------------------------------------------------------- /bin/dc-component-analyze.pl: -------------------------------------------------------------------------------- 1 | #!perl 2 | # PODNAME: dc-component-analyze.pl 3 | use strict; 4 | use warnings; 5 | 6 | use FindBin; 7 | use Path::Tiny qw(path); 8 | use YAML (); 9 | 10 | use DreamCatcher::Packet; 11 | use DreamCatcher::Feathers; 12 | 13 | use POE qw( 14 | Filter::Line 15 | Wheel::ReadWrite 16 | Component::Log4perl 17 | ); 18 | 19 | # Global Object Instances 20 | my $config_file = $ARGV[0]; 21 | my $path_base = path($config_file)->parent; 22 | my $CFG = YAML::LoadFile( $config_file ); 23 | my $FEATHERS = DreamCatcher::Feathers->new( 24 | Config => $CFG, 25 | Log => sub { $poe_kernel->post( log => @_ ); }, 26 | ); 27 | 28 | # POE Sessions 29 | my $log_id = POE::Component::Log4perl->spawn( 30 | Alias => 'log', 31 | Category => 'Parser', 32 | ConfigFile => $path_base->child('logging.conf')->realpath->canonpath, 33 | ); 34 | my $session_id = POE::Session->create(inline_states => { 35 | _start => \&start_session, 36 | _stop => sub { $poe_kernel->shutdown; }, 37 | input => \&handle_input, 38 | error => \&handle_error, 39 | schedule => \&set_schedule, 40 | analyze => \&run_analyze, 41 | }); 42 | 43 | POE::Kernel->run(); 44 | exit(0); 45 | 46 | sub start_session { 47 | my ($kernel,$heap) = @_[KERNEL,HEAP]; 48 | 49 | # So we know when the parent dies 50 | $heap->{wheel} = POE::Wheel::ReadWrite->new( 51 | InputHandle => \*STDIN, 52 | OutputHandle => \*STDOUT, 53 | Filter => POE::Filter::Line->new(), 54 | InputEvent => 'input', 55 | ErrorEvent => 'error', 56 | ); 57 | $kernel->yield('schedule'); 58 | my $id = path($0)->basename; 59 | $kernel->post(log => 'info' => "$id startup successful"); 60 | } 61 | 62 | sub handle_input { 63 | my ($kernel,$heap,$msg) = @_[KERNEL,HEAP,ARG0]; 64 | $kernel->post(log => info => "Received input from parent: $msg"); 65 | } 66 | 67 | sub handle_error { 68 | my ($operation, $errnum, $errstr, $id, $handle) = @_[ARG0..ARG4]; 69 | $poe_kernel->post(log => warn => "Wheel $id $handle encountered $operation error $errnum: $errstr\n"); 70 | if ($operation eq "read" && $errnum == 0) { 71 | $poe_kernel->post(log => fatal => "Analysis Wheel $id received EOF, shutting down."); 72 | $poe_kernel->shutdown(); 73 | } 74 | } 75 | sub set_schedule { 76 | my ($kernel,$heap) = @_[KERNEL,HEAP]; 77 | 78 | # Retrieve the schedule 79 | $heap->{schedule} = $FEATHERS->schedule(); 80 | 81 | foreach my $name (keys %{ $heap->{schedule} }) { 82 | # First run will be in the first minute 83 | my $start = int rand(60); 84 | $kernel->post(log => info => "Schedule set for $name is every $heap->{schedule}{$name} : First run in $start seconds."); 85 | $kernel->delay_add( analyze => $start, $name ); 86 | } 87 | } 88 | 89 | sub run_analyze { 90 | my ($kernel,$heap,$name) = @_[KERNEL,HEAP,ARG0]; 91 | 92 | my $F = $FEATHERS->hash; 93 | 94 | if( !exists $F->{$name} ) { 95 | $kernel->post(log => error => "run_analyze($name): unknown analyzer"); 96 | return; 97 | } 98 | 99 | $kernel->post(log => info => "running analyzer $name"); 100 | eval { $F->{$name}->analyze() }; 101 | if(my $ex = $@) { 102 | $kernel->post(log => error => "$name encountered a fatal error: $ex"); 103 | } 104 | $kernel->delay_add( analyze => $F->{$name}->interval, $name); 105 | $kernel->post(log => debug => "rescheduled analyzer $name"); 106 | } 107 | -------------------------------------------------------------------------------- /bin/dc-component-parse.pl: -------------------------------------------------------------------------------- 1 | #!perl 2 | # PODNAME: dc-component-parse.pl 3 | use strict; 4 | use warnings; 5 | 6 | use FindBin; 7 | use Path::Tiny qw(path); 8 | use YAML (); 9 | 10 | use DreamCatcher::Packet; 11 | use DreamCatcher::Feathers; 12 | 13 | use POE qw( 14 | Filter::Reference 15 | Wheel::ReadWrite 16 | Component::Log4perl 17 | ); 18 | 19 | # Global Object Instances 20 | my $config_file = $ARGV[0]; 21 | my $path_base = path($config_file)->parent; 22 | my $CFG = YAML::LoadFile( $config_file ); 23 | my $FEATHERS = DreamCatcher::Feathers->new( 24 | Config => $CFG, 25 | Log => sub { $poe_kernel->post( log => @_ ); }, 26 | ); 27 | 28 | # POE Sessions 29 | my $log_id = POE::Component::Log4perl->spawn( 30 | Alias => 'log', 31 | Category => 'Parser', 32 | ConfigFile => $path_base->child('logging.conf')->realpath->canonpath, 33 | ); 34 | my $session_id = POE::Session->create(inline_states => { 35 | _start => \&start_session, 36 | _stop => sub { }, 37 | input => \&handle_input, 38 | error => \&handle_error, 39 | }); 40 | 41 | POE::Kernel->run(); 42 | exit(0); 43 | 44 | sub start_session { 45 | my ($kernel,$heap) = @_[KERNEL,HEAP]; 46 | 47 | # IO :) 48 | $heap->{wheel} = POE::Wheel::ReadWrite->new( 49 | InputHandle => \*STDIN, 50 | OutputHandle => \*STDOUT, 51 | Filter => POE::Filter::Reference->new(), 52 | InputEvent => 'input', 53 | ErrorEvent => 'error', 54 | ); 55 | my $id = path($0)->basename; 56 | $kernel->post(log => 'info' => "$id startup successful"); 57 | } 58 | 59 | sub handle_input { 60 | my ($kernel,$heap,$raw_packet) = @_[KERNEL,HEAP,ARG0]; 61 | 62 | my $packet = DreamCatcher::Packet->new( Raw => $raw_packet ); 63 | 64 | if( $packet->valid ) { 65 | $FEATHERS->process( $packet ); 66 | 67 | my $dt = $packet->details; 68 | my ($q) = $packet->dns->question; 69 | my $ques = join(' ', $q->qclass, $q->qtype, $q->qname); 70 | $kernel->post( log => debug => "$dt->{qa} $dt->{client} ($dt->{client_id}) to $dt->{server} ($dt->{server_id}) : $ques"); 71 | $heap->{wheel}->put( $dt ); 72 | } 73 | else { 74 | $kernel->post( log => error => $packet->error ); 75 | print STDERR $packet->error . "\n"; 76 | } 77 | } 78 | 79 | sub handle_error { 80 | my ($operation, $errnum, $errstr, $id, $handle) = @_[ARG0..ARG4]; 81 | $poe_kernel->post(log => warn => "Sniffer Wheel $id ($handle) encountered $operation error $errnum: $errstr\n"); 82 | 83 | if ($operation eq "read" and $errnum == 0) { 84 | $poe_kernel->post(log => fatal => "Sniffer received EOF, shutting down."); 85 | $poe_kernel->shutdown(); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /bin/domain-jumble.pl: -------------------------------------------------------------------------------- 1 | #!perl 2 | # PODNAME: domain-jumble.pl 3 | # ABSTRACT: Utility for combining words and checking the availability of domains. 4 | use strict; 5 | use warnings; 6 | 7 | use Algorithm::Permute qw(permute); 8 | use CLI::Helpers qw(:output); 9 | use Getopt::Long::Descriptive; 10 | use Net::Whois::Parser; 11 | use Net::Whois::Raw; 12 | use List::Util qw(any); 13 | use Pod::Usage; 14 | 15 | # Configure Net::Whois::Raw 16 | $Net::Whois::Raw::OMIT_MSG = 1; 17 | $Net::Whois::Raw::CHECK_FAIL = 0; 18 | $Net::Whois::Raw::CACHE_DIR = "$ENV{HOME}/tmp"; 19 | $Net::Whois::Raw::TIMEOUT = 10; 20 | 21 | # Options Parsing 22 | my ($opt,$usage) = describe_options( 23 | "%c %o [list of optional words to include]", 24 | [], 25 | [ 'tlds=s', "Comma separated list of TLDs to search (Default: 'com,net')", {default => 'com,net'} ], 26 | [ 'required|r=s', "Comma separated list of required words", ], 27 | [ 'maxwords|n=i', "Maximum number of words to allow, (Default: 4)", {default => 4}, ], 28 | [ 'exclusive|e=s', "Comma separated list of exclusive required words", ], 29 | [ 'separator|seperator=s', "String to use to separate words (Default: -)", {default => '-'},], 30 | [], 31 | [ 'help|h', 'print this menu and exit'], 32 | [ 'manual|m', 'print the manual'], 33 | ); 34 | pod2usage(-exit=>0,-verbose=>2) if $opt->manual; 35 | output($usage->text) if $opt->help; 36 | 37 | my $domain; 38 | my $max_words = $opt->maxwords; 39 | $max_words = 2 if $max_words < 2; 40 | my @TLDS = split /\,/, $opt->tlds; 41 | my %Required = defined $opt->required ? map { $_ => 1 } split(',', $opt->required) : (); 42 | my %words = map { lc($_) => 1 } map { split ','; } @ARGV, keys %Required; 43 | my @words = sort keys %words; 44 | my %variations; 45 | 46 | if(defined $opt->exclusive) { 47 | foreach my $word (split ',', $opt->exclusive) { 48 | $Required{$word} ||= 0; 49 | $Required{$word}++; 50 | @variations{build_list(@words,$word)} = (); 51 | delete $Required{$word} unless --$Required{$word}; 52 | } 53 | } 54 | else { 55 | @variations{build_list(@words)} = (); 56 | } 57 | verbose({level=>2}, "Will check variations:"); 58 | verbose({level=>2}, $_) for sort keys %variations; 59 | 60 | sub build_list { 61 | my @words = @_; 62 | my %collected = (); 63 | for(my $i=0; $i < @words; $i++ ) { 64 | my $key = $words[$i]; 65 | my @subset = (); 66 | foreach my $other (@words) { 67 | next if $other eq $words[$i]; 68 | push @subset, $other; 69 | 70 | shift @subset if @subset == $max_words; 71 | 72 | if( keys %Required && !exists $Required{$key}) { 73 | next unless any { exists $Required{$_} } @subset; 74 | } 75 | 76 | push @subset, $key; 77 | permute { my $name=join($opt->separator, @subset); $collected{"$name.$_"}=1 for @TLDS; } @subset; 78 | pop @subset; 79 | } 80 | } 81 | return keys %collected; 82 | } 83 | my $num_variants = scalar keys %variations; 84 | my @available = (); 85 | my @NotFound = split /\n/, < $raw, domain => $variation ); 108 | die "parse error" unless defined $result && ref $result eq 'HASH'; 109 | 110 | if( exists $result->{nameservers} ) { 111 | $info = join (',', sort map { exists $_->{domain} ? $_->{domain} : $_->{ip} } @{ $result->{nameservers} } ); 112 | } 113 | elsif(exists $result->{emails} && defined $result->{emails} ) { 114 | $info = join( ',', sort grep { defined $_ } @{ $result->{emails} } ); 115 | } 116 | else { 117 | foreach my $f (qw(admin_email tech_email billing_email)) { 118 | last if length $info; 119 | $info = $result->{$f} if exists $result->{$f}; 120 | } 121 | } 122 | }; 123 | $error .= "\n$@" if $@; 124 | } 125 | 126 | my $color = defined $info ? 'cyan' 127 | : defined $error ? 'red' 128 | : 'green'; 129 | verbose({color=>$color}, 130 | sprintf("Variation %s is %s", 131 | $variation, 132 | defined $info ? "taken ($info)" : 133 | defined $error ? '!! ERROR !!' : '** AVAILABLE **' 134 | ), $error ? $error : (), 135 | ); 136 | next if defined $error && length $error; 137 | 138 | push @available, $variation if !defined $info; 139 | } 140 | if( @available ) { 141 | verbose({clear=>1},"# Available variations",""); 142 | output({indent=>1},$_) for sort @available; 143 | output({clear=>1},sprintf "# Variations %d of %d available (%0.2f%%)", scalar(@available), $num_variants, 100*(scalar(@available) / $num_variants)); 144 | } 145 | __END__ 146 | =pod 147 | 148 | =head1 SYNOPSIS 149 | 150 | domain-jumble.pl [options] 151 | 152 | See 153 | 154 | domain-jumble.pl --help 155 | 156 | For a list of options 157 | 158 | =head1 DESCRIPTION 159 | 160 | This tool allows you to transform a list of words into a list of possible domain names with some intelligence 161 | and check the availability of those domains using whois. 162 | 163 | =head2 EXAMPLES 164 | 165 | To check for domains which require the word "apple" and optionally have the words "rotten" or "fresh": 166 | 167 | 168 | domain-jumble.pl --tlds com --require apple rotten fresh 169 | 170 | Looks for: 171 | 172 | apple-fresh-rotten.com 173 | apple-rotten-fresh.com 174 | apple-rotten.com 175 | fresh-apple-rotten.com 176 | fresh-apple.com 177 | fresh-rotten-apple.com 178 | rotten-apple-fresh.com 179 | rotten-apple.com 180 | rotten-fresh-apple.com 181 | 182 | If you want to have "fresh" or "rotten" match exclusively: 183 | 184 | domain-jumble.pl --tlds com --require apple --exclusive fresh,rotten 185 | 186 | Will look for: 187 | 188 | apple-fresh.com 189 | apple-rotten.com 190 | fresh-apple.com 191 | rotten-apple.com 192 | 193 | Add fruit as an optional parameter: 194 | 195 | domain-jumble.pl --tlds com --require apple --exclusive fresh,rotten fruit 196 | 197 | Will check: 198 | 199 | apple-fresh-fruit.com 200 | apple-fresh.com 201 | apple-fruit-fresh.com 202 | apple-fruit-rotten.com 203 | apple-fruit.com 204 | apple-rotten-fruit.com 205 | apple-rotten.com 206 | fresh-apple-fruit.com 207 | fresh-apple.com 208 | fresh-fruit-apple.com 209 | fruit-apple-fresh.com 210 | fruit-apple-rotten.com 211 | fruit-apple.com 212 | fruit-fresh-apple.com 213 | fruit-rotten-apple.com 214 | rotten-apple-fruit.com 215 | rotten-apple.com 216 | rotten-fruit-apple.com 217 | 218 | You can also specify --maxwords to limit how many words to join together, the default is 4: 219 | 220 | domain-jumble.pl --tlds com --maxwords 2 --require apple --exclusive fresh,rotten fruit 221 | 222 | Will now only look for: 223 | 224 | apple-fresh.com 225 | apple-fruit.com 226 | apple-rotten.com 227 | fresh-apple.com 228 | fruit-apple.com 229 | rotten-apple.com 230 | 231 | Adjust options as necessary, caches used as available. 232 | -------------------------------------------------------------------------------- /cpanfile: -------------------------------------------------------------------------------- 1 | requires "Algorithm::Permute" => "0"; 2 | requires "CHI" => "0"; 3 | requires "CLI::Helpers" => "0"; 4 | requires "Cache::FastMmap" => "0"; 5 | requires "Const::Fast" => "0"; 6 | requires "DBIx::Connector" => "0"; 7 | requires "Daemon::Daemonize" => "0"; 8 | requires "DateTime" => "0"; 9 | requires "DateTime::Format::Pg" => "0"; 10 | requires "Exception::Class::DBI" => "0"; 11 | requires "FindBin" => "0"; 12 | requires "Getopt::Long::Descriptive" => "0"; 13 | requires "HTML::Entities" => "0"; 14 | requires "JSON::MaybeXS" => "0"; 15 | requires "LWP::Simple" => "0"; 16 | requires "List::Util" => "0"; 17 | requires "Log::Dispatch::FileRotate" => "0"; 18 | requires "Log::Log4perl" => "0"; 19 | requires "Module::Pluggable" => "0"; 20 | requires "Mojo::Base" => "0"; 21 | requires "Mojolicious::Plugin" => "0"; 22 | requires "Mojolicious::Plugin::YamlConfig" => "0"; 23 | requires "Moose" => "0"; 24 | requires "Moose::Role" => "0"; 25 | requires "MooseX::Types" => "0"; 26 | requires "MooseX::Types::Moose" => "0"; 27 | requires "Net::DNS" => "0"; 28 | requires "Net::DNS::Nameserver" => "0"; 29 | requires "Net::DNS::Packet" => "0"; 30 | requires "Net::DNS::SEC" => "0"; 31 | requires "Net::IP" => "0"; 32 | requires "Net::Whois::Parser" => "0"; 33 | requires "Net::Whois::Raw" => "0"; 34 | requires "NetPacket::Ethernet" => "0"; 35 | requires "NetPacket::IP" => "0"; 36 | requires "NetPacket::IPv6" => "0"; 37 | requires "NetPacket::TCP" => "0"; 38 | requires "NetPacket::UDP" => "0"; 39 | requires "POE" => "0"; 40 | requires "POE::Component::Log4perl" => "0"; 41 | requires "POE::Component::Pcap" => "0"; 42 | requires "POE::Filter::Line" => "0"; 43 | requires "POE::Filter::Reference" => "0"; 44 | requires "POE::Wheel::ReadWrite" => "0"; 45 | requires "POE::Wheel::Run" => "0"; 46 | requires "POSIX" => "0"; 47 | requires "Path::Tiny" => "0"; 48 | requires "Pod::Usage" => "0"; 49 | requires "Sys::CpuAffinity" => "0"; 50 | requires "Sys::Syslog" => "0"; 51 | requires "Text::Soundex" => "0"; 52 | requires "Text::Unidecode" => "0"; 53 | requires "Tree::DAG_Node" => "0"; 54 | requires "YAML" => "0"; 55 | requires "base" => "0"; 56 | requires "feature" => "0"; 57 | requires "namespace::autoclean" => "0"; 58 | requires "perl" => "5.010"; 59 | requires "strict" => "0"; 60 | requires "warnings" => "0"; 61 | 62 | on 'test' => sub { 63 | requires "File::Spec" => "0"; 64 | requires "IO::Handle" => "0"; 65 | requires "IPC::Open3" => "0"; 66 | requires "Pod::Coverage::TrustPod" => "0"; 67 | requires "Test::Mojo" => "0"; 68 | requires "Test::More" => "0"; 69 | requires "Test::Perl::Critic" => "0"; 70 | requires "blib" => "1.01"; 71 | requires "perl" => "5.010"; 72 | }; 73 | 74 | on 'configure' => sub { 75 | requires "ExtUtils::MakeMaker" => "0"; 76 | requires "perl" => "5.010"; 77 | }; 78 | 79 | on 'develop' => sub { 80 | requires "Pod::Coverage::TrustPod" => "0"; 81 | requires "Test::EOL" => "0"; 82 | requires "Test::More" => "0.88"; 83 | requires "Test::Pod" => "1.41"; 84 | requires "Test::Pod::Coverage" => "1.08"; 85 | }; 86 | -------------------------------------------------------------------------------- /dist.ini: -------------------------------------------------------------------------------- 1 | name = DreamCatcher 2 | author = Brad Lhotsky 3 | license = BSD 4 | copyright_holder = Brad Lhotsky 5 | copyright_year = 2017 6 | 7 | [PruneCruft] 8 | [MakeMaker] 9 | [Manifest] 10 | 11 | [Encoding] 12 | encoding = bytes 13 | match = png 14 | 15 | [ExecDir] 16 | dir = bin 17 | 18 | ; Prerequesites 19 | [AutoPrereqs] 20 | [CheckPrereqsIndexed] ; ensure prereqs are on CPAN 21 | [Prereqs] 22 | Mojolicious::Plugin::YamlConfig = 0 23 | [Prereqs / TestRequires] 24 | Pod::Coverage::TrustPod = 0 25 | Test::Perl::Critic = 0 26 | ; authordep Pod::Weaver::Section::Contributors 27 | ; authordep Pod::Weaver::Section::Support 28 | ; authordep Pod::Weaver::Section::Collect::FromOther 29 | ; authordep Pod::Elemental::Transformer::List 30 | 31 | [MinimumPerl] 32 | [PkgVersion] 33 | [MetaConfig] 34 | [MetaJSON] 35 | [CPANFile] 36 | 37 | ; Docs 38 | [PodWeaver] 39 | [ContributorsFromGit] 40 | [License] 41 | [InsertCopyright] 42 | [ReadmeMarkdownFromPod] 43 | [PodCoverageTests] 44 | [PodSyntaxTests] 45 | [Test::Compile] 46 | [Test::EOL] 47 | 48 | 49 | ;Changelog 50 | [ChangelogFromGit] 51 | max_age = 730 52 | tag_regexp = ^release-(\d.*) 53 | file_name = Changes 54 | wrap_column = 80 55 | copy_to_root = 0 56 | exclude_message = ^(v\d\.\d|Archive|Merge pull request|bug fix) 57 | 58 | ; Handle Files 59 | [CopyFilesFromBuild] 60 | copy = README.mkdn 61 | copy = cpanfile 62 | 63 | [PruneFiles] 64 | filename = dist.ini 65 | filename = weaver.ini 66 | filename = Ideas.mkdn 67 | 68 | ; Git stuff 69 | [Git::GatherDir] 70 | exclude_filename = README.mkdn 71 | exclude_filename = META.json 72 | exclude_filename = Changes 73 | exclude_filename = cpanfile 74 | 75 | [Git::NextVersion] 76 | ; get version from last release tag 77 | version_regexp = ^release-(.+)$ 78 | [OurPkgVersion] 79 | 80 | [Git::Check] 81 | ; ensure all files checked in 82 | allow_dirty = dist.ini 83 | allow_dirty = Changes 84 | allow_dirty = README.mkdn 85 | allow_dirty = Maintenance.mkdn 86 | allow_dirty = Searching.mkdn 87 | allow_dirty = CopyIndexes.mkdn 88 | allow_dirty = META.json 89 | 90 | [GithubMeta] 91 | issues = 1 92 | 93 | ; Commit handling / Release? 94 | [ConfirmRelease] 95 | [TestRelease] 96 | [UploadToCPAN] 97 | [Git::Commit / Commit_Dirty_Files] 98 | 99 | [Git::Tag] 100 | tag_format = release-%v 101 | 102 | [NextRelease] 103 | 104 | [Git::Commit / Commit_Changes] ; commit Changes (for new dev) 105 | 106 | [Git::Push] ; push repo to remote 107 | push_to = origin 108 | 109 | ; Install this on release 110 | [InstallRelease] 111 | -------------------------------------------------------------------------------- /dreamcatcher.yml.default: -------------------------------------------------------------------------------- 1 | --- 2 | time_zone: America/New_York 3 | db: 4 | dsn: dbi:Pg:host=localhost;database=dreamcatcher 5 | user: admin 6 | pass: 7 | 8 | network: 9 | nameservers: &GLOBALnameservers 10 | - 8.8.8.8 11 | - 8.8.4.4 12 | clients: &GLOBALclients 13 | - 192.168.1.0/24 14 | 15 | pcap: 16 | dev: any 17 | snaplen: 1518 18 | timeout: 100 19 | filter: (tcp or udp) and port 53 20 | promisc: 0 21 | 22 | sniffer: 23 | workers: 4 24 | 25 | analysis: 26 | disabled: 0 27 | 28 | feather: 29 | conversation: 30 | disabled: 0 31 | -------------------------------------------------------------------------------- /examples/bitflip-server-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | domains: 3 | booking.com: 4 | records: 5 | A: 127.0.0.1 6 | AAAA: "::1" 7 | MX: 10 mx.example.com 8 | flipped: 9 | - bnoking.com 10 | - bookiog.com 11 | -------------------------------------------------------------------------------- /examples/screenshots/0-main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/examples/screenshots/0-main.png -------------------------------------------------------------------------------- /examples/screenshots/1-server.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/examples/screenshots/1-server.png -------------------------------------------------------------------------------- /examples/screenshots/3-questions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/examples/screenshots/3-questions.png -------------------------------------------------------------------------------- /examples/test-feathers.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | # 3 | use strict; 4 | use warnings; 5 | use YAML qw(Dump LoadFile); 6 | 7 | use lib "$ENV{HOME}/code/dreamcatcher/lib"; 8 | use DreamCatcher::Feathers; 9 | 10 | my $feathers = DreamCatcher::Feathers->new(Config => LoadFile("$ENV{HOME}/code/dreamcatcher/dreamcatcher.yml")); 11 | printf("Feathers found %s(%s): => %s\n", $_->name, $_->priority, $_->parent ) for values %{ $feathers->hash }; 12 | 13 | my $collection = $feathers->chain('analysis'); 14 | 15 | foreach my $feather ( @{ $collection }) { 16 | printf "chained %s [%d] after %s\n", $feather->name, $feather->priority, $feather->parent; 17 | } 18 | -------------------------------------------------------------------------------- /lib/DreamCatcher.pm: -------------------------------------------------------------------------------- 1 | # ABSTRACT: DreamCatcher is a DNS Monitoring Suite 2 | package DreamCatcher; 3 | 4 | # VERSION 5 | 6 | use DreamCatcher::Helpers; 7 | use Mojo::Base 'Mojolicious'; 8 | 9 | # This method will run once at server start 10 | sub startup { 11 | my $self = shift; 12 | 13 | # Setup 14 | $self->secrets(['look at my horse, my horse is amazing']); 15 | $self->mode('development'); 16 | $self->sessions->default_expiration(3600*24*7); 17 | 18 | # App Configuration 19 | my $config = $self->plugin( yaml_config => { 20 | file => 'dreamcatcher.yml', 21 | stash_key => 'config', 22 | class => 'YAML::XS', 23 | } ); 24 | 25 | # Helpers 26 | $self->plugin('DreamCatcher::Helpers'); 27 | 28 | # Configure Defaults 29 | $self->defaults( 30 | layout => 'bootstrap', 31 | ); 32 | 33 | # Router 34 | my $r = $self->routes; 35 | $r->namespaces([qw(DreamCatcher::Controller)]); 36 | 37 | # Normal route to controller 38 | $r->get('/')->to('main#index'); 39 | 40 | # Questions Module 41 | $r->get('/questions')->to('questions#index'); 42 | 43 | # Server Module 44 | $r->get('/server')->to('server#index'); 45 | $r->get('/server/:id')->to('server#view'); 46 | 47 | # Conversation Module 48 | $r->get('/conversation')->to('conversation#index'); 49 | $r->get('/conversation/:id')->to('conversation#view'); 50 | 51 | # List Module 52 | $r->get('/list')->to('list#index'); 53 | $r->get('/list/:id')->to('list#view'); 54 | 55 | # Utilities Module 56 | $r->get('/utility')->to('utility#index'); 57 | $r->get('/utilities')->to('utility#index'); 58 | $r->post('/utility/reverse')->to(controller => 'Utility', action => 'reverse_lookup'); 59 | $r->post('/utility/clients_asking')->to(controller => 'Utility', action => 'clients_asking'); 60 | $r->get('/utility/csmap')->to(controller => 'Utility', action => 'client_server_map'); 61 | } 62 | 63 | 1; 64 | __END__ 65 | =pod 66 | 67 | =head1 SYNOPSIS 68 | 69 | This is a complete DNS Monitoring Suite. It is currently in B status. 70 | 71 | A libpcap based sniffer daemon listens to DNS traffic on your network. The 72 | conversations are recorded and analyzed to provide insight. 73 | 74 | =head1 PREREQUISISTES 75 | 76 | =over 77 | 78 | =item B 79 | 80 | 5.14.2 or better 81 | 82 | =item B 83 | 84 | 9.4 or better with the B extension 85 | 86 | =back 87 | 88 | =head1 INSTALLATION 89 | 90 | Installation in the works, for now try: 91 | 92 | perl Makefile.PL 93 | make 94 | 95 | Then install the schema: 96 | 97 | cd sql 98 | ./deploy_database_schema.pl install 99 | 100 | Configure the instance: 101 | 102 | cp dreamcatcher.yml.default dreamcatcher.yml 103 | $EDITOR dreamcatcher.yml 104 | 105 | Configure logging: 106 | 107 | $EDITOR logging.conf 108 | 109 | =head1 USAGE 110 | 111 | Once you have the database schema and the dreamcatcher.yaml configured, run the collector: 112 | 113 | sudo ./bin/dreamcatcher.pl start 114 | 115 | Now start the web application for viewing the data: 116 | 117 | morbo -v script/dream_catcher 118 | 119 | =head1 SCREENSHOTS 120 | 121 | =over 122 | 123 | =item L 124 | 125 | =item L 126 | 127 | =item L 128 | 129 | =back 130 | 131 | =cut 132 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller; 2 | 3 | use Mojo::Base qw( Mojolicious::Controller ); 4 | 5 | use DBIx::Connector; 6 | 7 | my $_dbh; 8 | 9 | # Database Connection 10 | has dbconn => sub { 11 | my ($self) = @_; 12 | if( ! defined $_dbh ) { 13 | my %c = %{ $self->app->config->{db} }; 14 | $_dbh = DBIx::Connector->new( @c{qw{dsn user pass}}); 15 | } 16 | return $_dbh; 17 | }; 18 | 19 | # Prepare a hash of statements 20 | sub prepare_statements { 21 | my ($self,$sql) = @_; 22 | 23 | my %sth = (); 24 | foreach my $s ( keys %{ $sql } ) { 25 | $sth{$s} = $self->prepare_statement( $sql->{$s} ); 26 | } 27 | return wantarray ? %sth : \%sth; 28 | } 29 | 30 | # Prepare a single statement 31 | sub prepare_statement { 32 | my ($self,$sql) = @_; 33 | 34 | return $self->dbconn->run( fixup => sub { 35 | my ($dbh) = @_; 36 | return $dbh->prepare( $sql ); 37 | }); 38 | } 39 | 40 | # Common Queries 41 | has common_queries => sub { 42 | return { 43 | top_servers => qq{ 44 | select server.id, server.ip as ip, count(1) as clients, 45 | bool_and(is_authorized) as is_authorized, 46 | min(conversation.first_ts) as first_ts, 47 | max(conversation.last_ts) as last_ts, 48 | sum(conversation.reference_count) as conversations 49 | from conversation 50 | inner join server on conversation.server_id = server.id 51 | where conversation.last_ts > NOW() - interval '30 days' 52 | group by server.id, server.ip 53 | order by conversations DESC , clients DESC, first_ts DESC 54 | limit 200 55 | }, 56 | top_zones => qq{ 57 | select id, name, reference_count from zone order by reference_count DESC limit 200 58 | }, 59 | server_responses => q{ 60 | select 61 | srv.id, srv.ip, r.opcode, r.status, count(1) as queries, sum(count(1)) OVER (PARTITION BY r.server_id) as total 62 | from response r 63 | inner join server srv on r.server_id = srv.id 64 | group by srv.id, r.server_id, r.opcode, r.status 65 | order by total DESC, queries DESC 66 | limit 200 67 | }, 68 | top_questions => qq{ 69 | select r.*, aq.* 70 | from question r 71 | left join anomaly_question aq on r.id = aq.id 72 | order by reference_count DESC limit 200 73 | }, 74 | recent_questions => qq{ 75 | select * from question 76 | order by first_ts DESC limit 200 77 | }, 78 | missed_questions => qq{ 79 | select 80 | prq.class, 81 | prq.type, 82 | prq.name, 83 | min(prq.first_ts) as first_ts, 84 | max(prq.last_ts) as last_ts, 85 | count(1) as misses 86 | from response pr 87 | inner join meta_query_response pmqr on pr.id = pmqr.response_id 88 | inner join meta_question pmq on pmqr.query_id = pmq.query_id 89 | inner join question prq on pmq.question_id = prq.id 90 | where pr.status = 'NXDOMAIN' 91 | group by prq.class, prq.type, prq.name 92 | order by misses DESC 93 | limit 200 94 | }, 95 | }; 96 | }; 97 | 98 | # Prepare a common query and stash it 99 | sub common_query { 100 | my ($self,$query, @parms) = @_; 101 | 102 | return unless exists $self->common_queries->{$query}; 103 | 104 | $self->stash->{STH} = {} unless exists $self->stash->{STH}; 105 | $self->stash->{STH}{$query} = $self->prepare_statement( $self->common_queries->{$query} ); 106 | $self->stash->{STH}{$query}->execute(@parms); 107 | } 108 | 109 | # Return true 110 | 1; 111 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/Client.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::Client; 2 | # ABSTRACT: DreamCatcher Client Pages 3 | use Mojo::Base 'DreamCatcher::Controller'; 4 | 5 | sub index { 6 | my $self = shift; 7 | $self->render(); 8 | } 9 | 10 | sub view { 11 | my $self = shift; 12 | my $id = $self->stash('id'); 13 | 14 | my %sql = ( 15 | client => qq{ 16 | select * from client 17 | where id = ? 18 | }, 19 | servers => qq{ 20 | select srv.ip, 21 | cv.first_ts, 22 | cv.last_ts, 23 | cv.reference_count as conversation_count, 24 | srv.reference_count as total_count 25 | from conversation cv 26 | inner join server srv on cv.server_id = srv.id 27 | where cv.client_id = ? 28 | }, 29 | 30 | ); 31 | 32 | # Prepare SQL 33 | my $STH = $self->prepare_statements( \%sql ); 34 | # Execute it! 35 | $STH->{$_}->execute($id) for keys %{ $STH }; 36 | 37 | # TODO: Unknown client handle 38 | 39 | # Stash Details 40 | $self->stash( client => $STH->{client}->fetchrow_hashref ); 41 | $self->stash( servers_sth => $STH->{servers} ); 42 | 43 | $self->render(); 44 | } 45 | 46 | 1; 47 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/Conversation.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::Conversation; 2 | # ABSTRACT: DreamCatcher Conversation Pages 3 | use Mojo::Base 'DreamCatcher::Controller'; 4 | 5 | sub index { 6 | my $self = shift; 7 | 8 | 9 | $self->render(); 10 | } 11 | 12 | sub view { 13 | my $self = shift; 14 | my $id = $self->stash('id'); 15 | 16 | my %sql = ( 17 | meta => q{ 18 | select 19 | cv.server_id, 20 | cv.client_id, 21 | c.ip as client, 22 | s.ip as server, 23 | cv.first_ts, 24 | cv.last_ts, 25 | cv.reference_count as references 26 | from conversation cv 27 | inner join client c on c.id = cv.client_id 28 | inner join server s on s.id = cv.server_id 29 | where cv.id = ? 30 | }, 31 | queries => q{ 32 | select 33 | q.query_ts as query_ts, 34 | q.query_serial as serial, 35 | q.client_port as client_port, 36 | q.server_port as server_port, 37 | q.opcode as opcode, 38 | q.flag_recursive, q.flag_truncated, q.flag_checking, 39 | qs.class as qclass, 40 | qs.type as qtype, 41 | qs.name as qname, 42 | mqr.response_id, 43 | r.status as status, 44 | mqr.timing as took 45 | 46 | from conversation cv 47 | inner join query q on q.conversation_id = cv.id 48 | inner join meta_question mq on q.id = mq.query_id 49 | inner join question qs on mq.question_id = qs.id 50 | left join meta_query_response mqr on q.id = mqr.query_id 51 | left join response r on mqr.response_id = r.id 52 | where cv.id = ? 53 | order by q.query_ts DESC 54 | limit 1000 55 | }, 56 | responses => q{ 57 | select 58 | r.response_ts, 59 | r.client_port, 60 | r.server_port, 61 | r.query_serial as serial, 62 | r.opcode as opcode, 63 | a.class as aclass, 64 | a.type as atype, 65 | a.name as aname, 66 | a.value as value, 67 | a.opts as opts, 68 | r.status as status, 69 | r.flag_authoritative, r.flag_recursion_available, 70 | ma.section as section, 71 | mqr.timing as took 72 | 73 | from conversation cv 74 | inner join response r on r.conversation_id = cv.id 75 | left join meta_query_response mqr on r.id = mqr.response_id 76 | left join meta_answer ma on r.id = ma.response_id 77 | left join answer a on ma.answer_id = a.id 78 | where cv.id = ? 79 | order by r.response_ts DESC 80 | limit 1000 81 | }, 82 | ); 83 | 84 | # Prepare SQL 85 | my $STH = $self->prepare_statements( \%sql ); 86 | # Execute it! 87 | $STH->{$_}->execute($id) for keys %{ $STH }; 88 | 89 | # Stash Details 90 | $self->stash( 91 | meta => $STH->{meta}->fetchrow_hashref, 92 | query_sth => $STH->{queries}, 93 | response_sth => $STH->{responses}, 94 | ); 95 | 96 | $self->render(); 97 | } 98 | 99 | 1; 100 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/List.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::List; 2 | # ABSTRACT: DreamCatcher List Pages 3 | use Mojo::Base 'DreamCatcher::Controller'; 4 | 5 | sub index { 6 | my $self = shift; 7 | 8 | my $sth = $self->prepare_statement(q{ 9 | select l.*, 10 | lt.name as type, 11 | (select count(1) from list_entry WHERE list_id = l.id) as entries 12 | from list l 13 | inner join list_type lt on l.type_id = lt.id 14 | }); 15 | $sth->execute(); 16 | 17 | my %l = (); 18 | while ( my $ent = $sth->fetchrow_hashref ) { 19 | $l{$ent->{id}} = $ent; 20 | } 21 | 22 | $self->stash(lists => \%l); 23 | 24 | $self->render(); 25 | } 26 | 27 | sub view { 28 | my $self = shift; 29 | my $id = $self->stash('id'); 30 | 31 | my %sql = ( 32 | entries => q{ 33 | SELECT 34 | le.* 35 | FROM list_entry AS le 36 | WHERE le.list_id = ? 37 | }, 38 | list => q{ 39 | SELECT 40 | l.*, 41 | lt.name as type, 42 | lt.score 43 | FROM list AS l 44 | INNER JOIN list_type AS lt ON l.type_id = lt.id 45 | WHERE l.id = ? 46 | }, 47 | tracking => q{ 48 | SELECT 49 | COUNT(1) AS clients, 50 | MIN(first_ts) AS first_ts, 51 | MAX(last_ts) AS last_ts, 52 | SUM(reference_count) AS total 53 | FROM list_tracking_client AS ltc 54 | WHERE ltc.list_id = ? 55 | GROUP BY ltc.list_id 56 | }, 57 | ); 58 | 59 | my %sth = (); 60 | foreach my $s (keys %sql) { 61 | $sth{$s} = $self->prepare_statement($sql{$s}); 62 | } 63 | foreach my $s (qw(list entries tracking)) { 64 | $sth{$s}->execute($id); 65 | } 66 | 67 | my @entries = (); 68 | while( my $e = $sth{entries}->fetchrow_hashref ) { 69 | push @entries, $e; 70 | } 71 | 72 | $self->stash( 73 | list => $sth{list}->fetchrow_hashref, 74 | entries => \@entries, 75 | tracking => $sth{tracking}->fetchrow_hashref, 76 | ); 77 | 78 | $self->render; 79 | } 80 | 81 | 1; 82 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/Main.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::Main; 2 | # ABSTRACT: DreamCatcher Front Pages 3 | use Mojo::Base 'DreamCatcher::Controller'; 4 | 5 | sub index { 6 | my $self = shift; 7 | 8 | # Load some common queries 9 | $self->common_query( $_ ) for qw{top_servers top_zones server_responses}; 10 | 11 | $self->render(); 12 | } 13 | 14 | 1; 15 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/Questions.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::Questions; 2 | # ABSTRACT: DreamCatcher Question Pages 3 | use Mojo::Base 'DreamCatcher::Controller'; 4 | 5 | sub index { 6 | my $self = shift; 7 | 8 | # Load some common queries 9 | $self->common_query( $_ ) for qw{top_questions recent_questions missed_questions}; 10 | 11 | $self->render(); 12 | } 13 | 14 | 1; 15 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/Server.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::Server; 2 | # ABSTRACT: DreamCatcher Server Pages 3 | use Mojo::Base 'DreamCatcher::Controller'; 4 | 5 | sub index { 6 | my $self = shift; 7 | 8 | my %sql = ( 9 | ); 10 | # Prepare SQL 11 | my $STH = $self->prepare_statements( \%sql ); 12 | # Execute it! 13 | $STH->{$_}->execute() for keys %{ $STH }; 14 | # Stash 15 | $self->stash( STH => $STH ); 16 | 17 | $self->common_query( $_ ) for qw{top_servers server_responses}; 18 | 19 | $self->render(); 20 | } 21 | 22 | sub view { 23 | my $self = shift; 24 | 25 | my $id = $self->stash('id'); 26 | 27 | my %sql = ( 28 | server => qq{ 29 | select * from server 30 | where id = ? 31 | }, 32 | clients => qq{ 33 | select cli.ip, 34 | cv.id, 35 | cv.first_ts, 36 | cv.last_ts, 37 | cv.reference_count as conversation_count, 38 | cli.reference_count as total_count 39 | from conversation cv 40 | inner join client cli on cv.client_id = cli.id 41 | where cv.server_id = ? 42 | }, 43 | 44 | ); 45 | 46 | # Prepare SQL 47 | my $STH = $self->prepare_statements( \%sql ); 48 | # Execute it! 49 | $STH->{$_}->execute($id) for keys %{ $STH }; 50 | 51 | # TODO: Unknown server handle 52 | 53 | # Stash Details 54 | $self->stash( server => $STH->{server}->fetchrow_hashref ); 55 | $self->stash( clients_sth => $STH->{clients} ); 56 | 57 | $self->render(); 58 | } 59 | 60 | 1; 61 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Controller/Utility.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Controller::Utility; 2 | 3 | # ABSTRACT: DreamCatcher Utility Pages 4 | use Mojo::Base 'DreamCatcher::Controller'; 5 | 6 | use Net::IP; 7 | 8 | sub index { 9 | my $self = shift; 10 | 11 | $self->render(); 12 | } 13 | 14 | sub reverse_lookup { 15 | my $self = shift; 16 | 17 | my $ip = $self->param('ip'); 18 | 19 | # Fast check for valid IP Address 20 | if( $ip !~ /^([0-9]{1,3}\.){3}[0-9]{1,3}$/ ) { 21 | $self->flash( error => "Invalid IP address passed to Utility::reverse_lookup"); 22 | $self->redirect_to("/utility"); 23 | return; 24 | } 25 | 26 | my %sql = ( 27 | reverse_lookup => q{ 28 | select 29 | id, 30 | first_ts, 31 | last_ts, 32 | name, 33 | type, 34 | class, 35 | value, 36 | reference_count 37 | from answer 38 | where value = ? 39 | }, 40 | ); 41 | 42 | # Prepare SQL 43 | my $STH = $self->prepare_statements( \%sql ); 44 | # Execute it! 45 | $STH->{$_}->execute($ip) for keys %{ $STH }; 46 | 47 | $self->stash( 48 | ip => $ip, 49 | STH => $STH, 50 | ); 51 | 52 | $self->render("utility/reverse"); 53 | } 54 | 55 | sub clients_asking { 56 | my ($self) = shift; 57 | 58 | # Normalize the parts 59 | my $question = $self->param('question'); 60 | my @parts = split /\s+/, $question; 61 | 62 | my $class = 'IN'; 63 | my $type = 'A'; 64 | my $name = $parts[-1]; 65 | my $path = join '.', reverse split /\./, $name; 66 | 67 | if( @parts == 3 ) { 68 | ($class,$type) = map { uc } @parts[0,1]; 69 | } 70 | 71 | # SQL Queries 72 | my %sql = ( 73 | question => q{ 74 | select id from question where class = ? and type = ? and name = ? 75 | }, 76 | zone => q{ 77 | select id from zone where path <@ ? 78 | }, 79 | clients_question => q{ 80 | select 81 | ip as client, min(q.query_ts) as first_ts, max(q.query_ts) as last_ts, count(1) as reference_count 82 | from meta_question mq 83 | inner join query q on mq.query_id = q.id 84 | inner join client c on q.client_id = c.id 85 | where 86 | mq.question_id = ? 87 | group by ip 88 | }, 89 | clients_zone => q{ 90 | select 91 | ip as client, min(q.query_ts) as first_ts, max(q.query_ts) as last_ts, count(1) as reference_count 92 | from zone_question zq 93 | inner join meta_question mq on zq.question_id = mq.question_id 94 | inner join query q on mq.query_id = q.id 95 | inner join client c on q.client_id = c.id 96 | where 97 | zq.zone_id = ? 98 | group by ip 99 | }, 100 | ); 101 | # Prepare SQL 102 | my $STH = $self->prepare_statements( \%sql ); 103 | 104 | # Find the query 105 | my $by; 106 | my $id; 107 | 108 | $STH->{question}->execute( $class, $type, $name ); 109 | if( $STH->{question}->rows > 0 ) { 110 | $by = 'question'; 111 | ($id) = $STH->{question}->fetchrow_array; 112 | } 113 | elsif( $STH->{zone}->execute($path) && $STH->{zone}->rows > 0 ) { 114 | $by = 'zone'; 115 | ($id) = $STH->{zone}->fetchrow_array; 116 | } 117 | 118 | $STH->{"clients_$by"}->execute($id) if defined $id; 119 | 120 | $self->stash( 121 | STH => $STH, 122 | name => $name, 123 | type => $type, 124 | class => $class, 125 | question => $question, 126 | by => $by, 127 | found => defined $id && defined $by, 128 | ); 129 | $self->render('utility/clients_asking'); 130 | } 131 | 132 | sub client_server_map { 133 | my ($self,$app) = @_; 134 | 135 | my %sql = ( 136 | conversations => q{ 137 | select 138 | cv.id as id, 139 | c.ip as client, 140 | c.id as client_id, 141 | s.ip as server, 142 | s.id as server_id, 143 | cv.reference_count as total 144 | from conversation cv 145 | inner join client c on c.id = cv.client_id 146 | inner join server s on s.id = cv.server_id 147 | } 148 | ); 149 | my $STH = $self->prepare_statements(\%sql); 150 | 151 | $STH->{conversations}->execute(); 152 | 153 | my @conversations = (); 154 | my %nodes = (); 155 | my $total = 0; 156 | while ( my $row = $STH->{conversations}->fetchrow_hashref ) { 157 | push @conversations, $row; 158 | foreach my $n (qw(client server)) { 159 | $nodes{$row->{$n}} ||= {}; 160 | my $hash = $nodes{$row->{$n}}; 161 | if( !exists $hash->{x} ) { 162 | my %c = _coords($row->{$n}); 163 | $hash->{x} = $c{x}; 164 | $hash->{y} = $c{y}; 165 | } 166 | $hash->{total} += $row->{total}; 167 | $total += $row->{total}; 168 | if( exists $hash->{color} && $hash->{color} eq '#FF0000' ){ 169 | $hash->{color} = '#FF0000'; 170 | } 171 | else { 172 | $hash->{color} = $n eq 'server' ? '#FF0000' : '#0174DF'; 173 | } 174 | } 175 | } 176 | 177 | foreach my $node (values %nodes) { 178 | my $ratio = $node->{total} / $total; 179 | $node->{size} = int($ratio * 10) + 1; 180 | } 181 | foreach my $cv (@conversations) { 182 | my $ratio = $cv->{total} / $total; 183 | $cv->{size} = int($ratio * 10) + 1; 184 | } 185 | 186 | $self->stash( 187 | conversations => \@conversations, 188 | nodes => \%nodes, 189 | ); 190 | } 191 | 192 | my $HALF_IP = 2 ** 16; 193 | sub _coords { 194 | my ($ip_str) = @_; 195 | my $ip = Net::IP->new( $ip_str ); 196 | my $as_int = $ip->intip; 197 | 198 | my %coords = ( 199 | x => $as_int % $HALF_IP, 200 | y => int( $as_int / $HALF_IP ), 201 | ); 202 | return wantarray ? %coords : \%coords; 203 | } 204 | 205 | 1; 206 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/anomaly/query.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::anomaly::query; 2 | # ABSTRACT: Calculate anomaly score for a query based on flags and opcodes 3 | 4 | use Const::Fast; 5 | use JSON::MaybeXS; 6 | use Moose; 7 | with qw( 8 | DreamCatcher::Role::Feather::Analysis 9 | DreamCatcher::Role::Feather::Anomaly 10 | ); 11 | use POSIX qw(strftime); 12 | 13 | sub _build_sql { 14 | return { 15 | check => q{ 16 | select 17 | s.ip as server, 18 | c.ip as client, 19 | q.id, 20 | q.opcode, 21 | q.count_questions, 22 | q.flag_recursive, 23 | q.flag_checking, 24 | q.flag_truncated, 25 | (select count(1) from meta_question where query_id = q.id) as actual_questions 26 | from query q 27 | inner join client c on q.client_id = c.id 28 | inner join server s on q.server_id = s.id 29 | left join anomaly_query aq on q.id = aq.id 30 | where 31 | query_ts > ? 32 | and aq.id is null 33 | }, 34 | insert => q{ 35 | insert into anomaly_query ( source, id, score, checks, results ) values ( 'query', ?, ?, ?, ? ) 36 | }, 37 | }; 38 | } 39 | 40 | sub analyze { 41 | my $self = shift; 42 | 43 | my %STH = map { $_ => $self->sth($_) } keys %{ $self->sql }; 44 | 45 | my $time = strftime("%FT%T", localtime(time() - $self->check_period)); 46 | $self->log(info => "anomaly::query checking queries since $time"); 47 | 48 | my $updates = 0; 49 | my $errors = 0; 50 | $STH{check}->execute($time); 51 | while( my $ent = $STH{check}->fetchrow_hashref ) { 52 | my $score = 0; 53 | my %analysis = (); 54 | 55 | # Check opcodes 56 | my $opcode_level = $self->anomaly_opcode($ent->{opcode}); 57 | if(defined $opcode_level && $opcode_level ne 'common') { 58 | $score += $self->score($opcode_level); 59 | my $key = sprintf "query_%s_opcode", $opcode_level; 60 | $analysis{$key} = $ent->{opcode}; 61 | } 62 | 63 | # Rare to have more than 1 question in a packet 64 | if( $ent->{count_questions} > 1 ) { 65 | my $type = $ent->{count_questions} > 10 ? 'malicious' : 'suspicious'; 66 | my $key = sprintf "query_%s_questions", $type; 67 | $score += $self->score($type); 68 | $analysis{$key} = $ent->{count_questions}; 69 | } 70 | if( $ent->{count_questions} != $ent->{actual_questions} ) { 71 | $score += $self->score('mismatch'); 72 | $analysis{query_mismatch_questions} = $ent->{actual_questions}; 73 | } 74 | 75 | # Rare to have all flags true 76 | if( $ent->{flag_recursive} && $ent->{flag_checking} && $ent->{flag_truncated} ) { 77 | $analysis{query_suspicious_flags} = 'all true'; 78 | $score += $self->score('suspicious'); 79 | } 80 | 81 | # Mark this as done 82 | my @checks = qw(questions opcodes flags); 83 | eval { $STH{insert}->execute($ent->{id},$score,\@checks,encode_json(\%analysis)) }; 84 | if(my $ex = $@) { 85 | $self->log(error => "anomaly::query - failed to score $ent->{query_id}: $ex"); 86 | $errors++; 87 | } 88 | else { 89 | $updates++; 90 | } 91 | } 92 | 93 | $self->log(info => "anomaly::query posted $updates updates, $errors errors"); 94 | } 95 | 96 | 97 | __PACKAGE__->meta->make_immutable; 98 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/anomaly/question.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::anomaly::question; 2 | # ABSTRACT: Calculate anomaly score for a question based on weirdness 3 | 4 | use Moose; 5 | use namespace::autoclean; 6 | with qw( 7 | DreamCatcher::Role::Feather::Analysis 8 | DreamCatcher::Role::Feather::Anomaly 9 | ); 10 | 11 | use JSON::MaybeXS; 12 | use POSIX qw(strftime); 13 | use Text::Soundex; 14 | use Text::Unidecode; 15 | 16 | sub _build_sql { 17 | return { 18 | check => q{ 19 | select 20 | q.id, 21 | q.class, 22 | q.type, 23 | q.name 24 | from question q 25 | left join anomaly_question aq on q.id = aq.id 26 | where 27 | aq.id is null 28 | }, 29 | insert => q{ 30 | insert into anomaly_question ( source, id, score, checks, results ) values ( 'question', ?, ?, ?, ? ) 31 | }, 32 | }; 33 | } 34 | 35 | sub analyze { 36 | my $self = shift; 37 | 38 | my %STH = map { $_ => $self->sth($_) } keys %{ $self->sql }; 39 | 40 | my $updates = 0; 41 | my $errors = 0; 42 | 43 | $STH{check}->execute(); 44 | while( my $ent = $STH{check}->fetchrow_hashref ) { 45 | my $score = 0; 46 | my %analysis = (); 47 | 48 | # Check classes 49 | my $class_level = $self->anomaly_class($ent->{class}); 50 | if((defined $class_level && $class_level ne 'common') || $ent->{class} =~ /^(?:CLASS)?([0-9]+)/ ) { 51 | my $id = $1; 52 | my $type = defined $class_level ? $class_level 53 | : defined($id) && $id >= 65280 && $id < 65535 ? 'private' 54 | : 'unassigned'; 55 | $score += $self->score($type); 56 | my $key = sprintf "question_%s_class", $type; 57 | $analysis{$key} = defined $id ? $id : $ent->{class}; 58 | } 59 | 60 | # Check Types 61 | my $type_level = $self->anomaly_type($ent->{type}); 62 | if((defined $type_level && $type_level ne 'common') || $ent->{type} =~ /^(?:TYPE)?([0-9]+)/ ) { 63 | my $id = $1; 64 | my $type = defined($type_level) ? $type_level 65 | : defined($id) && $id >= 65280 && $id < 65535 ? 'private' 66 | : defined($id) ? 'unassigned' 67 | : 'abnormal'; 68 | $score += $self->score($type); 69 | my $key = sprintf "question_%s_type", $type; 70 | $analysis{$key} = defined $id ? $id : $ent->{type}; 71 | } 72 | 73 | # Check for weird hostnames by length 74 | my $hostname = lc ($ent->{name}); 75 | my $name_length = defined $ent->{name} ? length $ent->{name} : 0; 76 | if( $name_length <= 1 || $name_length > 52 ) { 77 | my $type = $name_length <= 1 ? 'abnormal' 78 | : $name_length > 78 ? 'malicious' 79 | : 'suspicious'; 80 | my $key = sprintf "question_%s_length", $type; 81 | $score += $self->score($type); 82 | $analysis{$key} = $name_length; 83 | } 84 | 85 | # Check for lack of soundex outside of second-level domain 86 | if( my $short = $self->strip_sld($hostname) ) { 87 | my $valid_soundex=0; 88 | my $pieces=0; 89 | foreach my $piece ( grep { defined && length } split /[^a-z]+/, unidecode($short) ) { 90 | next unless length $piece >= 2; 91 | $pieces++; 92 | $valid_soundex++ if defined soundex($piece); 93 | } 94 | 95 | if( !$valid_soundex ) { 96 | my $type = $pieces > 10 ? 'malicious' 97 | : $pieces > 5 ? 'suspiciois' 98 | : 'abnormal'; 99 | $score += $self->score($type); 100 | $analysis{sprintf "question_%s_soundex", $type} = sprintf "short=%s,pieces=%s", $short, $pieces; 101 | } 102 | } 103 | 104 | # Mark this as done 105 | eval { $STH{insert}->execute($ent->{id},$score,[qw(classes types length soundex)],encode_json(\%analysis)) }; 106 | if(my $ex = $@) { 107 | $self->log(error => "anomaly::question - failed to score $ent->{id}: $ex"); 108 | $errors++; 109 | } 110 | else { 111 | $updates++; 112 | } 113 | } 114 | 115 | $self->log(info => "anomaly::question posted $updates updates, $errors errors"); 116 | } 117 | 118 | 119 | __PACKAGE__->meta->make_immutable; 120 | 1; 121 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/conversation.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::conversation; 2 | # ABSTRACT: Determine which conversation this packet belongs to. 3 | 4 | use strict; 5 | use warnings; 6 | use Moose; 7 | 8 | with qw( 9 | DreamCatcher::Role::Feather::Sniffer 10 | ); 11 | 12 | # Override default priority 13 | sub _build_priority { 1; } 14 | 15 | sub process { 16 | my ($self,$packet) = @_; 17 | 18 | # Conversations 19 | my $dbError = undef; 20 | my $sth = $self->dbh->run( fixup => sub { 21 | my $lsh; 22 | eval { 23 | $lsh = $_->prepare('select * from find_or_create_conversation( ?, ? )'); 24 | $lsh->execute( $packet->details->{client}, $packet->details->{server} ); 25 | }; 26 | if( my $err = $@ ) { 27 | $dbError = "find_or_create_conversation failed: " . join(' - ', ref $err, $err->errstr); 28 | } 29 | return $lsh; 30 | }); 31 | 32 | if( !defined $dbError && defined $sth && $sth->rows > 0 ) { 33 | # Set conversation id 34 | my $convo = $sth->fetchrow_hashref; 35 | $self->log(debug => "conversation bits: " . join( ",", map { "$_ => $convo->{$_}" } keys %{ $convo }) ); 36 | $packet->details->{client_id} = $convo->{client_id}; 37 | $packet->details->{server_id} = $convo->{server_id}; 38 | $packet->details->{conversation_id} = $convo->{id}; 39 | } 40 | else { 41 | $self->log(error => "failed conversation lookup: '$dbError'"); 42 | } 43 | } 44 | 45 | __PACKAGE__->meta->make_immutable; 46 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/list/meta.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::list::meta; 2 | # ABSTRACT: Link questions and answers to the list collection 3 | 4 | use Moose; 5 | with qw( 6 | DreamCatcher::Role::Feather::Analysis 7 | ); 8 | use POSIX qw(strftime); 9 | 10 | sub _build_sql { 11 | return { 12 | check => q{ 13 | select z.id as zone_id, l.id as list_entry_id, l.list_id as list_id 14 | from list_entry l 15 | inner join zone z on z.path <@ l.path 16 | }, 17 | check_answer => q{select answer_id from zone_answer where zone_id = ?}, 18 | check_question => q{select question_id from zone_question where zone_id = ?}, 19 | insert_answer => q{insert into list_meta_answer ( answer_id, list_entry_id, list_id ) values ( ?, ?, ? )}, 20 | insert_question => q{insert into list_meta_question ( question_id, list_entry_id, list_id ) values ( ?, ?, ? )}, 21 | }; 22 | } 23 | 24 | sub analyze { 25 | my ($self) = @_; 26 | 27 | my %STH = map { $_ => $self->sth($_) } keys %{ $self->sql }; 28 | 29 | $STH{check}->execute(); 30 | my $updates = 0; 31 | while( my $ent = $STH{check}->fetchrow_hashref ) { 32 | foreach my $type (qw(question answer)) { 33 | $STH{"check_$type"}->execute( $ent->{zone_id} ); 34 | while ( my ($id) = $STH{"check_$type"}->fetchrow_array ) { 35 | eval { 36 | $STH{"insert_$type"}->execute( $id, $ent->{list_entry_id}, $ent->{list_id} ); 37 | }; 38 | $updates++ unless $@; 39 | } 40 | } 41 | } 42 | $self->log( info => "list::meta posted $updates updates"); 43 | } 44 | 45 | __PACKAGE__->meta->make_immutable; 46 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/list/refresh.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::list::refresh; 2 | # ABSTRACT: Perform list based refreshes 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use LWP::Simple; 8 | use Moose; 9 | with qw( 10 | DreamCatcher::Role::Feather::Analysis 11 | ); 12 | 13 | 14 | # Run check once an hour 15 | sub _build_interval { 3600; } 16 | 17 | sub _build_sql { 18 | return { 19 | check => q{ 20 | select 21 | id, 22 | refresh_url, 23 | refresh_last_ts 24 | from list 25 | where can_refresh = true 26 | and (refresh_last_ts IS NULL OR NOW() - refresh_last_ts > refresh_every) 27 | }, 28 | refresh_entry => q{select refresh_list_entry(?, ?, ?)}, 29 | unset_refresh => q{update list_entry set refreshed = false where list_id = ?}, 30 | }; 31 | } 32 | 33 | sub analyze { 34 | my ($self) = @_; 35 | 36 | my %STH = map { $_ => $self->sth($_) } keys %{ $self->sql }; 37 | 38 | $STH{check}->execute(); 39 | 40 | my $total = 0; 41 | while(my $list = $STH{check}->fetchrow_hashref) { 42 | # Attempt to pull the refresh 43 | my $content = get($list->{refresh_url}); 44 | if(!defined $content) { 45 | $self->log(warn => sprintf "Refreshing failed %s, no content.", $list->{refresh_url}); 46 | next; 47 | } 48 | $self->log(info => sprintf "list_refresh: %d from %s produced %d bytes.", $list->{id}, $list->{refresh_url}, length($content)); 49 | my $entries = 0; 50 | # Set refresh to false 51 | $STH{unset_refresh}->execute($list->{id}); 52 | foreach my $entry (split /(?:\r?\n)+/, $content) { 53 | $entry ||= ''; 54 | $entry =~ s/\s*#.*//g; 55 | next unless length $entry; 56 | 57 | # Split based on commas, semi-coloons, and spaces 58 | my @cols = split /[\s,;]+/, $entry; 59 | next unless @cols; 60 | 61 | # The last column is the one we tend to want courtesy MDL 62 | my $zone = $cols[-1]; 63 | 64 | # Path translation 65 | my $path = join('.', reverse split /\./, $zone); 66 | $path =~ s/\-/_/g; 67 | next if $path =~ /[^a-zA-Z0-9.\_]/; # TODO: utf8 handling 68 | 69 | $self->log(debug => sprintf 'list::refresh [%d] %s <-> %s', $list->{id}, $zone, $path); 70 | eval { $STH{refresh_entry}->execute($list->{id}, $zone, $path); }; 71 | next if $@; 72 | $entries++; 73 | $total++; 74 | } 75 | $self->log(info => sprintf "list::refresh for list_id:%d has %d entries.", $list->{id}, $entries); 76 | } 77 | $self->log(info => sprintf "list::refresh updated %d entries.", $total); 78 | } 79 | 80 | __PACKAGE__->meta->make_immutable; 81 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/list/tracking.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::list::tracking; 2 | # ABSTRACT: Track clients hitting certain lists 3 | 4 | use Moose; 5 | with qw( 6 | DreamCatcher::Role::Feather::Analysis 7 | ); 8 | use POSIX qw(strftime); 9 | 10 | sub _build_check_period { 86400*30 } 11 | 12 | sub _build_sql { 13 | return { 14 | questions => q{ 15 | select c.id as client_id, 16 | lmq.list_id, 17 | max(q.query_ts) as last_ts, 18 | min(q.query_ts) as first_ts, 19 | count(1) as reference_count 20 | from list_meta_question lmq 21 | inner join meta_question mq on lmq.question_id = mq.question_id 22 | inner join query q on mq.query_id = q.id 23 | inner join client c on q.client_id = c.id 24 | where 25 | q.query_ts > ? 26 | group by lmq.list_id, c.id 27 | }, 28 | answers => q{ 29 | select c.id as client_id, 30 | lma.list_id, 31 | max(r.response_ts) as last_ts, 32 | min(r.response_ts) as first_ts, 33 | count(1) as reference_count 34 | from list_meta_answer lma 35 | inner join meta_answer ma on lma.answer_id = ma.answer_id 36 | inner join response r on ma.response_id = r.id 37 | inner join client c on r.client_id = c.id 38 | where 39 | r.response_ts > ? 40 | group by lma.list_id, c.id 41 | }, 42 | client => q{select list_tracking_client(?,?,?,?,?)}, 43 | }; 44 | } 45 | 46 | sub analyze { 47 | my ($self) = @_; 48 | 49 | my $check_ts = strftime('%FT%T',localtime(time - $self->check_period)); 50 | $self->log(debug => 51 | sprintf "list::tracking starting analysis for past %d seconds (from %s), max %d records.", 52 | $self->check_period, 53 | $check_ts, 54 | $self->batch_max, 55 | ); 56 | my $updates = 0; 57 | my $errors = 0; 58 | 59 | my %STH = map { $_ => $self->sth($_) } keys %{ $self->sql }; 60 | my %results = (); 61 | foreach my $type (qw(answers questions)) { 62 | $STH{questions}->execute($check_ts); 63 | while( my $ent = $STH{questions}->fetchrow_hashref ) { 64 | my $id = join(":", $ent->{list_id}, $ent->{client_id}); 65 | if(exists $results{$id}) { 66 | $results{$id}->{reference_count} += $ent->{reference_count}; 67 | $results{$id}->{first_ts} = $results{$id}->{first_ts} lt $ent->{first_ts} ? $ent->{first_ts} 68 | : $results{$id}->{first_ts}; 69 | $results{$id}->{last_ts} = $results{$id}->{last_ts} gt $ent->{last_ts} ? $ent->{last_ts} 70 | : $results{$id}->{last_ts}; 71 | } 72 | else { 73 | $results{$id} = { 74 | reference_count => $ent->{reference_count}, 75 | first_ts => $ent->{first_ts}, 76 | last_ts => $ent->{last_ts}, 77 | }; 78 | } 79 | } 80 | 81 | } 82 | foreach my $id (keys %results) { 83 | my ($list_id,$client_id) = split ':', $id; 84 | eval { 85 | $STH{client}->execute($list_id,$client_id, @{$results{$id}}{qw(first_ts last_ts reference_count)}); 86 | $updates++; 87 | 1; 88 | } or do { 89 | $self->log(debug => "ERROR: $@"); 90 | $errors++; 91 | }; 92 | } 93 | 94 | $self->log(info => "list::tracking posted $updates updates, $errors errors."); 95 | } 96 | 97 | __PACKAGE__->meta->make_immutable; 98 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/query/response.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::query::response; 2 | # ABSTRACT: Link query and responses not found in the sniffer 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Moose; 8 | use namespace::autoclean; 9 | 10 | use DateTime; 11 | use DateTime::Format::Pg; 12 | use POSIX qw(strftime); 13 | 14 | with qw( 15 | DreamCatcher::Role::Feather::Analysis 16 | ); 17 | 18 | sub _build_interval { 60 } 19 | sub _build_check_period { 3600*12 } 20 | sub _build_batch_max { 5_000 } 21 | 22 | sub _build_sql { 23 | return { 24 | null_response => q{ 25 | select q.* from query q 26 | left join meta_query_response m on q.id = m.query_id 27 | where m.response_id is null 28 | and q.query_ts > ? 29 | order by q.query_ts limit ? 30 | }, 31 | find_response => q{ 32 | select * from response 33 | where conversation_id = ? 34 | and query_serial = ? 35 | and response_ts between ? and ? 36 | }, 37 | set_response => q{ 38 | select link_query_response( ?, ?, ?, ? ) 39 | }, 40 | find_questions => q{ 41 | select mq.question_id, ma.answer_id, count(1) as reference_count, min(query_ts) as first_ts, max(response_ts) as last_ts 42 | from meta_question mq 43 | inner join meta_query_response mqr on mq.query_id = mqr.query_id 44 | inner join meta_answer ma on mqr.response_id = ma.response_id and ma.section = 'answer' 45 | inner join response r on mqr.response_id = r.id 46 | inner join query q on mqr.query_id = q.id 47 | where mqr.query_id = ? 48 | and r.status = 'NOERROR' 49 | group by mq.question_id, ma.answer_id 50 | }, 51 | link_question_answer => q{ 52 | select link_question_answer(?, ?, ?, ?, ?) 53 | }, 54 | }; 55 | } 56 | 57 | sub analyze { 58 | my ($self) = @_; 59 | 60 | $self->log(debug => sprintf "query::response starting analysis for past %d seconds, max %d records.", 61 | $self->check_period, 62 | $self->batch_max, 63 | ); 64 | 65 | my $check_ts = strftime('%FT%T',localtime(time - $self->check_period)); 66 | 67 | my %STH = map { $_ => $self->sth($_) } qw( 68 | null_response find_response set_response 69 | find_questions link_question_answer 70 | ); 71 | 72 | $STH{null_response}->execute( $check_ts, $self->batch_max ); 73 | $self->log(debug => sprintf "null_response provided %d records to check.", $STH{null_response}->rows); 74 | 75 | my $updates = 0; 76 | my $errors = 0; 77 | my $subdates = 0; 78 | while( my $q = $STH{null_response}->fetchrow_hashref ) { 79 | my $qt = DateTime::Format::Pg->parse_datetime( $q->{query_ts} ); 80 | # Find the response 81 | eval { 82 | $STH{find_response}->execute( $q->{conversation_id}, $q->{query_serial}, 83 | $qt->clone->datetime, 84 | $qt->clone()->add( seconds => 10 )->datetime 85 | ); 86 | }; 87 | if( my $ex = $@ ) { 88 | $self->log(error => "Lookup for a matching query/response for $q->{id}"); 89 | $self->log(debug => "ERROR: $ex"); 90 | next; 91 | } 92 | # If we found 1, do something! 93 | if( $STH{find_response}->rows == 1 ) { 94 | my($r) = $STH{find_response}->fetchrow_hashref; 95 | my @args = ( 96 | $q->{id}, 97 | $r->{id}, 98 | $q->{conversation_id}, 99 | defined $r->{capture_time} && defined $q->{capture_time} ? $r->{capture_time} - $q->{capture_time} : undef 100 | ); 101 | eval { 102 | $STH{set_response}->execute(@args); 103 | }; 104 | if( my $ex = $@ ) { 105 | $errors++; 106 | $self->log(debug => sprintf "ERROR: Linking queries(%s): %s", join(',', @args), $ex); 107 | next; 108 | } 109 | else { 110 | # Do question/answer discovery 111 | $STH{find_questions}->execute($q->{id}); 112 | $self->log(debug => sprintf "- query_id:%d found %d questions with answers.", $q->{id}, $STH{find_questions}->rows); 113 | 114 | while( my @fields = $STH{find_questions}->fetchrow_array ) { 115 | my $rc = eval { 116 | $STH{link_question_answer}->execute(@fields); 117 | 1; 118 | }; 119 | if(!defined $rc || $rc != 1) { 120 | $self->log(debug => "ERROR: $@") 121 | } else { 122 | $subdates++; 123 | } 124 | } 125 | } 126 | $updates++; 127 | } 128 | elsif( $STH{find_response}->rows > 0 ) { 129 | $self->log(info => sprintf "Attempting to find responses for query_id:%d yielded %d results.", 130 | $q->{id}, 131 | $STH{find_response}->rows 132 | ); 133 | } 134 | } 135 | 136 | $_->finish for values %STH; 137 | $self->log(info => sprintf "query::response posted %d updates and %d errors, %d questions/answers linked", $updates, $errors, $subdates); 138 | } 139 | 140 | __PACKAGE__->meta->make_immutable; 141 | no Moose; 142 | 1; 143 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/store.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::store; 2 | # ABSTRACT: Store the DNS packet in the database. 3 | 4 | use strict; 5 | use warnings; 6 | use Moose; 7 | 8 | with qw( 9 | DreamCatcher::Role::Feather::Sniffer 10 | DreamCatcher::Role::RRData 11 | ); 12 | 13 | sub _build_parent { 'conversation'; } 14 | 15 | # Queries we'd like the cache 16 | sub _build_sql { 17 | return { 18 | query => q{select add_query( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )}, 19 | question => q{select find_or_create_question( ?, ?, ?, ? ) }, 20 | response => q{select add_response( ?, ?, ?, ?, ?, ?, ?, 21 | ?, ?, ?, ?, ?, ?, ?, 22 | ?, ?, ?, ?, ?, ? )}, 23 | answer => q{select find_or_create_answer( ?, ?, ?, ?, ?, ?, ?, ? )}, 24 | }; 25 | } 26 | 27 | sub process { 28 | my ( $self,$packet ) = @_; 29 | 30 | # Packet ID 31 | my $dnsp = $packet->dns; 32 | my $info = $packet->details; 33 | 34 | # Check for query/response 35 | if( $dnsp->header->qr ) { 36 | # Answer 37 | my $sth = $self->sth('response'); 38 | eval { 39 | $sth->execute( 40 | $info->{conversation_id}, 41 | $info->{client_id}, 42 | $info->{client_port}, 43 | $info->{server_id}, 44 | $info->{server_port}, 45 | $dnsp->header->id, 46 | $dnsp->header->opcode, 47 | $dnsp->header->rcode, 48 | $dnsp->answersize, 49 | $dnsp->header->ancount, 50 | $dnsp->header->arcount, 51 | $dnsp->header->nscount, 52 | $dnsp->header->qdcount, 53 | $dnsp->header->aa, 54 | $dnsp->header->ad, 55 | $dnsp->header->tc, 56 | $dnsp->header->cd, 57 | $dnsp->header->rd, 58 | $dnsp->header->ra, 59 | $info->{time}, 60 | ); 61 | }; 62 | if( my $ex = $@ ) { 63 | $self->log(error => "store failed: $ex->errstr"); 64 | return; 65 | } 66 | 67 | my ($response_id) = $sth->fetchrow_array; 68 | return unless defined $response_id && $response_id > 0; 69 | 70 | 71 | my @sets = (); 72 | foreach my $section (qw(answer additional authority pre prerequisite update zone)) { 73 | my @records = (); 74 | eval { 75 | no strict; 76 | @records = $dnsp->$section(); 77 | }; 78 | if( @records ) { 79 | push @sets, { name => $section eq 'pre' ? 'prerequisite' : $section, rr => \@records }; 80 | } 81 | } 82 | foreach my $set ( @sets ) { 83 | foreach my $pa ( @{ $set->{rr} } ) { 84 | my %data = $self->rr_data( $pa ); 85 | 86 | next unless defined $data{value} && length $data{value}; 87 | 88 | my $lsh = $self->sth('answer'); 89 | eval { 90 | $lsh->execute( 91 | $response_id, 92 | $set->{name}, 93 | $data{ttl}, 94 | $pa->class, 95 | $pa->type, 96 | $pa->name, 97 | $data{value}, 98 | $data{opts}, 99 | ); 100 | }; 101 | if( my $ex = $@ ) { 102 | $self->log(error => "Attempt to create answer failed: $ex->errstr"); 103 | next; 104 | } 105 | } 106 | } 107 | } 108 | else { 109 | # Query 110 | my $sth = $self->sth('query'); 111 | eval { 112 | $sth->execute( 113 | $info->{conversation_id}, 114 | $info->{client_id}, 115 | $info->{client_port}, 116 | $info->{server_id}, 117 | $info->{server_port}, 118 | $dnsp->header->id, 119 | $dnsp->header->opcode, 120 | $dnsp->header->qdcount, 121 | $dnsp->header->rd, 122 | $dnsp->header->tc, 123 | $dnsp->header->cd, 124 | $info->{time}, 125 | ); 126 | }; 127 | if (my $ex = $@) { 128 | $self->log(error => "Failed to create query object: $ex->errstr"); 129 | return; 130 | } 131 | 132 | my ($query_id) = $sth->fetchrow_array; 133 | return unless defined $query_id && $query_id > 0; 134 | 135 | # Tag questions 136 | foreach my $pq ( $dnsp->question ) { 137 | my $lsh = $self->sth('question'); 138 | eval { 139 | $lsh->execute( 140 | $query_id, 141 | $pq->qclass, 142 | $pq->qtype, 143 | $pq->qname 144 | ); 145 | }; 146 | } 147 | } 148 | } 149 | 150 | __PACKAGE__->meta->make_immutable; 151 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feather/zone/discovery.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feather::zone::discovery; 2 | # ABSTRACT: Populates zone tables 3 | 4 | use Moose; 5 | with qw( 6 | DreamCatcher::Role::Feather::Analysis 7 | ); 8 | 9 | sub _build_sql { 10 | return { 11 | answer => q{ 12 | select a.* from answer a 13 | left join zone_answer z on a.id = z.answer_id 14 | where 15 | a.class NOT LIKE 'CLASS%' 16 | and a.type NOT LIKE 'TYPE%' 17 | and z.answer_id is null 18 | order by first_ts asc 19 | }, 20 | question => q{ 21 | select q.* from question q 22 | left join zone_question z on q.id = z.question_id 23 | where 24 | q.class NOT LIKE 'CLASS%' 25 | and q.type NOT LIKE 'TYPE%' 26 | and z.question_id is null 27 | order by first_ts asc 28 | }, 29 | zone_id => q{select get_zone_id( ?, ?, ?, ? )}, 30 | link_zone_answer => q{select link_zone_answer( ?, ? )}, 31 | link_zone_question => q{select link_zone_question( ?, ? )}, 32 | }; 33 | } 34 | 35 | sub analyze { 36 | my ($self) = @_; 37 | 38 | my %stats = map { $_ => 0 } qw(questions answers); 39 | 40 | # DB Connections 41 | my %STH = map { $_ => $self->sth($_) } keys %{ $self->sql }; 42 | 43 | # Get questions 44 | $STH{question}->execute(); 45 | while(my $q = $STH{question}->fetchrow_hashref) { 46 | my ($name,$zone) = split( /\./, $q->{name}, 2 ); 47 | if (! defined $zone ) { 48 | $self->log(debug => "error parsing zone for $q->{id} $q->{class} $q->{type} $q->{name}"); 49 | next; 50 | } 51 | my @path = split( /\./, $zone ); 52 | # We want SLD's at the lowest level 53 | unshift @path, $name if @path == 1; 54 | my $path = join('.', reverse @path); 55 | $path =~ s/\-/_/g; 56 | 57 | next if $path =~ /[^a-zA-Z0-9.\_]/; # TODO: utf8 handling 58 | 59 | $STH{zone_id}->execute( $zone, $path, $q->{first_ts}, $q->{last_ts} ); 60 | my ($zone_id) = $STH{zone_id}->fetchrow_array; 61 | next unless defined $zone_id and $zone_id > 0; 62 | eval { $STH{link_zone_question}->execute( $zone_id, $q->{id} ); }; 63 | if(my $ex = $@) { 64 | $self->log(error => sprintf "zone::discovery: linking zone_id(%d) to question(%d) failed: %s", 65 | $zone_id, 66 | $q->{id}, 67 | $ex->errstr 68 | ); 69 | $stats{errors}++; 70 | } 71 | else { 72 | $stats{questions}++; 73 | } 74 | } 75 | $self->log(info => "zone::discovery: questions linked $stats{questions}"); 76 | 77 | # Get answers 78 | $STH{answer}->execute(); 79 | while(my $q = $STH{answer}->fetchrow_hashref) { 80 | foreach my $field ( qw( name value ) ) { 81 | next if $q->{$field} =~ /(\d{1,3}\.){3}\d{1,3}/; 82 | my ($name,$zone) = map { lc } split( /\./, $q->{$field}, 2 ); 83 | if (! defined $zone || ! length $zone ) { 84 | $self->log(debug => "error parsing zone for $q->{id} $q->{class} $q->{type} $q->{name}"); 85 | next; 86 | } 87 | my @path = split( /\./, $zone ); 88 | 89 | # We want SLD's at the lowest level 90 | unshift @path, $name if @path == 1; 91 | 92 | my $path = join('.', reverse @path ); 93 | $path =~ s/\-/_/g; 94 | next if $path =~ /[^a-zA-Z0-9.\-]/; # TODO: utf8 handling 95 | 96 | $STH{zone_id}->execute( $zone, $path, $q->{first_ts}, $q->{last_ts} ); 97 | my ($zone_id) = $STH{zone_id}->fetchrow_array; 98 | next unless defined $zone_id and $zone_id > 0; 99 | eval { $STH{link_zone_answer}->execute( $zone_id, $q->{id} ); }; 100 | if(my $ex = $@) { 101 | $self->log(error => sprintf "zone::discovery: linking zone_id(%d) to answer(%d) failed: %s", 102 | $zone_id, 103 | $q->{id}, 104 | $ex->errstr 105 | ); 106 | $stats{errors}++; 107 | } 108 | else { 109 | $stats{answers}++; 110 | } 111 | } 112 | } 113 | $self->log(info => "zone::discovery: answers linked $stats{answers}"); 114 | } 115 | 1; 116 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Feathers.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Feathers; 2 | 3 | use Moose; 4 | 5 | use Tree::DAG_Node; 6 | use Module::Pluggable ( 7 | instantiate => 'new', 8 | search_path => 'DreamCatcher::Feather', 9 | ); 10 | 11 | # Attributes 12 | has 'tree' => ( 13 | is => 'ro', 14 | isa => 'Tree::DAG_Node', 15 | lazy => 1, 16 | builder => '_build_tree', 17 | ); 18 | # DAG Tree for determining plugin ordering 19 | sub _build_tree { 20 | my $self = shift; 21 | my $tree = Tree::DAG_Node->new(); 22 | $tree->name("DreamCatcher::Feathers"); 23 | 24 | # Build the feathers list 25 | my $F = $self->hash; 26 | my @feathers = map { { tries => 0, obj => $F->{$_} } } sort { $F->{$a}->priority <=> $F->{$b}->priority } keys %{ $F }; 27 | 28 | # Object cache 29 | my %objects = (); 30 | 31 | # Cycle through the feathers 32 | while ( my $feather = shift @feathers ) { 33 | my $node; 34 | my $obj = $feather->{obj}; 35 | 36 | # Skip feathers that are disabled 37 | next unless $obj->enabled; 38 | 39 | if( $obj->parent eq 'none' ) { 40 | $node = $tree->new_daughter(); 41 | } 42 | elsif( exists $objects{$obj->parent} ) { 43 | $node = $objects{$obj->parent}->new_daughter(); 44 | } 45 | else { 46 | # Retry, may be out of order 47 | $feather->{tries}++; 48 | if($feather->{tries} > 3) { 49 | warn sprintf "DreamCatcher::Feathers::_build_tree failed to load %s, tries exceeded.", $obj->name; 50 | } 51 | else { 52 | push @feathers, $feather; 53 | } 54 | next; 55 | } 56 | $node->name($obj->name); 57 | $objects{$obj->name} = $node; 58 | } 59 | return $tree; 60 | } 61 | 62 | has 'config' => ( 63 | is => 'ro', 64 | isa => 'HashRef', 65 | init_arg => 'Config', 66 | ); 67 | 68 | has 'log_callback' => ( 69 | is => 'ro', 70 | isa => 'CodeRef', 71 | default => sub { my $l = sub { warn join(": ", @_), "\n" }; return $l; }, 72 | init_arg => 'Log', 73 | ); 74 | 75 | has 'hash' => ( 76 | is => 'ro', 77 | isa => 'HashRef', 78 | lazy => 1, 79 | builder => '_build_hash' 80 | ); 81 | sub _build_hash { 82 | my $self = shift; 83 | return { map { $_->name => $_ } grep { $_->enabled } $self->plugins( Config => $self->config, Log => $self->log_callback ) }; 84 | } 85 | 86 | has 'schedule' => ( 87 | is => 'ro', 88 | isa => 'HashRef', 89 | lazy => 1, 90 | builder => '_build_schedule' 91 | ); 92 | sub _build_schedule { 93 | my ($self) = @_; 94 | my %sched = map { $_->name => $_->interval } @{ $self->chain('analysis') }; 95 | return \%sched; 96 | } 97 | 98 | # Order the plugins using their "parent" attributes 99 | sub chain { 100 | my ($self,$function) = @_; 101 | my $F = $self->hash; 102 | return [ map { $F->{$_} } map { $_->name } grep { defined $function ? $F->{$_->name}->function eq $function : 1; } $self->tree->descendants ]; 103 | } 104 | 105 | # Run the processing 106 | sub process { 107 | my ($self,$packet) = @_; 108 | 109 | # Skip junk data 110 | if( !defined $packet || !$packet->valid ) { 111 | return; 112 | } 113 | foreach my $feather (@{ $self->chain('sniffer') }) { 114 | $feather->process($packet); 115 | } 116 | return 1; 117 | } 118 | 119 | 120 | __PACKAGE__->meta->make_immutable; 121 | 1; 122 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Helpers.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Helpers; 2 | # ABSTRACT: Mojolicious Helpers for DreamCatcher 3 | use strict; 4 | use warnings; 5 | 6 | use base 'Mojolicious::Plugin'; 7 | 8 | use Const::Fast; 9 | use HTML::Entities; 10 | 11 | const my %helpers => ( 12 | make_badge => \&make_badge, 13 | ); 14 | 15 | sub register { 16 | my($self,$app) = @_; 17 | $app->helper( %helpers ); 18 | } 19 | 20 | const my %_BADGES => ( 21 | query_status => { 22 | NOERROR => 'badge-success', 23 | NXDOMAIN => 'badge-warning', 24 | SERVFAIL => 'badge-important', 25 | REFUSED => 'badge-important', 26 | }, 27 | ); 28 | 29 | sub make_badge { 30 | my ($self,$type,$state) = @_; 31 | 32 | return '' unless defined $state and length $state; 33 | 34 | # Make safe for the == 35 | $state = encode_entities( decode_entities ( $state ) ); 36 | return $state unless exists $_BADGES{$type}; 37 | 38 | my $class = exists $_BADGES{$type}{$state} ? $_BADGES{$type}{$state} : undef; 39 | # Make sa 40 | return defined $class ? sprintf('%s', $class, $state) : $state; 41 | } 42 | 43 | # Return True; 44 | 1; 45 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Packet.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Packet; 2 | 3 | # ABSTRACT: DreamCatcher Packet Parsing object 4 | use Moose; 5 | use namespace::autoclean; 6 | 7 | # Packet Parsing 8 | use NetPacket::Ethernet qw(:types); 9 | use NetPacket::IP qw(:strip :protos); 10 | use NetPacket::IPv6; 11 | use NetPacket::UDP; 12 | use NetPacket::TCP; 13 | 14 | # DNS Decoding 15 | use Net::DNS::SEC; 16 | use Net::DNS::Packet; 17 | 18 | # The Raw Packet off the wire 19 | has 'raw_packet' => ( 20 | is => 'ro', 21 | isa => 'ArrayRef', 22 | required => 1, 23 | init_arg => 'Raw', 24 | ); 25 | # Process the packet, ready for other attributes 26 | has 'raw_data' => ( 27 | is => 'ro', 28 | init_arg => undef, 29 | lazy => 1, 30 | builder => '_build_raw_data', 31 | ); 32 | # Is the packet valid 33 | has 'valid' => ( 34 | is => 'ro', 35 | isa => 'Bool', 36 | init_arg => undef, 37 | lazy => 1, 38 | builder => '_build_valid', 39 | ); 40 | # Details Extracted by BUILD 41 | has 'details' => ( 42 | is => 'rw', 43 | isa => 'HashRef', 44 | init_arg => undef, 45 | lazy => 1, 46 | builder => '_build_details', 47 | ); 48 | # Net::DNS::Packet Object 49 | has 'dns' => ( 50 | is => 'ro', 51 | isa => 'Net::DNS::Packet', 52 | init_arg => undef, 53 | lazy => 1, 54 | builder => '_build_dns', 55 | ); 56 | # Errors with this packet 57 | has 'error' => ( 58 | is => 'ro', 59 | isa => 'Any', 60 | init_arg => undef, 61 | lazy => 1, 62 | builder => '_build_error', 63 | ); 64 | 65 | # BUILDERS FOR ATTRIBUTES 66 | sub _get_data { 67 | my $self = shift; 68 | my $field = shift; 69 | my $data = $self->raw_data(); 70 | return exists $data->{$field} ? $data->{$field} : undef; 71 | } 72 | sub _build_valid { 73 | my $self = shift; 74 | return $self->_get_data( 'valid' ); 75 | } 76 | sub _build_details { 77 | my $self = shift; 78 | my $dt = $self->_get_data( 'details' ); 79 | return defined $dt ? $dt : {}; 80 | } 81 | sub _build_dns { 82 | my $self = shift; 83 | return $self->_get_data( 'dns' ); 84 | } 85 | sub _build_error { 86 | my $self = shift; 87 | return $self->_get_data( 'error' ); 88 | } 89 | sub _build_raw_data { 90 | my $self = shift; 91 | my %data = ( 92 | details => undef, 93 | dns => undef, 94 | valid => 0, 95 | ); 96 | 97 | # Begin Decoding 98 | my ($hdr,$packet) = @{ $self->raw_packet }; 99 | my $eth_pkt = NetPacket::Ethernet->decode($packet); 100 | my $ip_pkt = $eth_pkt->{type} == ETH_TYPE_IP ? NetPacket::IP->decode($eth_pkt->{data}) 101 | : $eth_pkt->{type} == ETH_TYPE_IPv6 ? NetPacket::IPv6->decode($eth_pkt->{data}) 102 | : undef; 103 | 104 | return { %data, error => "NetPacket decode failed!" } unless defined $ip_pkt; 105 | return { %data, error => "NetPacket decode failed, no protocol!" } unless $ip_pkt->{proto}; 106 | 107 | # NetPacket::IPv6 header for L4 proto is called "nxt" 108 | if( $eth_pkt->{type} == ETH_TYPE_IPv6 ) { 109 | $ip_pkt->{proto} = $ip_pkt->{nxt}; 110 | } 111 | 112 | # Transport Layer Processing 113 | my $layer4 = undef; 114 | if( $ip_pkt->{proto} == IP_PROTO_UDP ) { 115 | $layer4 = NetPacket::UDP->decode( $ip_pkt->{data} ); 116 | $data{details}->{proto} = 'udp'; 117 | } 118 | elsif ( $ip_pkt->{proto} == IP_PROTO_TCP ) { 119 | $layer4 = NetPacket::TCP->decode( $ip_pkt->{data} ); 120 | $data{details}->{proto} = 'tcp'; 121 | } 122 | else { 123 | # Bail before referencing this data 124 | return { %data, error => "Decode failed, not TCP or UDP" }; 125 | } 126 | 127 | # Informations! 128 | $data{details}->{bytes} = length $packet; 129 | $data{details}->{time} = join('.', $hdr->{tv_sec}, sprintf("%0.6d", $hdr->{tv_usec}) ); 130 | $data{details}->{src_ip} = $ip_pkt->{src_ip}; 131 | $data{details}->{dest_ip} = $ip_pkt->{dest_ip}; 132 | $data{details}->{src_port} = $layer4->{src_port}; 133 | $data{details}->{dest_port} = $layer4->{dest_port}; 134 | 135 | # Parse DNS Packet 136 | my $dnsp = undef; 137 | eval { 138 | $dnsp = Net::DNS::Packet->new( \$layer4->{data} ); 139 | }; 140 | return { %data, error => "Net::DNS unable to parse DNS Packet" } unless defined $dnsp; 141 | 142 | # Server Accounting. 143 | $data{details}->{qa} = $dnsp->header->qr ? 'answer' : 'question'; 144 | 145 | if( $data{details}->{qa} eq 'answer' ) { 146 | $data{details}->{server} = $data{details}->{src_ip}; 147 | $data{details}->{server_port} = $data{details}->{src_port}; 148 | $data{details}->{client} = $data{details}->{dest_ip}; 149 | $data{details}->{client_port} = $data{details}->{dest_port}; 150 | } 151 | else { 152 | $data{details}->{server} = $data{details}->{dest_ip}; 153 | $data{details}->{server_port} = $data{details}->{dest_port}; 154 | $data{details}->{client} = $data{details}->{src_ip}; 155 | $data{details}->{client_port} = $data{details}->{src_port}; 156 | } 157 | 158 | $data{valid} = 1; 159 | $data{dns} = $dnsp; 160 | 161 | return \%data; 162 | } 163 | 164 | __PACKAGE__->meta->make_immutable; 165 | 1; 166 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Anomaly.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Anomaly; 2 | 3 | use Moose::Role; 4 | use namespace::autoclean; 5 | 6 | # Interface to Implement 7 | requires qw( 8 | dbh 9 | target 10 | check_name 11 | check 12 | ); 13 | 14 | sub create_table { 15 | my $self = shift; 16 | } 17 | 18 | no Moose::Role; 19 | # Return True 20 | 1; 21 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Anomaly/Query.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Anomaly::Query; 2 | 3 | use Moose::Role; 4 | 5 | has 'src_table' => ( 6 | is => 'ro', 7 | default => 'packet_query', 8 | ); 9 | 10 | # Return True 11 | 1; 12 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Cache.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Cache; 2 | # ABSTRACT: Provides a caching API for the feathers 3 | 4 | use Moose::Role; 5 | use namespace::autoclean; 6 | use CHI; 7 | use Cache::FastMmap; 8 | 9 | has 'cache' => ( 10 | is => 'ro', 11 | lazy => 1, 12 | builder => '_build_cache', 13 | init_arg => undef, 14 | ); 15 | 16 | sub _build_cache { 17 | my ($self) = @_; 18 | 19 | return CHI->new(driver => 'FastMmap', namespace => $self->name, expires_in => 30); 20 | } 21 | 22 | no Moose::Role; 23 | # Return TRUE 24 | 1; 25 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/DBH.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::DBH; 2 | # ABSTRACT: Provides the database connection for the feathers 3 | 4 | use Moose::Role; 5 | use namespace::autoclean; 6 | 7 | use DBIx::Connector; 8 | use Exception::Class::DBI; 9 | 10 | with qw( 11 | DreamCatcher::Role::Logger 12 | DreamCatcher::Role::Cache 13 | ); 14 | 15 | has 'dbh' => ( 16 | is => 'ro', 17 | lazy => 1, 18 | builder => '_build_dbh', 19 | init_arg => undef, 20 | ); 21 | 22 | has 'sql' => ( 23 | is => 'rw', 24 | isa => 'HashRef', 25 | builder => '_build_sql', 26 | ); 27 | 28 | sub _build_sql { {} } 29 | 30 | sub sth { 31 | my ($self,$name) = @_; 32 | 33 | if( !exists $self->sql->{$name} ) { 34 | $self->log(error => "attempt to load unknown statement:$name"); 35 | return; 36 | } 37 | 38 | return $self->dbh->run( fixup => sub { 39 | my $sth; 40 | eval { 41 | $self->log(debug => "preparing $name"); 42 | $sth = $_->prepare( $self->sql->{$name} ); 43 | }; 44 | if( my $err = $@ ) { 45 | $self->log( error => join(" - ", ref $err, $err->errstr )); 46 | } 47 | return $sth; 48 | }); 49 | } 50 | 51 | sub _build_dbh { 52 | my ($self) = @_; 53 | 54 | die "No db section in config!" unless exists $self->config->{db} && ref $self->config->{db} eq 'HASH'; 55 | 56 | my %db = %{ $self->config->{db} }; 57 | my $dbconn = DBIx::Connector->new( @db{qw(dsn user pass)}, { 58 | PrintError => 0, 59 | RaiseError => 0, 60 | HandleError => Exception::Class::DBI->handler, 61 | }); 62 | if( !defined $dbconn ) { 63 | $self->log( fatal => "Database setup failed" ); 64 | die "DB setup failed"; 65 | } 66 | $self->log( debug => "Database established" ); 67 | return $dbconn; 68 | } 69 | 70 | no Moose::Role; 71 | # Return TRUE 72 | 1; 73 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Feather.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Feather; 2 | 3 | use Moose::Role; 4 | use namespace::autoclean; 5 | 6 | has 'name' => ( 7 | is => 'ro', 8 | isa => 'Str', 9 | builder => '_build_name', 10 | ); 11 | has 'function' => ( 12 | is => 'ro', 13 | isa => 'Str', 14 | builder => '_build_function', 15 | init_arg => undef, 16 | ); 17 | has 'parent' => ( 18 | is => 'ro', 19 | isa => 'Str', 20 | builder => '_build_parent', 21 | init_arg => undef, 22 | ); 23 | has 'priority' => ( 24 | is => 'ro', 25 | isa => 'Str', 26 | builder => '_build_priority', 27 | init_arg => undef, 28 | ); 29 | has 'enabled' => ( 30 | is => 'ro', 31 | isa => 'Bool', 32 | lazy => 1, 33 | builder => '_build_enabled', 34 | ); 35 | has 'config' => ( 36 | is => 'ro', 37 | isa => 'HashRef', 38 | init_arg => 'Config', 39 | ); 40 | # Default, can be overridden in children 41 | sub _build_priority { 10; } 42 | 43 | # Default, can be overridden in children 44 | sub _build_parent { 'none'; } 45 | 46 | # Default, enabled 47 | sub _build_enabled { 48 | my $self = shift; 49 | my $config = $self->config; 50 | 51 | if( defined $config && ref $config eq 'HASH' && exists $config->{disabled} ) { 52 | return !$config->{disabled}; 53 | } 54 | return 1; 55 | } 56 | 57 | # Default Naming Convention 58 | sub _build_name { 59 | my $self = shift; 60 | my $class = ref $self; 61 | 62 | if( my($name) = ( $class =~ /\:\:Feather\:\:(.*)/ ) ) { 63 | return $name; 64 | } 65 | die "cannot guess $class name and none is set, override _build_name().\n"; 66 | } 67 | 68 | # Wrap the BUILDARGS function 69 | around BUILDARGS => sub { 70 | my $orig = shift; 71 | my $class = shift; 72 | my %args = @_; 73 | 74 | my $prefix = "DreamCatcher::Feather::"; 75 | my $name = substr($class, length $prefix); 76 | 77 | my %FeatherConfig = (); 78 | if( exists $args{Config} ) { 79 | if( exists $args{Config}->{$name} ) { 80 | %FeatherConfig = %{ $args{Config}->{$name} }; 81 | } 82 | if( exists $args{Config}->{db} ) { 83 | $FeatherConfig{db} = \%{ $args{Config}->{db} }; 84 | } 85 | } 86 | $class->$orig( Config => \%FeatherConfig, Log => $args{Log} ); 87 | }; 88 | 89 | no Moose::Role; 90 | # Return True 91 | 1; 92 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Feather/Analysis.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Feather::Analysis; 2 | 3 | use Moose::Role; 4 | use namespace::autoclean; 5 | with qw( 6 | DreamCatcher::Role::Feather 7 | DreamCatcher::Role::DBH 8 | DreamCatcher::Role::Logger 9 | ); 10 | use DreamCatcher::Types qw(PositiveInt); 11 | 12 | requires qw(analyze); 13 | 14 | has 'default_interval' => ( 15 | is => 'ro', 16 | isa => PositiveInt, 17 | builder => '_build_interval', 18 | ); 19 | 20 | has 'check_period' => ( 21 | is => 'rw', 22 | isa => PositiveInt, 23 | builder => '_build_check_period', 24 | lazy => 1, 25 | ); 26 | 27 | has 'batch_max' => ( 28 | is => 'rw', 29 | isa => PositiveInt, 30 | builder => '_build_batch_max', 31 | ); 32 | 33 | # Set the function 34 | sub _build_function { 'analysis'; } 35 | 36 | # Default is process every 10 minutes 37 | sub _build_interval { 600; } 38 | 39 | # Default is check back two hours 40 | sub _build_check_period { 3600*2; } 41 | 42 | # Default is 2,000 records 43 | sub _build_batch_max { 2_000; } 44 | 45 | sub interval { 46 | my $self = shift; 47 | return exists $self->config->{interval} && $self->config->{interval} > 0 ? $self->config->{interval} : $self->default_interval; 48 | } 49 | 50 | my %_sld_needs_more = map { $_ => 1 } qw(co com net org); 51 | sub strip_sld { 52 | my $self = shift; 53 | my ($domain) = @_; 54 | 55 | my $without_sld = undef; 56 | chomp($domain); 57 | my @parts = map { lc } split /\./, $domain; 58 | 59 | return unless @parts > 2; 60 | 61 | return if $parts[-1] eq 'arpa' && $parts[-2] eq 'in-addr'; 62 | pop @parts; 63 | pop @parts; 64 | pop @parts if exists $_sld_needs_more{$parts[-1]}; 65 | 66 | return join('.', @parts); 67 | } 68 | 69 | no Moose::Role; 70 | # Return True 71 | 1; 72 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Feather/Anomaly.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Feather::Anomaly; 2 | # ABSTRACT: Role container to handle anomaly scores 3 | 4 | use Moose::Role; 5 | use namespace::autoclean; 6 | 7 | with 'DreamCatcher::Role::Feather'; 8 | 9 | use Const::Fast; 10 | 11 | const my %OPCODES => ( 12 | # Raw # Name 13 | 0 => 'common', QUERY => 'common', 14 | 1 => 'obsolete', IQUERY => 'obsolete', 15 | 2 => 'common', STATUS => 'common', 16 | 3 => 'unassigned', 17 | 4 => 'common', NOTIFY => 'common', 18 | 5 => 'common', UPDATE => 'common', 19 | 6 => 'unassigned', 20 | 7 => 'unassigned', 21 | 8 => 'unassigned', 22 | 9 => 'unassigned', 23 | 10 => 'unassigned', 24 | 11 => 'unassigned', 25 | 12 => 'unassigned', 26 | 13 => 'unassigned', 27 | 14 => 'unassigned', 28 | 15 => 'unassigned', 29 | ); 30 | const my %RR_TYPES => ( 31 | # Common 32 | A => 'common', 33 | AAAA => 'common', 34 | CAA => 'common', 35 | CNAME => 'common', 36 | DLV => 'common', 37 | DNSKEY => 'common', 38 | DNAME => 'common', 39 | DS => 'common', 40 | KEY => 'common', 41 | MX => 'common', 42 | NS => 'common', 43 | PTR => 'common', 44 | RRSIG => 'common', 45 | SOA => 'common', 46 | SIG => 'common', 47 | SPF => 'common', 48 | SRV => 'common', 49 | SSHFP => 'common', 50 | TA => 'common', 51 | TKEY => 'common', 52 | TSIG => 'common', 53 | # Abnormal 54 | URI => 'abnormal', 55 | # Obsolete 56 | A6 => 'obsolete', 57 | MAILA => 'obsolete', 58 | MAILB => 'obsolete', 59 | MD => 'obsolete', 60 | MF => 'obsolete', 61 | MINFO => 'obsolete', 62 | # Suspicious 63 | ANY => 'suspicious', 64 | AXFR => 'suspicious', 65 | IXFR => 'suspicious', 66 | HINFO => 'suspicious', 67 | # Experimental 68 | MB => 'experimental', 69 | MG => 'experimental', 70 | MR => 'experimental', 71 | NULL => 'experimental', 72 | ); 73 | const my %CLASSES => ( 74 | IN => 'common', 75 | # Obsolete 76 | CH => 'obsolete', 77 | HS => 'obsolete', 78 | ); 79 | 80 | # Attributes 81 | 82 | has 'scores' => ( 83 | is => 'ro', 84 | isa => 'HashRef', 85 | lazy => 1, 86 | builder => '_build_scores', 87 | init_arg => undef, 88 | ); 89 | 90 | # Builders 91 | 92 | sub _build_scores { 93 | return { 94 | abnormal => 10, 95 | obsolete => 10, 96 | mismatch => 15, 97 | unassigned => 15, 98 | private => 20, 99 | experimental => 30, 100 | suspicious => 50, 101 | malicious => 100, 102 | }; 103 | } 104 | 105 | # Methods 106 | sub score { 107 | my $self = shift; 108 | my $type = shift; 109 | return exists $self->scores->{$type} ? $self->scores->{$type} : 5; 110 | } 111 | 112 | sub anomaly_class { 113 | my $self = shift; 114 | my $class = shift; 115 | return $CLASSES{$class} if exists $CLASSES{$class}; 116 | return; 117 | } 118 | sub anomaly_type { 119 | my $self = shift; 120 | my $type = shift; 121 | return $RR_TYPES{$type} if exists $RR_TYPES{$type}; 122 | return; 123 | } 124 | sub anomaly_opcode { 125 | my $self = shift; 126 | my $code = shift; 127 | return $OPCODES{$code} if exists $OPCODES{$code}; 128 | return; 129 | } 130 | 131 | no Moose::Role; 132 | # Return True 133 | 1; 134 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Feather/Sniffer.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Feather::Sniffer; 2 | 3 | use Moose::Role; 4 | use namespace::autoclean; 5 | 6 | with qw( 7 | DreamCatcher::Role::Feather 8 | DreamCatcher::Role::DBH 9 | DreamCatcher::Role::Cache 10 | DreamCatcher::Role::Logger 11 | ); 12 | 13 | requires qw(process); 14 | 15 | sub _build_function { 'sniffer'; } 16 | 17 | no Moose::Role; 18 | # Return True 19 | 1; 20 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/Logger.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::Logger; 2 | # ABSTRACT: Provides logging for the feathers 3 | 4 | use Moose::Role; 5 | use namespace::autoclean; 6 | use Log::Log4perl; 7 | use Log::Dispatch::FileRotate; 8 | 9 | has 'logger' => ( 10 | is => 'ro', 11 | isa => 'CodeRef', 12 | required => 1, 13 | init_arg => 'Log', 14 | ); 15 | 16 | sub log { 17 | my $self = shift; 18 | 19 | $self->logger->( @_ ); 20 | } 21 | 22 | no Moose::Role; 23 | # Return TRUE 24 | 1; 25 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Role/RRData.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Role::RRData; 2 | # ABSTRACT: Provides functions for normalizing RR Data 3 | 4 | use Moose::Role; 5 | use namespace::autoclean; 6 | 7 | sub rr_data { 8 | my ($self,$pa) = @_; 9 | 10 | my %data = ( value => undef, opts => undef, ttl => undef ); 11 | 12 | my $class = ref $pa; 13 | 14 | if( index($class, 'Net::DNS::RR') >= 0) { 15 | eval { 16 | if( $pa->type eq 'A' || $pa->type eq 'AAAA' ) { 17 | $data{value} = $pa->address; 18 | } 19 | elsif( $pa->type eq 'CNAME' ) { 20 | $data{value} = $pa->cname; 21 | } 22 | elsif( $pa->type eq 'DNAME' ) { 23 | $data{value} = $pa->dname; 24 | } 25 | elsif( $pa->type eq 'MX' ) { 26 | $data{value} = $pa->exchange; 27 | $data{opts} = $pa->preference; 28 | } 29 | elsif( $pa->type eq 'NS' ) { 30 | $data{value} = $pa->nsdname; 31 | } 32 | elsif( $pa->type eq 'PTR' ) { 33 | $data{value} = $pa->ptrdname; 34 | } 35 | elsif( $pa->type eq 'SRV' ) { 36 | $data{value} = $pa->target; 37 | $data{value} .= ':' . $pa->port if $pa->port; 38 | $data{opts} = $pa->priority; 39 | $data{opts} .= ';' . $pa->priority if defined $pa->weight; 40 | } 41 | elsif( $pa->type eq 'SPF' || $pa->type eq 'TXT' ) { 42 | $data{value} = $pa->txtdata; 43 | } 44 | # Return True 45 | 1; 46 | } or eval { 47 | $data{value} = (split /\s+/, $pa->string)[-1]; 48 | }; 49 | eval { $data{ttl} = $pa->ttl; }; 50 | } 51 | elsif( $class eq 'Net::DNS::Question' ) { 52 | $data{value} = defined $pa->zname ? $pa->zname 53 | : defined $pa->qname ? $pa->qname 54 | : undef; 55 | } 56 | 57 | return wantarray ? %data : \%data; 58 | } 59 | 60 | no Moose::Role; 61 | # Return TRUE 62 | 1; 63 | -------------------------------------------------------------------------------- /lib/DreamCatcher/Types.pm: -------------------------------------------------------------------------------- 1 | package DreamCatcher::Types; 2 | 3 | use MooseX::Types -declare => [qw( 4 | PositiveInt 5 | )]; 6 | 7 | use MooseX::Types::Moose qw(Int); 8 | 9 | subtype PositiveInt, 10 | as Int, 11 | where { $_ > 0 }, 12 | message { "Must be a positive integer." }; 13 | -------------------------------------------------------------------------------- /logging.conf: -------------------------------------------------------------------------------- 1 | # Categories 2 | log4perl.category.default = INFO, DreamCatcher 3 | log4perl.category.Parser = INFO, SyncParser 4 | 5 | # Definitions 6 | log4perl.appender.DreamCatcher = Log::Dispatch::FileRotate 7 | log4perl.appender.DreamCatcher.autoflush = 1 8 | log4perl.appender.DreamCatcher.max = 5 9 | log4perl.appender.DreamCatcher.filename = dreamcatcher.log 10 | log4perl.appender.DreamCatcher.mode = append 11 | log4perl.appender.DreamCatcher.layout = PatternLayout 12 | log4perl.appender.DreamCatcher.layout.ConversionPattern = %d{ISO8601} [%p] - %m{chomp}%n 13 | 14 | log4perl.appender.ParserFile = Log::Dispatch::FileRotate 15 | log4perl.appender.ParserFile.autoflush = 1 16 | log4perl.appender.ParserFile.max = 5 17 | log4perl.appender.ParserFile.filename = parser.log 18 | log4perl.appender.ParserFile.mode = append 19 | log4perl.appender.ParserFile.layout = PatternLayout 20 | log4perl.appender.ParserFile.layout.ConversionPattern = %d{ISO8601} (PID:%P) [%p] - %m{chomp}%n 21 | 22 | # Synchronize the Parser Files 23 | log4perl.appender.SyncParser = Log::Log4perl::Appender::Synchronized 24 | log4perl.appender.SyncParser.appender = ParserFile 25 | -------------------------------------------------------------------------------- /public/css/jquery.dataTables.css: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Table 4 | */ 5 | table.dataTable { 6 | margin: 0 auto; 7 | clear: both; 8 | width: 100%; 9 | } 10 | 11 | table.dataTable thead th { 12 | padding: 3px 18px 3px 10px; 13 | border-bottom: 1px solid black; 14 | font-weight: bold; 15 | cursor: pointer; 16 | *cursor: hand; 17 | } 18 | 19 | table.dataTable tfoot th { 20 | padding: 3px 18px 3px 10px; 21 | border-top: 1px solid black; 22 | font-weight: bold; 23 | } 24 | 25 | table.dataTable td { 26 | padding: 3px 10px; 27 | } 28 | 29 | table.dataTable td.center, 30 | table.dataTable td.dataTables_empty { 31 | text-align: center; 32 | } 33 | 34 | table.dataTable tr.odd { background-color: #E2E4FF; } 35 | table.dataTable tr.even { background-color: white; } 36 | 37 | table.dataTable tr.odd td.sorting_1 { background-color: #D3D6FF; } 38 | table.dataTable tr.odd td.sorting_2 { background-color: #DADCFF; } 39 | table.dataTable tr.odd td.sorting_3 { background-color: #E0E2FF; } 40 | table.dataTable tr.even td.sorting_1 { background-color: #EAEBFF; } 41 | table.dataTable tr.even td.sorting_2 { background-color: #F2F3FF; } 42 | table.dataTable tr.even td.sorting_3 { background-color: #F9F9FF; } 43 | 44 | 45 | /* 46 | * Table wrapper 47 | */ 48 | .dataTables_wrapper { 49 | position: relative; 50 | clear: both; 51 | *zoom: 1; 52 | } 53 | 54 | 55 | /* 56 | * Page length menu 57 | */ 58 | .dataTables_length { 59 | float: left; 60 | } 61 | 62 | 63 | /* 64 | * Filter 65 | */ 66 | .dataTables_filter { 67 | float: right; 68 | text-align: right; 69 | } 70 | 71 | 72 | /* 73 | * Table information 74 | */ 75 | .dataTables_info { 76 | clear: both; 77 | float: left; 78 | } 79 | 80 | 81 | /* 82 | * Pagination 83 | */ 84 | .dataTables_paginate { 85 | float: right; 86 | text-align: right; 87 | } 88 | 89 | /* Two button pagination - previous / next */ 90 | .paginate_disabled_previous, 91 | .paginate_enabled_previous, 92 | .paginate_disabled_next, 93 | .paginate_enabled_next { 94 | height: 19px; 95 | float: left; 96 | cursor: pointer; 97 | *cursor: hand; 98 | color: #111 !important; 99 | } 100 | .paginate_disabled_previous:hover, 101 | .paginate_enabled_previous:hover, 102 | .paginate_disabled_next:hover, 103 | .paginate_enabled_next:hover { 104 | text-decoration: none !important; 105 | } 106 | .paginate_disabled_previous:active, 107 | .paginate_enabled_previous:active, 108 | .paginate_disabled_next:active, 109 | .paginate_enabled_next:active { 110 | outline: none; 111 | } 112 | 113 | .paginate_disabled_previous, 114 | .paginate_disabled_next { 115 | color: #666 !important; 116 | } 117 | .paginate_disabled_previous, 118 | .paginate_enabled_previous { 119 | padding-left: 23px; 120 | } 121 | .paginate_disabled_next, 122 | .paginate_enabled_next { 123 | padding-right: 23px; 124 | margin-left: 10px; 125 | } 126 | 127 | .paginate_enabled_previous { background: url('../img/back_enabled.png') no-repeat top left; } 128 | .paginate_enabled_previous:hover { background: url('../img/back_enabled_hover.png') no-repeat top left; } 129 | .paginate_disabled_previous { background: url('../img/back_disabled.png') no-repeat top left; } 130 | 131 | .paginate_enabled_next { background: url('../img/forward_enabled.png') no-repeat top right; } 132 | .paginate_enabled_next:hover { background: url('../img/forward_enabled_hover.png') no-repeat top right; } 133 | .paginate_disabled_next { background: url('../img/forward_disabled.png') no-repeat top right; } 134 | 135 | /* Full number pagination */ 136 | .paging_full_numbers { 137 | height: 22px; 138 | line-height: 22px; 139 | } 140 | .paging_full_numbers a:active { 141 | outline: none 142 | } 143 | .paging_full_numbers a:hover { 144 | text-decoration: none; 145 | } 146 | 147 | .paging_full_numbers a.paginate_button, 148 | .paging_full_numbers a.paginate_active { 149 | border: 1px solid #aaa; 150 | -webkit-border-radius: 5px; 151 | -moz-border-radius: 5px; 152 | border-radius: 5px; 153 | padding: 2px 5px; 154 | margin: 0 3px; 155 | cursor: pointer; 156 | *cursor: hand; 157 | color: #333 !important; 158 | } 159 | 160 | .paging_full_numbers a.paginate_button { 161 | background-color: #ddd; 162 | } 163 | 164 | .paging_full_numbers a.paginate_button:hover { 165 | background-color: #ccc; 166 | text-decoration: none !important; 167 | } 168 | 169 | .paging_full_numbers a.paginate_active { 170 | background-color: #99B3FF; 171 | } 172 | 173 | 174 | /* 175 | * Processing indicator 176 | */ 177 | .dataTables_processing { 178 | position: absolute; 179 | top: 50%; 180 | left: 50%; 181 | width: 250px; 182 | height: 30px; 183 | margin-left: -125px; 184 | margin-top: -15px; 185 | padding: 14px 0 2px 0; 186 | border: 1px solid #ddd; 187 | text-align: center; 188 | color: #999; 189 | font-size: 14px; 190 | background-color: white; 191 | } 192 | 193 | 194 | /* 195 | * Sorting 196 | */ 197 | .sorting { background: url('../img/sort_both.png') no-repeat center right; } 198 | .sorting_asc { background: url('../img/sort_asc.png') no-repeat center right; } 199 | .sorting_desc { background: url('../img/sort_desc.png') no-repeat center right; } 200 | 201 | .sorting_asc_disabled { background: url('../img/sort_asc_disabled.png') no-repeat center right; } 202 | .sorting_desc_disabled { background: url('../img/sort_desc_disabled.png') no-repeat center right; } 203 | 204 | table.dataTable th:active { 205 | outline: none; 206 | } 207 | 208 | 209 | /* 210 | * Scrolling 211 | */ 212 | .dataTables_scroll { 213 | clear: both; 214 | } 215 | 216 | .dataTables_scrollBody { 217 | *margin-top: -1px; 218 | -webkit-overflow-scrolling: touch; 219 | } 220 | 221 | -------------------------------------------------------------------------------- /public/css/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 60px; 3 | } 4 | div#content { 5 | clear: left; 6 | } 7 | /* DataTables CSS */ 8 | table.table thead .sorting, 9 | table.table thead .sorting_asc, 10 | table.table thead .sorting_desc, 11 | table.table thead .sorting_asc_disabled, 12 | table.table thead .sorting_desc_disabled { 13 | cursor: pointer; 14 | *cursor: hand; 15 | } 16 | 17 | table.table thead .sorting { background: url('../img/sort_both.png') no-repeat center right; } 18 | table.table thead .sorting_asc { background: url('../img/sort_asc.png') no-repeat center right; } 19 | table.table thead .sorting_desc { background: url('../img/sort_desc.png') no-repeat center right; } 20 | 21 | table.table thead .sorting_asc_disabled { background: url('../img/sort_asc_disabled.png') no-repeat center right; } 22 | table.table thead .sorting_desc_disabled { background: url(../'img/sort_desc_disabled.png') no-repeat center right; } 23 | 24 | /* sigma.js context : */ 25 | .sigma-parent { 26 | position: relative; 27 | border-radius: 4px; 28 | -moz-border-radius: 4px; 29 | -webkit-border-radius: 4px; 30 | background: #222; 31 | height: 800px; 32 | } 33 | 34 | .sigma-expand { 35 | position: absolute; 36 | width: 100%; 37 | height: 100%; 38 | top: 0; 39 | left: 0; 40 | } 41 | -------------------------------------------------------------------------------- /public/img/back_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/back_disabled.png -------------------------------------------------------------------------------- /public/img/back_enabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/back_enabled.png -------------------------------------------------------------------------------- /public/img/back_enabled_hover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/back_enabled_hover.png -------------------------------------------------------------------------------- /public/img/forward_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/forward_disabled.png -------------------------------------------------------------------------------- /public/img/forward_enabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/forward_enabled.png -------------------------------------------------------------------------------- /public/img/forward_enabled_hover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/forward_enabled_hover.png -------------------------------------------------------------------------------- /public/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /public/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /public/img/sort_asc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/sort_asc.png -------------------------------------------------------------------------------- /public/img/sort_asc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/sort_asc_disabled.png -------------------------------------------------------------------------------- /public/img/sort_both.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/sort_both.png -------------------------------------------------------------------------------- /public/img/sort_desc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/sort_desc.png -------------------------------------------------------------------------------- /public/img/sort_desc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reyjrar/DreamCatcher/a9109ca3b776a835965eb869162c81fe73c51dcb/public/img/sort_desc_disabled.png -------------------------------------------------------------------------------- /public/js/bootstrap-datatables.js: -------------------------------------------------------------------------------- 1 | /* Default class modification */ 2 | $.extend( $.fn.dataTableExt.oStdClasses, { 3 | "sWrapper": "dataTables_wrapper form-inline" 4 | } ); 5 | /* Set defaults */ 6 | $.fn.DataTable.defaults.sDom = "<'row-fluid'<'span6'l><'span6'f>r>t<'row-fluid'<'span6'i><'span6'p>>"; 7 | /* API method to get paging information */ 8 | $.fn.dataTableExt.oApi.fnPagingInfo = function ( oSettings ) 9 | { 10 | return { 11 | "iStart": oSettings._iDisplayStart, 12 | "iEnd": oSettings.fnDisplayEnd(), 13 | "iLength": oSettings._iDisplayLength, 14 | "iTotal": oSettings.fnRecordsTotal(), 15 | "iFilteredTotal": oSettings.fnRecordsDisplay(), 16 | "iPage": Math.ceil( oSettings._iDisplayStart / oSettings._iDisplayLength ), 17 | "iTotalPages": Math.ceil( oSettings.fnRecordsDisplay() / oSettings._iDisplayLength ) 18 | }; 19 | } 20 | 21 | /* Bootstrap style pagination control */ 22 | $.extend( $.fn.dataTableExt.oPagination, { 23 | "bootstrap": { 24 | "fnInit": function( oSettings, nPaging, fnDraw ) { 25 | var oLang = oSettings.oLanguage.oPaginate; 26 | var fnClickHandler = function ( e ) { 27 | e.preventDefault(); 28 | if ( oSettings.oApi._fnPageChange(oSettings, e.data.action) ) { 29 | fnDraw( oSettings ); 30 | } 31 | }; 32 | 33 | $(nPaging).addClass('pagination').append( 34 | '' 38 | ); 39 | var els = $('a', nPaging); 40 | $(els[0]).bind( 'click.DT', { action: "previous" }, fnClickHandler ); 41 | $(els[1]).bind( 'click.DT', { action: "next" }, fnClickHandler ); 42 | }, 43 | 44 | "fnUpdate": function ( oSettings, fnDraw ) { 45 | var iListLength = 5; 46 | var oPaging = oSettings.oInstance.fnPagingInfo(); 47 | var an = oSettings.aanFeatures.p; 48 | var i, j, sClass, iStart, iEnd, iHalf=Math.floor(iListLength/2); 49 | 50 | if ( oPaging.iTotalPages < iListLength) { 51 | iStart = 1; 52 | iEnd = oPaging.iTotalPages; 53 | } 54 | else if ( oPaging.iPage <= iHalf ) { 55 | iStart = 1; 56 | iEnd = iListLength; 57 | } else if ( oPaging.iPage >= (oPaging.iTotalPages-iHalf) ) { 58 | iStart = oPaging.iTotalPages - iListLength + 1; 59 | iEnd = oPaging.iTotalPages; 60 | } else { 61 | iStart = oPaging.iPage - iHalf + 1; 62 | iEnd = iStart + iListLength - 1; 63 | } 64 | 65 | for ( i=0, iLen=an.length ; i'+j+'') 73 | .insertBefore( $('li:last', an[i])[0] ) 74 | .bind('click', function (e) { 75 | e.preventDefault(); 76 | oSettings._iDisplayStart = (parseInt($('a', this).text(),10)-1) * oPaging.iLength; 77 | fnDraw( oSettings ); 78 | } ); 79 | } 80 | 81 | // Add / remove disabled classes from the static elements 82 | if ( oPaging.iPage === 0 ) { 83 | $('li:first', an[i]).addClass('disabled'); 84 | } else { 85 | $('li:first', an[i]).removeClass('disabled'); 86 | } 87 | 88 | if ( oPaging.iPage === oPaging.iTotalPages-1 || oPaging.iTotalPages === 0 ) { 89 | $('li:last', an[i]).addClass('disabled'); 90 | } else { 91 | $('li:last', an[i]).removeClass('disabled'); 92 | } 93 | } 94 | } 95 | } 96 | } ); 97 | -------------------------------------------------------------------------------- /public/js/dreamcatcher.js: -------------------------------------------------------------------------------- 1 | // Functions 2 | function dc_format_date(dateString) { 3 | var dateObj = Date.parse( dateString.replace(/\..+/, '') ); 4 | return dateObj.toString('yyyy-MM-dd HH:mm'); 5 | } 6 | 7 | // Initilization 8 | 9 | $(function () { 10 | $.extend( $.fn.dataTable.defaults, { 11 | "iDisplayLength": 10, 12 | "bSortClasses": false, 13 | "sPaginationType": "bootstrap", 14 | // "oLanguage": { 15 | // "sLengthMenu": "_MENU_ records per page" 16 | // }, 17 | "fnPreDrawCallback": function( oSettings ) { 18 | $('.textDate').each(function(idx) { 19 | var newStr = dc_format_date( $(this).text() ); 20 | $(this).text( newStr ); 21 | }); 22 | }, 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /public/js/jquery.bootstrap-growl.js: -------------------------------------------------------------------------------- 1 | /* https://github.com/ifightcrime/bootstrap-growl */ 2 | (function(e){e.bootstrapGrowl=function(t,n){var n=e.extend({},e.bootstrapGrowl.default_options,n),r=e("
");r.attr("class","bootstrap-growl alert"),n.type&&r.addClass("alert-"+n.type),n.allow_dismiss&&r.append('×'),r.append(t),n.top_offset&&(n.offset={from:"top",amount:n.top_offset});var i=e(".bootstrap-growl",n.ele);offsetAmount=n.offset.amount,e.each(i,function(){offsetAmount=offsetAmount+e(this).outerHeight()+n.stackup_spacing}),css={position:"absolute",margin:0,"z-index":"9999",display:"none"},css[n.offset.from]=offsetAmount+"px",r.css(css),n.width!=="auto"&&r.css("width",n.width+"px"),e(n.ele).append(r);switch(n.align){case"center":r.css({left:"50%","margin-left":"-"+r.outerWidth()/2+"px"});break;case"left":r.css("left","20px");break;default:r.css("right","20px")}r.fadeIn(),n.delay>=0&&r.delay(n.delay).fadeOut("slow",function(){e(this).remove()})},e.bootstrapGrowl.default_options={ele:"body",type:null,offset:{from:"top",amount:20},align:"right",width:250,delay:4e3,allow_dismiss:!0,stackup_spacing:10}})(jQuery); 3 | -------------------------------------------------------------------------------- /script/dream_catcher: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use Mojo::Base -strict; 3 | # PODNAME: dream_catcher 4 | 5 | use File::Basename 'dirname'; 6 | use File::Spec::Functions qw(catdir splitdir); 7 | 8 | # Source directory has precedence 9 | my @base = (splitdir(dirname(__FILE__)), '..'); 10 | my $lib = join('/', @base, 'lib'); 11 | -e catdir(@base, 't') ? unshift(@INC, $lib) : push(@INC, $lib); 12 | 13 | # Start commands for application 14 | require Mojolicious::Commands; 15 | Mojolicious::Commands->start_app('DreamCatcher'); 16 | -------------------------------------------------------------------------------- /sql/deploy_database_schema.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | use FindBin; 6 | 7 | # Add local lib path 8 | use lib "$FindBin::Bin/../lib"; 9 | 10 | use File::Spec; 11 | use File::Basename; 12 | use File::Slurp qw( slurp ); 13 | use YAML; 14 | use DBI; 15 | 16 | #------------------------------------------------------------------------# 17 | # Grab Mode 18 | my %OPTIONS = (); 19 | my %MODES = ( 20 | install => sub { 1 }, 21 | upgrade => sub { 22 | my $upgrade_from = shift @ARGV; 23 | if( !$upgrade_from ) { 24 | die "usage: $0 upgrade VERSION_TO_START_FROM\n"; 25 | } 26 | $OPTIONS{upgrade_from} = $upgrade_from; 27 | }, 28 | ); 29 | my $MODE = shift @ARGV; 30 | $MODE //= 'install'; 31 | 32 | if( !exists $MODES{$MODE} ) { 33 | die "usage: $0 [mode: " . join(',', keys %MODES) . "]\n"; 34 | } 35 | # Do Argument Check 36 | $MODES{$MODE}->(); 37 | 38 | #------------------------------------------------------------------------# 39 | # Locate all the necessary directories 40 | my @BasePath = File::Spec->splitdir( $FindBin::Bin ); 41 | pop @BasePath; 42 | 43 | #------------------------------------------------------------------------# 44 | # Load Configuration 45 | my $configFile = File::Spec->catfile( @BasePath, 'dreamcatcher.yml' ); 46 | my $CFG = YAML::LoadFile( $configFile ) or die "unable to load $configFile: $!\n"; 47 | 48 | 49 | #------------------------------------------------------------------------# 50 | # Load Schema YAML 51 | my $deployFile = File::Spec->catfile( @BasePath, 'sql', 'schema', 'deploy.yml' ); 52 | my $SCHEMA = YAML::LoadFile( $deployFile ) or die "unable to load $deployFile: $!\n"; 53 | 54 | # Connect to the Database: 55 | my $dbConn = DBI->connect( $CFG->{db}{dsn}, $CFG->{db}{user}, $CFG->{db}{pass}, 56 | {PrintError => 0, RaiseError => 1} ); 57 | 58 | if( $MODE eq 'install' ) { 59 | install( $SCHEMA->{install} ); 60 | if( exists $SCHEMA->{upgrade} and ref $SCHEMA->{upgrade} eq 'HASH' ) { 61 | foreach my $version ( sort keys %{ $SCHEMA->{upgrade} } ) { 62 | upgrade( $SCHEMA->{upgrade}{$version}, $version ); 63 | } 64 | } 65 | } 66 | elsif( $MODE eq 'upgrade' ) { 67 | if ( !exists $SCHEMA->{upgrade}{$OPTIONS{upgrade_from}} ) { 68 | my $upgrades = join(", ", keys %{ $SCHEMA->{upgrade} }); 69 | die "Unknown upgrade starting point, valid options are:\n - $upgrades\n"; 70 | } 71 | foreach my $version ( sort keys %{ $SCHEMA->{upgrade} } ) { 72 | next unless $version >= $OPTIONS{upgrade_from}; 73 | upgrade( $SCHEMA->{upgrade}{$version}, $version ); 74 | } 75 | } 76 | 77 | 78 | sub install { 79 | my $schema = shift; 80 | 81 | # Install Base 82 | foreach my $entity (@{ $schema->{base} } ) { 83 | my $srcFile = File::Spec->catfile( @BasePath, qw(sql schema install base), "$entity.sql" ); 84 | die "$srcFile does not exist!\n" unless -f $srcFile; 85 | 86 | my $sql = slurp( $srcFile ); 87 | my $rc = eval { 88 | $dbConn->do($sql); 89 | 1; 90 | }; 91 | my $error = defined $rc && $rc == 1 ? undef : $@; 92 | 93 | if (!defined($error) ) { 94 | print " - applied base $entity!\n"; 95 | } 96 | elsif ( $error =~ /relation \"$entity\" already exists/ ) { 97 | print " - base $entity already exists!\n"; 98 | } 99 | else { 100 | warn " - applying base $entity failed with error: $error\n"; 101 | exit 1 unless exists $ENV{SQL_NO_FATAL} && $ENV{SQL_NO_FATAL}; 102 | } 103 | } 104 | 105 | # Install Plugins 106 | foreach my $plugin (sort { $schema->{plugins}{$a}{level} <=> $schema->{plugins}{$b}{level} } keys %{ $schema->{plugins} } ) { 107 | my $plugin_pathpart = $plugin; 108 | $plugin_pathpart =~ s/\:\:/_/g; 109 | my @path = ( @BasePath, qw( sql schema install plugins ), $plugin_pathpart ); 110 | print "+ Processing plugin $plugin:\n"; 111 | foreach my $entity ( @{ $schema->{plugins}{$plugin}{entities} } ) { 112 | my $srcFile = File::Spec->catfile( @path, "$entity.sql" ); 113 | die "$srcFile does not exist!\n" unless -f $srcFile; 114 | 115 | my $sql = slurp( $srcFile ); 116 | my $rc = eval { 117 | $dbConn->do($sql); 118 | 1; 119 | }; 120 | my $error = defined $rc && $rc == 1 ? undef : $@; 121 | 122 | if (!defined $error ) { 123 | print " - applied plugin($plugin) $entity!\n"; 124 | } 125 | elsif ( $error =~ /relation \"$entity\" already exists/ ) { 126 | print " - plugin($plugin) $entity already exists!\n"; 127 | } 128 | else { 129 | die " - applying plugin($plugin) $entity failed with error: $error\n"; 130 | } 131 | } 132 | } 133 | } 134 | 135 | sub upgrade { 136 | my ($schema,$ver) = @_; 137 | 138 | print "\nApplying upgrade $ver..\n"; 139 | 140 | if( exists $schema->{base} ) { 141 | # Upgrade Base 142 | print " Processing base.. \n"; 143 | foreach my $entity (@{ $schema->{base} } ) { 144 | my $srcFile = File::Spec->catfile( @BasePath, qw(sql schema upgrade), $ver, 'base', "$entity.sql" ); 145 | die " !! $srcFile does not exist!\n" unless -f $srcFile; 146 | 147 | my $sql = slurp( $srcFile ); 148 | my $rc = eval { 149 | $dbConn->do($sql); 150 | 1; 151 | }; 152 | my $error = defined $rc && $rc == 1 ? undef : $@; 153 | die " - applying base $entity failed with error: $error\n" if $error; 154 | print " - applied base $entity!\n"; 155 | } 156 | } 157 | 158 | if( exists $schema->{plugins} ) { 159 | # Upgrade Plugins 160 | foreach my $plugin (sort { $schema->{plugins}{$a}{level} <=> $schema->{plugins}{$b}{level} } keys %{ $schema->{plugins} } ) { 161 | my $plugin_pathpart = $plugin; 162 | $plugin_pathpart =~ s/\:\:/_/g; 163 | my @path = ( @BasePath, qw( sql schema upgrade ), $ver, 'plugins', $plugin_pathpart ); 164 | print " + Processing plugin $plugin:\n"; 165 | foreach my $entity ( @{ $schema->{plugins}{$plugin}{entities} } ) { 166 | my $srcFile = File::Spec->catfile( @path, "$entity.sql" ); 167 | die " !! $srcFile does not exist!\n" unless -f $srcFile; 168 | 169 | my $sql = slurp( $srcFile ); 170 | my $rc = eval { 171 | $dbConn->do($sql); 172 | 1; 173 | }; 174 | my $error = defined $rc && $rc == 1 ? undef : $@; 175 | die " - applying plugin($plugin) $entity failed with error: $error\n" if $error; 176 | print " - applied plugin($plugin) $entity!\n"; 177 | 178 | } 179 | } 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /sql/schema/deploy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | install: 3 | base: 4 | - server 5 | - client 6 | - conversation 7 | - find_or_create_conversation 8 | plugins: 9 | packet_store: 10 | level: 2 11 | entities: 12 | - query 13 | - response 14 | - question 15 | - answer 16 | - meta_query_response 17 | - meta_question 18 | - meta_answer 19 | - meta_question_answer 20 | - link_query_response 21 | - link_question_answer 22 | - find_or_create_question 23 | - find_or_create_answer 24 | - add_query 25 | - add_response 26 | - store_cleanup 27 | zone_discovery: 28 | level: 3 29 | entities: 30 | - zone 31 | - get_zone_id 32 | - zone_answer 33 | - zone_question 34 | - link_zone_answer 35 | - link_zone_question 36 | list: 37 | level: 4 38 | entities: 39 | - list_type 40 | - list 41 | - list_entry 42 | - list_tracking_client 43 | - list_meta_question 44 | - list_meta_answer 45 | - refresh_list_entry 46 | anomaly: 47 | level: 5 48 | entities: 49 | - anomaly 50 | - create_anomaly_table 51 | -------------------------------------------------------------------------------- /sql/schema/install/base/client.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE client 2 | ( 3 | id serial NOT NULL, 4 | ip inet NOT NULL, 5 | hostname TEXT, 6 | first_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 7 | last_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 8 | is_local boolean NOT NULL DEFAULT false, 9 | role_server_id integer, 10 | reference_count bigint NOT NULL DEFAULT 1, 11 | CONSTRAINT client_pkey PRIMARY KEY (id), 12 | CONSTRAINT client_role_server_id_fkey FOREIGN KEY (role_server_id) 13 | REFERENCES server (id) MATCH SIMPLE 14 | ON UPDATE CASCADE ON DELETE SET NULL DEFERRABLE INITIALLY IMMEDIATE, 15 | CONSTRAINT client_uniq_ip UNIQUE (ip) 16 | ) 17 | WITH ( 18 | OIDS=FALSE 19 | ); 20 | 21 | CREATE INDEX client_idx_role_server_id 22 | ON client 23 | USING btree 24 | (role_server_id); 25 | -------------------------------------------------------------------------------- /sql/schema/install/base/conversation.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE conversation 2 | ( 3 | id bigserial NOT NULL, 4 | server_id integer NOT NULL, 5 | client_id integer NOT NULL, 6 | first_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 7 | last_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 8 | client_is_server boolean NOT NULL DEFAULT false, 9 | reference_count bigint NOT NULL DEFAULT 1, 10 | CONSTRAINT conversation_pkey PRIMARY KEY (id), 11 | CONSTRAINT conversation_client_id_fkey FOREIGN KEY (client_id) 12 | REFERENCES client (id) MATCH SIMPLE 13 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, 14 | CONSTRAINT conversation_server_id_fkey FOREIGN KEY (server_id) 15 | REFERENCES server (id) MATCH SIMPLE 16 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, 17 | CONSTRAINT conversation_uniq_server_client UNIQUE (server_id, client_id) 18 | ) 19 | WITH ( 20 | OIDS=FALSE 21 | ); 22 | 23 | CREATE INDEX conversation_idx_client_id 24 | ON conversation 25 | USING btree 26 | (client_id); 27 | 28 | CREATE INDEX conversation_idx_server_id 29 | ON conversation 30 | USING btree 31 | (server_id); 32 | 33 | 34 | -------------------------------------------------------------------------------- /sql/schema/install/base/find_or_create_conversation.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION find_or_create_conversation(text, text) 2 | RETURNS conversation AS 3 | $BODY$DECLARE 4 | in_client_ip inet := CAST($1 as inet); 5 | in_server_ip inet := CAST($2 as inet); 6 | var_client_id BIGINT := 0; 7 | var_server_id BIGINT := 0; 8 | var_client_server_id BIGINT := 0; 9 | var_client_is_server BOOLEAN := FALSE; 10 | var_convo_id BIGINT; 11 | out_convo_row conversation; 12 | BEGIN 13 | -- Find the Client ID 14 | select into var_client_id, var_client_server_id id, role_server_id 15 | from client where ip = in_client_ip; 16 | 17 | IF NOT FOUND THEN 18 | insert into client ( ip ) values ( in_client_ip ); 19 | select currval('client_id_seq') into var_client_id; 20 | ELSE 21 | var_client_is_server := var_client_server_id is not null; 22 | update client set last_ts = NOW(), reference_count = reference_count + 1 where id = var_client_id; 23 | END IF; 24 | 25 | -- Find the Server ID 26 | select id into var_server_id from server where ip = in_server_ip; 27 | 28 | IF NOT FOUND THEN 29 | insert into server ( ip ) values ( in_server_ip ); 30 | select currval('server_id_seq') into var_server_id; 31 | ELSE 32 | update server set last_ts = NOW(), reference_count = reference_count + 1 where id = var_server_id; 33 | END IF; 34 | 35 | -- Find the Conversation Record 36 | select into var_convo_id id from conversation 37 | where client_id = var_client_id and server_id = var_server_id; 38 | 39 | IF NOT FOUND THEN 40 | insert into conversation ( client_id, server_id, client_is_server ) 41 | values ( var_client_id, var_server_id, var_client_is_server ); 42 | select currval('conversation_id_seq') into var_convo_id; 43 | ELSE 44 | update conversation set last_ts = NOW(), reference_count = reference_count + 1 where id = var_convo_id; 45 | END IF; 46 | 47 | select * into out_convo_row from conversation where id = var_convo_id; 48 | RETURN out_convo_row; 49 | END; 50 | $BODY$ 51 | LANGUAGE plpgsql VOLATILE 52 | COST 100; 53 | -------------------------------------------------------------------------------- /sql/schema/install/base/server.sql: -------------------------------------------------------------------------------- 1 | -- Server Table 2 | CREATE TABLE server 3 | ( 4 | id bigserial NOT NULL, 5 | ip inet NOT NULL, 6 | hostname TEXT, 7 | first_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 8 | last_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 9 | is_authorized boolean NOT NULL DEFAULT false, 10 | reference_count bigint NOT NULL DEFAULT 1, 11 | CONSTRAINT server_pkey PRIMARY KEY (id), 12 | CONSTRAINT server_uniq_ip UNIQUE (ip) 13 | ) 14 | WITH ( 15 | OIDS=FALSE 16 | ); 17 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/anomaly/anomaly.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE anomaly ( 2 | id BIGINT NOT NULL, 3 | source TEXT NOT NULL, 4 | score integer NOT NULL DEFAULT 0, 5 | checks TEXT[], 6 | results JSONB 7 | ) WITH ( OIDS = FALSE ); 8 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/anomaly/create_anomaly_table.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION create_anomaly_table( 2 | in_source_table TEXT 3 | ) 4 | RETURNS boolean AS $BODY$ 5 | DECLARE 6 | var_tablename TEXT := 'anomaly_' || in_source_table; 7 | BEGIN 8 | 9 | -- Need the source table to exist 10 | IF NOT EXISTS (SELECT relname FROM pg_class where relname = in_source_table) THEN 11 | RAISE EXCEPTION 'Source table "%" does not exist.', in_source_table; 12 | END IF; 13 | 14 | -- Nothing to see here. 15 | IF EXISTS (SELECT relname FROM pg_class WHERE relname = var_tablename) THEN 16 | RETURN FALSE; 17 | END IF; 18 | 19 | EXECUTE format('CREATE TABLE %I ( 20 | CHECK( source = % ), 21 | PRIMARY KEY (source, id) 22 | ) INHERITS (anomaly)', 23 | var_tablename, in_source_table 24 | ); 25 | 26 | -- Standard Index 27 | EXECUTE format('CREATE INDEX %I ON %I (id)', 'idx_' || var_tablename || 'id', var_tablename); 28 | 29 | -- Functional Indexes 30 | EXECUTE format('CREATE INDEX %I ON %I USING gin (results)', 'gin_' || var_tablename, var_tablename); 31 | 32 | -- Foreign Keys 33 | EXECUTE format('ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (id) 34 | REFERENCES %I (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE', 35 | var_tablename, 36 | 'fki_' || var_tablename || '_source', 37 | in_source_table 38 | ); 39 | 40 | RETURN TRUE; 41 | END 42 | $BODY$ 43 | LANGUAGE plpgsql VOLATILE 44 | COST 100; 45 | 46 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/list.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE list 2 | ( 3 | id serial NOT NULL, 4 | "name" TEXT NOT NULL, 5 | type_id smallint NOT NULL, 6 | track BOOLEAN NOT NULL DEFAULT false, 7 | can_refresh boolean NOT NULL DEFAULT false, 8 | refresh_url TEXT, 9 | refresh_every interval DEFAULT '7 days'::interval, 10 | refresh_last_ts timestamp without time zone, 11 | CONSTRAINT list_pkey PRIMARY KEY (id), 12 | CONSTRAINT list_fki_type FOREIGN KEY (type_id) 13 | REFERENCES list_type (id) MATCH SIMPLE 14 | ON UPDATE CASCADE ON DELETE SET NULL 15 | ) 16 | WITH ( 17 | OIDS=FALSE 18 | ); 19 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/list_entry.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE list_entry 2 | ( 3 | id serial NOT NULL, 4 | list_id integer NOT NULL, 5 | "zone" TEXT NOT NULL, 6 | path ltree NOT NULL, 7 | refreshed boolean NOT NULL DEFAULT false, 8 | first_ts timestamp without time zone NOT NULL DEFAULT now(), 9 | last_ts timestamp without time zone NOT NULL DEFAULT now(), 10 | CONSTRAINT list_entry_pkey PRIMARY KEY (id), 11 | CONSTRAINT list_entry_fki_list FOREIGN KEY (list_id) 12 | REFERENCES list (id) MATCH SIMPLE 13 | ON UPDATE CASCADE ON DELETE CASCADE, 14 | CONSTRAINT list_entry_uniq UNIQUE (zone, list_id) 15 | ) 16 | WITH ( 17 | OIDS=FALSE 18 | ); 19 | 20 | CREATE INDEX list_entry_idx_path_btree on list_entry using BTREE (path); 21 | CREATE INDEX list_entry_idx_path_gist on list_entry using GIST (path); 22 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/list_meta_answer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE list_meta_answer 2 | ( 3 | list_id INTEGER NOT NULL, 4 | list_entry_id integer NOT NULL, 5 | answer_id bigint NOT NULL, 6 | CONSTRAINT list_meta_answer_pkey PRIMARY KEY (answer_id, list_entry_id), 7 | CONSTRAINT list_meta_answer_list FOREIGN KEY (list_id) 8 | REFERENCES list (id) 9 | ON UPDATE NO ACTION ON DELETE NO ACTION, 10 | CONSTRAINT list_meta_answer_fki_list_entry FOREIGN KEY (list_entry_id) 11 | REFERENCES list_entry (id) MATCH SIMPLE 12 | ON UPDATE CASCADE ON DELETE CASCADE, 13 | CONSTRAINT list_meta_answer_fki_answer FOREIGN KEY (answer_id) 14 | REFERENCES answer (id) MATCH SIMPLE 15 | ON UPDATE CASCADE ON DELETE CASCADE 16 | ) 17 | WITH ( 18 | OIDS=FALSE 19 | ); 20 | 21 | CREATE INDEX list_meta_answer_idx_list_entry 22 | ON list_meta_answer 23 | USING btree 24 | (list_entry_id); 25 | 26 | CREATE INDEX fki_list_meta_answer_list_id ON list_meta_answer(list_id); 27 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/list_meta_question.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE list_meta_question 2 | ( 3 | list_id INTEGER NOT NULL, 4 | list_entry_id integer NOT NULL, 5 | question_id bigint NOT NULL, 6 | CONSTRAINT list_meta_question_pkey PRIMARY KEY (question_id, list_entry_id), 7 | CONSTRAINT list_meta_question_list FOREIGN KEY (list_id) 8 | REFERENCES list (id) 9 | ON UPDATE NO ACTION ON DELETE NO ACTION, 10 | CONSTRAINT list_meta_question_fki_list_entry FOREIGN KEY (list_entry_id) 11 | REFERENCES list_entry (id) MATCH SIMPLE 12 | ON UPDATE CASCADE ON DELETE CASCADE, 13 | CONSTRAINT list_meta_question_fki_question FOREIGN KEY (question_id) 14 | REFERENCES question (id) MATCH SIMPLE 15 | ON UPDATE CASCADE ON DELETE CASCADE 16 | ) 17 | WITH ( 18 | OIDS=FALSE 19 | ); 20 | 21 | CREATE INDEX list_meta_question_idx_list_entry 22 | ON list_meta_question 23 | USING btree 24 | (list_entry_id); 25 | 26 | CREATE INDEX fki_list_meta_question_list_id ON list_meta_question(list_id); 27 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/list_tracking_client.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE list_tracking_client 2 | ( 3 | id serial not null, 4 | list_id integer NOT NULL, 5 | client_id integer NOT NULL, 6 | reference_count integer DEFAULT 0, 7 | first_ts timestamp without time zone NOT NULL DEFAULT now(), 8 | last_ts timestamp without time zone NOT NULL DEFAULT now(), 9 | since_ts timestamp without time zone NOT NULL DEFAULT now(), 10 | CONSTRAINT list_tracking_client_pkey PRIMARY KEY (list_id, client_id), 11 | CONSTRAINT list_tracking_client_fki_list FOREIGN KEY (list_id) 12 | REFERENCES list (id) MATCH SIMPLE 13 | ON UPDATE CASCADE ON DELETE CASCADE, 14 | CONSTRAINT list_tracking_client_fki_client FOREIGN KEY (client_id) 15 | REFERENCES client (id) MATCH SIMPLE 16 | ON UPDATE CASCADE ON DELETE CASCADE 17 | ) 18 | WITH ( 19 | OIDS=FALSE 20 | ); 21 | 22 | CREATE OR REPLACE FUNCTION list_tracking_client( 23 | in_list_id integer, 24 | in_client_id integer, 25 | in_first_ts timestamp without time zone, 26 | in_last_ts timestamp without time zone, 27 | in_reference_count integer) 28 | RETURNS void AS 29 | $BODY$ 30 | BEGIN 31 | PERFORM 1 FROM list_tracking_client WHERE list_id = in_list_id AND client_id = in_client_id; 32 | IF NOT FOUND THEN 33 | INSERT INTO list_tracking_client ( list_id, client_id, first_ts, last_ts, reference_count, since_ts ) 34 | VALUES ( in_list_id, in_client_id, in_first_ts, in_last_ts, in_reference_count, in_first_ts ); 35 | ELSE 36 | UPDATE list_tracking_client 37 | SET reference_count = in_reference_count, 38 | first_ts = (CASE WHEN in_first_ts < first_ts THEN in_first_ts ELSE first_ts END), 39 | last_ts = (CASE WHEN in_last_ts > last_ts THEN in_last_ts ELSE last_ts END), 40 | since_ts = in_first_ts 41 | WHERE list_id = in_list_id AND client_id = in_client_id; 42 | END IF; 43 | 44 | RETURN; 45 | END$BODY$ 46 | LANGUAGE plpgsql VOLATILE 47 | COST 100; 48 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/list_type.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE list_type 2 | ( 3 | id serial NOT NULL, 4 | "name" TEXT NOT NULL, 5 | score smallint NOT NULL DEFAULT 0 CONSTRAINT list_type_score CHECK ( score >= 0 AND score <= 10 ), 6 | CONSTRAINT list_type_pkey PRIMARY KEY (id), 7 | CONSTRAINT list_type_uniq UNIQUE (name) 8 | ) 9 | WITH ( 10 | OIDS=FALSE 11 | ); 12 | 13 | INSERT INTO list_type ( "name", score ) VALUES ( 'safe', 0 ); 14 | INSERT INTO list_type ( "name", score ) VALUES ( 'adware', 5 ); 15 | INSERT INTO list_type ( "name", score ) VALUES ( 'malicious', 10 ); 16 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/list/refresh_list_entry.sql: -------------------------------------------------------------------------------- 1 | -- Function: refresh_list_entry(integer, text, text) 2 | 3 | -- DROP FUNCTION refresh_list_entry(integer, text, text); 4 | 5 | CREATE OR REPLACE FUNCTION refresh_list_entry(in_list_id integer, in_zone text, in_path text) 6 | RETURNS integer AS 7 | $BODY$DECLARE 8 | var_zone TEXT; 9 | var_path ltree; 10 | out_entry_id BIGINT := 0; 11 | BEGIN 12 | var_zone := lower( in_zone ); 13 | var_path := lower( in_path ); 14 | 15 | SELECT into out_entry_id id FROM list_entry 16 | WHERE list_id = in_list_id AND zone = var_zone; 17 | 18 | IF NOT FOUND THEN 19 | INSERT INTO list_entry ( list_id, zone, path, refreshed ) 20 | VALUES ( in_list_id, var_zone, var_path, TRUE ); 21 | SELECT currval('list_entry_id_seq') into out_entry_id; 22 | ELSE 23 | UPDATE list_entry SET refreshed = TRUE, last_ts = NOW() 24 | WHERE list_id = in_list_id AND zone = var_zone; 25 | END IF; 26 | UPDATE list SET refresh_last_ts = NOW() where id = in_list_id; 27 | RETURN out_entry_id; 28 | END; 29 | $BODY$ 30 | LANGUAGE plpgsql VOLATILE 31 | COST 100; 32 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/add_query.sql: -------------------------------------------------------------------------------- 1 | -- Create the new function 2 | CREATE OR REPLACE FUNCTION add_query(bigint, integer, integer, integer, integer, integer, TEXT, integer, boolean, boolean, boolean, numeric(16,6)) 3 | RETURNS bigint AS 4 | $BODY$DECLARE 5 | in_convo_id ALIAS FOR $1; 6 | in_client_id ALIAS FOR $2; 7 | in_client_port ALIAS FOR $3; 8 | in_server_id ALIAS FOR $4; 9 | in_server_port ALIAS FOR $5; 10 | in_query_serial ALIAS FOR $6; 11 | in_opcode ALIAS for $7; 12 | in_questions ALIAS for $8; 13 | in_recursive ALIAS for $9; 14 | in_truncated ALIAS for $10; 15 | in_checking ALIAS for $11; 16 | in_capture_time ALIAS for $12; 17 | out_query_id BIGINT := 0; 18 | BEGIN 19 | -- Insert into query 20 | insert into query ( conversation_id, client_id, client_port, server_id, server_port, 21 | query_serial, opcode, count_questions, flag_recursive, flag_truncated, 22 | flag_checking, capture_time ) 23 | values 24 | ( in_convo_id, in_client_id, in_client_port, in_server_id, in_server_port, 25 | in_query_serial, in_opcode, in_questions, in_recursive, in_truncated, 26 | in_checking, in_capture_time ); 27 | select currval('query_id_seq') into out_query_id; 28 | 29 | RETURN out_query_id; 30 | END;$BODY$ 31 | LANGUAGE plpgsql VOLATILE 32 | COST 100; 33 | 34 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/add_response.sql: -------------------------------------------------------------------------------- 1 | -- Create the New Function 2 | CREATE OR REPLACE FUNCTION add_response(in_convo_id bigint, in_client_id integer, in_client_port integer, in_server_id integer, in_server_port integer, in_query_serial integer, in_opcode TEXT, in_status TEXT, in_size_answer integer, in_cnt_answer integer, in_cnt_additional integer, in_cnt_authority integer, in_cnt_question integer, in_authoritative boolean, in_authenticated boolean, in_truncated boolean, in_checking_desired boolean, in_recursion_desired boolean, in_recursion_available boolean, in_capture_time numeric(16,6)) 3 | RETURNS bigint AS 4 | $BODY$DECLARE 5 | out_response_id BIGINT := 0; 6 | BEGIN 7 | -- Create Response 8 | insert into response ( conversation_id, client_id, client_port, server_id, server_port, 9 | query_serial, opcode, status, size_answer, count_answer, 10 | count_additional, count_authority, count_question, 11 | flag_authoritative, flag_authenticated, flag_truncated, 12 | flag_checking_desired, flag_recursion_desired, flag_recursion_available, capture_time ) 13 | values ( in_convo_id, in_client_id, in_client_port, in_server_id, in_server_port, 14 | in_query_serial, in_opcode, in_status, in_size_answer, in_cnt_answer, 15 | in_cnt_additional, in_cnt_authority, in_cnt_question, 16 | in_authoritative, in_authenticated, in_truncated, 17 | in_checking_desired, in_recursion_desired, in_recursion_available, in_capture_time ); 18 | 19 | -- Grab the ID 20 | select into out_response_id currval('response_id_seq'); 21 | 22 | RETURN out_response_id; 23 | END;$BODY$ 24 | LANGUAGE plpgsql VOLATILE 25 | COST 100; 26 | 27 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/answer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE answer 2 | ( 3 | id bigserial NOT NULL, 4 | first_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 5 | last_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 6 | "name" TEXT NOT NULL, 7 | "type" TEXT NOT NULL, 8 | "class" TEXT NOT NULL DEFAULT 'IN', 9 | "value" TEXT, 10 | opts TEXT, 11 | reference_count bigint NOT NULL DEFAULT 0, 12 | CONSTRAINT answer_pkey PRIMARY KEY (id), 13 | CONSTRAINT answer_uniq UNIQUE ("class", "type", "name", "value") 14 | ) 15 | WITH ( 16 | OIDS=FALSE 17 | ); 18 | 19 | CREATE INDEX answer_idx_last_ts 20 | ON answer 21 | USING btree 22 | (last_ts DESC); 23 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/find_or_create_answer.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION find_or_create_answer(in_response_id bigint, in_section text, in_ttl integer, in_class text, in_type text, in_name text, in_value text, in_opts text) 2 | RETURNS integer AS 3 | $BODY$DECLARE 4 | out_answer_id INTEGER := 0; 5 | norm_name TEXT := LOWER(in_name); 6 | norm_value TEXT := LOWER(in_value); 7 | BEGIN 8 | -- Find this Answer 9 | select into out_answer_id id from answer 10 | where class=in_class and type=in_type and name=norm_name and value=norm_value 11 | limit 1; 12 | 13 | IF NOT FOUND THEN 14 | -- Create it 15 | insert into answer ( class, type, name, value, opts ) 16 | values ( in_class, in_type, norm_name, norm_value, in_opts ) 17 | RETURNING id INTO out_answer_id; 18 | --select into out_answer_id currval('answer_id_seq'); 19 | END IF; 20 | 21 | -- Link the Answer / Response 22 | PERFORM 1 from meta_answer where response_id = in_response_id AND answer_id = out_answer_id AND section = in_section; 23 | IF NOT FOUND THEN 24 | insert into meta_answer ( response_id, answer_id, section, ttl ) 25 | values ( in_response_id, out_answer_id, in_section, in_ttl ); 26 | END IF; 27 | 28 | -- Update the answer tracking data 29 | update answer set last_ts=NOW(), reference_count=reference_count+1 where id=out_answer_id; 30 | 31 | return out_answer_id; 32 | END;$BODY$ 33 | LANGUAGE plpgsql VOLATILE 34 | COST 100; 35 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/find_or_create_question.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION find_or_create_question(bigint, TEXT, TEXT, TEXT) 2 | RETURNS integer AS 3 | $BODY$DECLARE 4 | in_query_id ALIAS FOR $1; 5 | in_class ALIAS FOR $2; 6 | in_type ALIAS FOR $3; 7 | in_name TEXT := LOWER($4); 8 | out_question_id INTEGER := 0; 9 | BEGIN 10 | -- Find this Question 11 | select id into out_question_id from question 12 | where class=in_class and type=in_type and name=in_name 13 | limit 1; -- When we find it, stop. 14 | 15 | IF NOT FOUND THEN 16 | -- create it 17 | insert into question ( class, type, name ) 18 | values ( in_class, in_type, in_name ); 19 | select currval('question_id_seq') into out_question_id; 20 | END IF; 21 | 22 | -- Link the Query / Question 23 | insert into meta_question ( query_id, question_id ) values ( in_query_id, out_question_id ); 24 | -- Update the question tracking data 25 | update question set last_ts=NOW(), reference_count=reference_count+1 where id=out_question_id; 26 | 27 | RETURN out_question_id; 28 | END;$BODY$ 29 | LANGUAGE plpgsql VOLATILE 30 | COST 100; 31 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/link_query_response.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION link_query_response(in_query_id bigint, in_response_id bigint, in_conversation_id bigint, in_timing numeric) 2 | RETURNS void AS 3 | $BODY$ 4 | BEGIN 5 | PERFORM 1 FROM meta_query_response WHERE query_id = in_query_id AND response_id = in_response_id; 6 | 7 | IF NOT FOUND THEN 8 | INSERT INTO meta_query_response ( query_id, response_id, conversation_id, timing ) 9 | VALUES ( in_query_id, in_response_id, in_conversation_id, in_timing ); 10 | END IF; 11 | 12 | RETURN; 13 | END$BODY$ 14 | LANGUAGE plpgsql VOLATILE 15 | COST 100; 16 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/link_question_answer.sql: -------------------------------------------------------------------------------- 1 | -- Function: link_question_answer(bigint, bigint, integer, timestamp without time zone, timestamp without time zone) 2 | 3 | -- DROP FUNCTION link_question_answer(bigint, bigint, integer, timestamp without time zone, timestamp without time zone); 4 | 5 | CREATE OR REPLACE FUNCTION link_question_answer( 6 | in_question_id bigint, 7 | in_answer_id bigint, 8 | in_reference_count integer DEFAULT 1, 9 | in_first_ts timestamp without time zone DEFAULT now(), 10 | in_last_ts timestamp without time zone DEFAULT now()) 11 | RETURNS void AS 12 | $BODY$BEGIN 13 | 14 | PERFORM 1 FROM meta_question_answer WHERE question_id = in_question_id AND answer_id = in_answer_id; 15 | IF FOUND THEN 16 | UPDATE meta_question_answer 17 | SET reference_count = reference_count + in_reference_count 18 | AND first_ts = (CASE WHEN first_ts < in_first_ts THEN first_ts ELSE in_first_ts END) 19 | AND last_ts = (CASE WHEN last_ts > in_last_ts THEN last_ts ELSE in_last_ts END) 20 | WHERE question_id = in_question_id AND answer_id = in_answer_id; 21 | ELSE 22 | INSERT INTO meta_question_answer ( question_id, answer_id, reference_count, first_ts, last_ts ) 23 | VALUES ( in_question_id, in_answer_id, in_reference_count, in_first_ts, in_last_ts ); 24 | END IF; 25 | RETURN; 26 | END;$BODY$ 27 | LANGUAGE plpgsql VOLATILE 28 | COST 100; 29 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/meta_answer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE meta_answer 2 | ( 3 | response_id bigint NOT NULL, 4 | answer_id bigint NOT NULL, 5 | ttl bigint, 6 | section TEXT NOT NULL DEFAULT 'answer'::text, 7 | CONSTRAINT meta_answer_pkey PRIMARY KEY (response_id, answer_id, section), 8 | CONSTRAINT meta_answer_answer_id_fkey FOREIGN KEY (answer_id) 9 | REFERENCES answer (id) MATCH SIMPLE 10 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, 11 | CONSTRAINT meta_answer_response_id_fkey FOREIGN KEY (response_id) 12 | REFERENCES response (id) MATCH SIMPLE 13 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE 14 | ) 15 | WITH ( 16 | OIDS=FALSE 17 | ); 18 | 19 | CREATE INDEX meta_answer_idx_answer 20 | ON meta_answer 21 | USING btree 22 | (answer_id); 23 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/meta_query_response.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE meta_query_response 2 | ( 3 | query_id bigint NOT NULL, 4 | response_id bigint NOT NULL, 5 | timing NUMERIC(11,6), 6 | conversation_id bigint NOT NULL, 7 | CONSTRAINT meta_query_response_pki PRIMARY KEY (query_id, response_id), 8 | CONSTRAINT meta_query_response_fki_query FOREIGN KEY (query_id) 9 | REFERENCES query (id) MATCH SIMPLE 10 | ON UPDATE CASCADE ON DELETE CASCADE, 11 | CONSTRAINT meta_query_response_fki_response FOREIGN KEY (response_id) 12 | REFERENCES response (id) MATCH SIMPLE 13 | ON UPDATE CASCADE ON DELETE CASCADE 14 | CONSTRAINT meta_query_response_fki_conversation FOREIGN KEY (conversation_id) 15 | REFERENCES conversation (id) MATCH SIMPLE 16 | ON UPDATE CASCADE ON DELETE CASCADE 17 | ) 18 | WITH ( 19 | OIDS=FALSE 20 | ); 21 | 22 | -- Add Indices 23 | CREATE INDEX meta_query_response_idx_response_id 24 | ON meta_query_response 25 | USING btree 26 | (response_id); 27 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/meta_question.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE meta_question 2 | ( 3 | query_id bigint NOT NULL, 4 | question_id bigint NOT NULL, 5 | CONSTRAINT meta_question_pkey PRIMARY KEY (query_id, question_id), 6 | CONSTRAINT meta_question_query_id_fkey FOREIGN KEY (query_id) 7 | REFERENCES query (id) MATCH SIMPLE 8 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, 9 | CONSTRAINT meta_question_question_id_fkey FOREIGN KEY (question_id) 10 | REFERENCES question (id) MATCH SIMPLE 11 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE 12 | ) 13 | WITH ( 14 | OIDS=FALSE 15 | ); 16 | 17 | CREATE INDEX meta_question_idx_question 18 | ON meta_question 19 | USING btree 20 | (question_id); 21 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/meta_question_answer.sql: -------------------------------------------------------------------------------- 1 | -- Table: meta_question_answer 2 | 3 | -- DROP TABLE meta_question_answer; 4 | 5 | CREATE TABLE meta_question_answer 6 | ( 7 | question_id bigint NOT NULL, 8 | answer_id bigint NOT NULL, 9 | reference_count integer NOT NULL DEFAULT 1, 10 | first_ts timestamp without time zone NOT NULL DEFAULT now(), 11 | last_ts timestamp without time zone NOT NULL DEFAULT now(), 12 | CONSTRAINT meta_question_answer_pki PRIMARY KEY (question_id, answer_id), 13 | CONSTRAINT meta_question_answer_fki_answer FOREIGN KEY (answer_id) 14 | REFERENCES answer (id) MATCH SIMPLE 15 | ON UPDATE CASCADE ON DELETE CASCADE, 16 | CONSTRAINT meta_question_answer_fki_question FOREIGN KEY (question_id) 17 | REFERENCES question (id) MATCH SIMPLE 18 | ON UPDATE CASCADE ON DELETE CASCADE 19 | ) 20 | WITH ( 21 | OIDS=FALSE 22 | ); 23 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/query.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE query 2 | ( 3 | id bigserial NOT NULL, 4 | client_id integer NOT NULL, 5 | client_port integer NOT NULL, 6 | server_id integer NOT NULL, 7 | server_port integer NOT NULL, 8 | query_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 9 | query_serial integer NOT NULL, 10 | conversation_id bigint, 11 | opcode TEXT NOT NULL, 12 | count_questions integer NOT NULL DEFAULT 1, 13 | flag_recursive boolean NOT NULL DEFAULT false, 14 | flag_truncated boolean NOT NULL DEFAULT false, 15 | flag_checking boolean NOT NULL DEFAULT false, 16 | capture_time NUMERIC(16,6), 17 | CONSTRAINT query_pkey PRIMARY KEY (id), 18 | CONSTRAINT query_client_id_fkey FOREIGN KEY (client_id) 19 | REFERENCES client (id) MATCH SIMPLE 20 | ON UPDATE CASCADE ON DELETE SET NULL DEFERRABLE INITIALLY IMMEDIATE, 21 | CONSTRAINT query_conversation_id_fkey FOREIGN KEY (conversation_id) 22 | REFERENCES conversation (id) MATCH SIMPLE 23 | ON UPDATE CASCADE ON DELETE SET NULL DEFERRABLE INITIALLY IMMEDIATE, 24 | CONSTRAINT query_server_id_fkey FOREIGN KEY (server_id) 25 | REFERENCES server (id) MATCH SIMPLE 26 | ON UPDATE CASCADE ON DELETE SET NULL DEFERRABLE INITIALLY IMMEDIATE 27 | ) 28 | WITH ( 29 | OIDS=FALSE 30 | ); 31 | 32 | CREATE INDEX query_idx_conversation_id 33 | ON query 34 | USING btree (conversation_id); 35 | 36 | CREATE INDEX query_idx_query_ts 37 | ON query 38 | USING btree (query_ts DESC NULLS LAST); 39 | 40 | CREATE INDEX query_idx_capture_time 41 | ON query 42 | USING btree (capture_time DESC NULLS LAST); 43 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/question.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE question 2 | ( 3 | id bigserial NOT NULL, 4 | first_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 5 | last_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 6 | "name" TEXT NOT NULL, 7 | "type" TEXT NOT NULL, 8 | "class" TEXT NOT NULL DEFAULT 'IN', 9 | reference_count bigint NOT NULL DEFAULT 0, 10 | CONSTRAINT question_pkey PRIMARY KEY (id), 11 | CONSTRAINT question_uniq UNIQUE ("class", "type", "name") 12 | ) 13 | WITH ( 14 | OIDS=FALSE 15 | ); 16 | 17 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/response.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE response 2 | ( 3 | id bigserial NOT NULL, 4 | client_id integer NOT NULL, 5 | client_port integer NOT NULL, 6 | server_id integer NOT NULL, 7 | server_port integer NOT NULL, 8 | query_serial integer NOT NULL, 9 | response_ts timestamp(6) without time zone NOT NULL DEFAULT now(), 10 | conversation_id bigint, 11 | opcode TEXT NOT NULL, 12 | status TEXT NOT NULL, 13 | size_answer integer NOT NULL DEFAULT 0, 14 | count_answer integer NOT NULL DEFAULT 0, 15 | count_additional integer NOT NULL DEFAULT 0, 16 | count_authority integer NOT NULL DEFAULT 0, 17 | count_question integer NOT NULL DEFAULT 0, 18 | flag_authoritative boolean NOT NULL DEFAULT false, 19 | flag_authenticated boolean NOT NULL DEFAULT false, 20 | flag_truncated boolean NOT NULL DEFAULT false, 21 | flag_checking_desired boolean NOT NULL DEFAULT false, 22 | flag_recursion_desired boolean NOT NULL DEFAULT false, 23 | flag_recursion_available boolean NOT NULL DEFAULT false, 24 | capture_time NUMERIC(16,6), 25 | CONSTRAINT response_pkey PRIMARY KEY (id), 26 | CONSTRAINT response_client_id_fkey FOREIGN KEY (client_id) 27 | REFERENCES client (id) MATCH SIMPLE 28 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, 29 | CONSTRAINT response_conversation_id_fkey FOREIGN KEY (conversation_id) 30 | REFERENCES conversation (id) MATCH SIMPLE 31 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, 32 | CONSTRAINT response_server_id_fkey FOREIGN KEY (server_id) 33 | REFERENCES server (id) MATCH SIMPLE 34 | ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE 35 | ) 36 | WITH ( 37 | OIDS=FALSE 38 | ); 39 | 40 | CREATE INDEX response_idx_conversation_id 41 | ON response 42 | USING btree 43 | (conversation_id); 44 | 45 | CREATE INDEX response_idx_query_serial 46 | ON response 47 | USING btree 48 | (query_serial); 49 | 50 | CREATE INDEX response_idx_response_ts 51 | ON response 52 | USING btree 53 | (response_ts DESC NULLS LAST); 54 | 55 | CREATE INDEX response_idx_capture_time 56 | ON response 57 | USING btree 58 | (capture_time DESC NULLS LAST); 59 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/packet_store/store_cleanup.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION store_cleanup(text) 2 | RETURNS integer AS 3 | $BODY$DECLARE 4 | in_interval INTERVAL := CAST($1 as INTERVAL); 5 | highest_id BIGINT; 6 | rows_deleted_query INTEGER; 7 | rows_deleted_response INTEGER; 8 | 9 | BEGIN 10 | 11 | select id into highest_id from query where query_ts > NOW() - in_interval order by id asc limit 1; 12 | DELETE from query where id in (select id from query where id < highest_id order by id asc limit 100000); 13 | GET DIAGNOSTICS rows_deleted_query := ROW_COUNT; 14 | 15 | select id into highest_id from response where response_ts > NOW() - in_interval order by id asc limit 1; 16 | DELETE from response where id in (select id from response where id < highest_id order by id asc limit 100000); 17 | GET DIAGNOSTICS rows_deleted_response := ROW_COUNT; 18 | 19 | RETURN rows_deleted_query + rows_deleted_response; 20 | END;$BODY$ 21 | LANGUAGE plpgsql VOLATILE 22 | COST 100; 23 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/zone_discovery/get_zone_id.sql: -------------------------------------------------------------------------------- 1 | -- Create the new function 2 | CREATE OR REPLACE FUNCTION get_zone_id(in_zone_name TEXT, in_zone_path TEXT, in_first_ts TIMESTAMP WITHOUT TIME ZONE, in_last_ts TIMESTAMP WITHOUT TIME ZONE) 3 | RETURNS integer AS $$ 4 | DECLARE 5 | out_zone_id INTEGER; 6 | var_zone_name TEXT; 7 | var_zone_path ltree; 8 | BEGIN 9 | var_zone_name := lower( in_zone_name ); 10 | var_zone_path := lower( in_zone_path ); 11 | 12 | 13 | -- Check for this zone 14 | select into out_zone_id id from zone where name = var_zone_name; 15 | 16 | -- Update Last Timestamp 17 | IF FOUND THEN 18 | update zone set last_ts = in_last_ts where id=out_zone_id; 19 | RETURN out_zone_id; 20 | END IF; 21 | 22 | -- Create it if it doesn't exist 23 | INSERT INTO zone ( name, path, first_ts, last_ts ) values ( var_zone_name, var_zone_path, in_first_ts, in_last_ts ); 24 | select into out_zone_id currval('zone_id_seq'); 25 | 26 | RETURN out_zone_id; 27 | END; 28 | $$ 29 | LANGUAGE plpgsql VOLATILE 30 | COST 100; 31 | 32 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/zone_discovery/link_zone_answer.sql: -------------------------------------------------------------------------------- 1 | -- Link zones to answers 2 | CREATE OR REPLACE FUNCTION link_zone_answer(IN in_zone_id integer, IN in_answer_id integer) 3 | RETURNS boolean AS $$ 4 | DECLARE 5 | var_record_ts TIMESTAMP WITHOUT TIME ZONE; 6 | var_zone_ts TIMESTAMP WITHOUT TIME ZONE; 7 | BEGIN 8 | -- Grab the time stamps; 9 | select last_ts into var_record_ts from answer where id=in_answer_id; 10 | select last_ts into var_zone_ts from zone where id=in_zone_id; 11 | 12 | -- Link Tables 13 | insert into zone_answer ( zone_id, answer_id ) values ( in_zone_id, in_answer_id ); 14 | 15 | -- Update Metadata 16 | IF ( var_record_ts > var_zone_ts ) THEN 17 | update zone set last_ts=var_record_ts, reference_count=reference_count+1 where id=in_zone_id; 18 | ELSE 19 | update zone set reference_count=reference_count+1 where id=in_zone_id; 20 | END IF; 21 | RETURN TRUE; 22 | END; 23 | $$ LANGUAGE plpgsql VOLATILE; 24 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/zone_discovery/link_zone_question.sql: -------------------------------------------------------------------------------- 1 | -- Link Zone and Questions 2 | CREATE OR REPLACE FUNCTION link_zone_question(IN in_zone_id integer, IN in_question_id integer) RETURNS boolean AS $$ 3 | DECLARE 4 | var_record_ts TIMESTAMP WITHOUT TIME ZONE; 5 | var_zone_ts TIMESTAMP WITHOUT TIME ZONE; 6 | BEGIN 7 | -- Grab the time stamps; 8 | select last_ts into var_record_ts from question where id=in_question_id; 9 | select last_ts into var_zone_ts from zone where id=in_zone_id; 10 | 11 | -- Link Tables 12 | insert into zone_question ( zone_id, question_id ) values ( in_zone_id, in_question_id ); 13 | 14 | -- Update Metadata 15 | IF ( var_record_ts > var_zone_ts ) THEN 16 | update zone set last_ts=var_record_ts, reference_count=reference_count+1 where id=in_zone_id; 17 | ELSE 18 | update zone set reference_count=reference_count+1 where id=in_zone_id; 19 | END IF; 20 | RETURN TRUE; 21 | END; 22 | $$ LANGUAGE plpgsql VOLATILE; 23 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/zone_discovery/zone.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE "zone" 2 | ( 3 | id bigserial NOT NULL, 4 | "name" TEXT NOT NULL, 5 | path ltree NOT NULL, 6 | reference_count BIGINT DEFAULT 0, 7 | first_ts TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), 8 | last_ts TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), 9 | CONSTRAINT zone_pki_id PRIMARY KEY (id), 10 | CONSTRAINT zone_uniq_name UNIQUE (name) 11 | ) 12 | WITH ( 13 | OIDS=FALSE 14 | ); 15 | 16 | CREATE INDEX zone_idx_path_btree on zone using BTREE (path); 17 | CREATE INDEX zone_idx_path_gist on zone using GIST (path); 18 | CREATE INDEX zone_idx_name on zone using BTREE (name); 19 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/zone_discovery/zone_answer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE zone_answer 2 | ( 3 | zone_id bigint NOT NULL, 4 | answer_id bigint NOT NULL, 5 | CONSTRAINT zone_answer_pki_ids PRIMARY KEY (zone_id, answer_id), 6 | CONSTRAINT zone_answer_fki_answer FOREIGN KEY (answer_id) 7 | REFERENCES answer (id) MATCH SIMPLE 8 | ON UPDATE CASCADE ON DELETE CASCADE, 9 | CONSTRAINT zone_answer_fki_zone FOREIGN KEY (zone_id) 10 | REFERENCES "zone" (id) MATCH SIMPLE 11 | ON UPDATE CASCADE ON DELETE CASCADE 12 | ) 13 | WITH ( 14 | OIDS=FALSE 15 | ); 16 | 17 | CREATE INDEX zone_answer_idx_answer_id ON zone_answer USING btree (answer_id); 18 | -------------------------------------------------------------------------------- /sql/schema/install/plugins/zone_discovery/zone_question.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE zone_question 2 | ( 3 | zone_id bigint NOT NULL, 4 | question_id bigint NOT NULL, 5 | CONSTRAINT zone_question_pki_ids PRIMARY KEY (zone_id, question_id), 6 | CONSTRAINT zone_question_fki_question FOREIGN KEY (question_id) 7 | REFERENCES question (id) MATCH SIMPLE 8 | ON UPDATE CASCADE ON DELETE CASCADE, 9 | CONSTRAINT zone_question_fki_zone FOREIGN KEY (zone_id) 10 | REFERENCES "zone" (id) MATCH SIMPLE 11 | ON UPDATE CASCADE ON DELETE CASCADE 12 | ) 13 | WITH ( 14 | OIDS=FALSE 15 | ); 16 | 17 | CREATE INDEX fki_zone_question_fki_question 18 | ON zone_question 19 | USING btree 20 | (question_id); 21 | 22 | -------------------------------------------------------------------------------- /sql/schema/upgrade/20150330/plugins/anomaly_query/anomaly_query.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE anomaly_query 2 | ( 3 | query_id bigint NOT NULL, 4 | score integer NOT NULL DEFAULT 0, 5 | analysis jsonb, 6 | CONSTRAINT pki_anomaly_query PRIMARY KEY (query_id), 7 | CONSTRAINT fki_anomaly_query FOREIGN KEY (query_id) 8 | REFERENCES query (id) MATCH SIMPLE 9 | ON UPDATE CASCADE ON DELETE CASCADE 10 | ) 11 | WITH ( 12 | OIDS=FALSE 13 | ); 14 | -------------------------------------------------------------------------------- /sql/schema/upgrade/20150330/plugins/anomaly_question/anomaly_question.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE anomaly_question 2 | ( 3 | question_id bigint NOT NULL, 4 | score integer NOT NULL DEFAULT 0, 5 | analysis jsonb, 6 | CONSTRAINT pki_anomaly_question PRIMARY KEY (question_id), 7 | CONSTRAINT fki_anomaly_question FOREIGN KEY (question_id) 8 | REFERENCES question (id) MATCH SIMPLE 9 | ON UPDATE CASCADE ON DELETE CASCADE 10 | ) 11 | WITH ( 12 | OIDS=FALSE 13 | ); 14 | -------------------------------------------------------------------------------- /t/feather-00-basic.t: -------------------------------------------------------------------------------- 1 | use strict; 2 | 3 | use Test::More; 4 | 5 | BEGIN { 6 | use_ok( 'DreamCatcher::Feathers' ); 7 | }; 8 | 9 | my $plumage = new_ok( "DreamCatcher::Feathers" ); 10 | 11 | # By calling chain, we test feathers and tree as well 12 | foreach my $feather ( @{ $plumage->chain } ) { 13 | can_ok( $feather, qw{name priority}); 14 | } 15 | 16 | # Now we're OK 17 | done_testing(); 18 | -------------------------------------------------------------------------------- /t/web-00-basic.t: -------------------------------------------------------------------------------- 1 | use Mojo::Base -strict; 2 | 3 | use Test::More tests => 3; 4 | use Test::Mojo; 5 | 6 | SKIP: { 7 | skip "config is broken", 3; 8 | my $t = Test::Mojo->new('DreamCatcher'); 9 | $t->get_ok('/')->status_is(200)->content_like(qr/DreamCatcher/); 10 | }; 11 | -------------------------------------------------------------------------------- /templates/conversation/view.html.ep: -------------------------------------------------------------------------------- 1 | %title "Conversation Details :: $meta->{client} asking $meta->{server}"; 2 | 3 |

An overview of the conversations between the client, <%= $meta->{client} %> and the 4 | server, <%= $meta->{server} %>. 5 |

6 | 7 |
8 |

Queries

9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | % while ( my $row = $query_sth->fetchrow_hashref) { 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 36 | 37 | 38 | % } 39 | 40 |
TimeTookOpQueryClient PortSerialFlagsStatus
<%= $row->{query_ts} %><%= defined $row->{took} ? sprintf "%0.3f", $row->{took} : '' %><%= $row->{opcode} %><%= join( " ", @{$row}{qw(qclass qtype qname)} ) %><%= $row->{client_port} %><%= $row->{serial} %> 32 | % foreach my $f (qw(recursive truncated checking)) { 33 | <%= $row->{"flag_$f"} ? uc substr($f,0,1) : '' %> 34 | % } 35 | <%== make_badge query_status => $row->{status} %>
41 | 42 |
43 |
44 |

Responses

45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | % while ( my $row = $response_sth->fetchrow_hashref) { 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 74 | 75 | 76 | % } 77 | 78 |
TimeTookClient PortSerialSectionQueryValueFlagsStatus
<%= $row->{response_ts} %><%= defined $row->{took} ? sprintf "%0.3f", $row->{took} : '' %><%= $row->{client_port} %><%= $row->{serial} %><%= defined $row->{section} ? $row->{section} : '' %><%= join( " ", grep { defined } @{$row}{qw(aclass atype aname)} ) %><%= defined $row->{opts} ? "($row->{opts}) " : '' %><%= defined $row->{value} ? $row->{value} : '' %> 70 | % foreach my $f (qw(authoritative recursion_available)) { 71 | <%= $row->{"flag_$f"} ? uc substr($f,0,1) : '' %> 72 | % } 73 | <%== make_badge query_status => $row->{status} %>
79 | 80 |
81 | %= javascript begin 82 | $( function() { 83 | $('#queries').dataTable({ 84 | "aaSorting": [[ 0, "desc" ]], 85 | }); 86 | $('#responses').dataTable({ 87 | "aaSorting": [[ 0, "desc" ]], 88 | }); 89 | }); 90 | %= end 91 | -------------------------------------------------------------------------------- /templates/graph/sigma.html.ep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /templates/layouts/blank.html.ep: -------------------------------------------------------------------------------- 1 | <%= content %> 2 | -------------------------------------------------------------------------------- /templates/layouts/bootstrap.html.ep: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | <%= title %> 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 |
19 |
20 |
21 | DreamCatcher 22 | 30 | 35 |
36 |
37 |
38 |
39 |

<%= title %>

40 | <%= content %> 41 |
42 | 43 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /templates/list/index.html.ep: -------------------------------------------------------------------------------- 1 | %title 'Lists Overview'; 2 | 3 |

Lists

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | % foreach my $id (keys %$lists) { 12 | % my $list = $lists->{$id}; 13 | 14 | 15 | 16 | 17 | 18 | % } 19 | 20 |
NameTypeEntries
"><%= $list->{name} %>{type_id} %>"><%= $list->{type} %><%= $list->{entries} %>
21 | 22 | %= javascript begin 23 | $(function() { 24 | $('#nav_lists').addClass('active'); 25 | 26 | $('#lists').dataTable({ 27 | "aaSorting": [[ 0, "asc" ]], 28 | }); 29 | }); 30 | %= end 31 | -------------------------------------------------------------------------------- /templates/list/view.html.ep: -------------------------------------------------------------------------------- 1 | %title 'List Overview :: ' . $list->{name}; 2 | 3 |

Details

4 | 5 |
6 |
Type
7 |
<%= ucfirst $list->{type} %> (<%= $list->{score} %>)
8 |
Tracking
9 | % if( $list->{track} ) { 10 |
Enabled
11 |
Clients Matched
12 |
<%= $tracking->{clients} %>
13 |
Total Matches
14 |
<%= $tracking->{total} %>
15 |
Timeframe
16 |
<%= $tracking->{first_ts} %> to <%= $tracking->{last_ts} %>
17 | % } else { 18 |
Disabled
19 | % } 20 |
Refreshed
21 | % if( $list->{can_refresh} ) { 22 |
Every <%= $list->{refresh_every} %>
23 |
Last Refreshed
24 |
<%== $list->{refresh_last_ts} ? '' . $list->{refresh_last_ts} . '' : 'Never' %>
25 |
Refresh URL
26 |
<%= $list->{refresh_url} %>
27 | % } else { 28 |
Disabled
29 | % } 30 |
31 | 32 |

Entries

33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | % foreach my $e (@{ $entries }) { 42 | 43 | 44 | 45 | 46 | 47 | 48 | % } 49 | 50 |
ZonePathFirst AddedLast Added
<%= $e->{zone} %><%= $e->{path} %><%= $e->{first_ts} %><%= $e->{last_ts} %>
51 | 52 | %= javascript begin 53 | $(function() { 54 | $('#nav_lists').addClass('active'); 55 | 56 | $('#entries').dataTable({ 57 | "aaSorting": [[ 0, "asc" ]], 58 | }); 59 | }); 60 | %= end 61 | -------------------------------------------------------------------------------- /templates/main/index.html.ep: -------------------------------------------------------------------------------- 1 | %title 'Quick Overview'; 2 | 3 |

DreamCatcher is a DNS inspection suite.

4 | 5 |
6 |
7 | %= include 'server/top_servers'; 8 |
9 |
10 | %= include 'zone/top_zones'; 11 |
12 | 13 |
14 | 15 |
16 |
17 | %= include 'server/server_responses'; 18 |
19 |
20 | 21 | %= javascript begin 22 | $(function() { 23 | $("#nav_main").addClass("active"); 24 | }); 25 | %= end 26 | -------------------------------------------------------------------------------- /templates/questions/index.html.ep: -------------------------------------------------------------------------------- 1 | %title 'Question Overview'; 2 | 3 |

Details about questions being viewed can be found here.

4 | 5 | 10 | 11 |
12 | 13 |
14 |

Top Questions

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | % while( my $row = $STH->{top_questions}->fetchrow_hashref ) { 27 | % my %icons = qw(danger exclamation-sign warning warning-sign info info-sign); 28 | % my $class = $row->{score} > 100 ? 'danger' 29 | % : $row->{score} > 30 ? 'warning' 30 | % : $row->{score} > 0 ? 'info' 31 | % : undef; 32 | 33 | 34 | 35 | 36 | 37 | 42 | 43 | 44 | % } 45 | 46 |
QueryFirstLastCountMore
<%= $row->{class} %> <%= $row->{type} %> <%= $row->{name} %><%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{reference_count} %> 38 | % if( defined $class ) { 39 | 40 | % } 41 |
47 |
48 | 49 |
50 |

Newest Questions

51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | % while( my $row = $STH->{recent_questions}->fetchrow_hashref ) { 62 | 63 | 64 | 65 | 66 | 67 | 68 | % } 69 | 70 |
QueryFirstLastCount
<%= $row->{class} %> <%= $row->{type} %> <%= $row->{name} %><%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{reference_count} %>
71 |
72 | 73 |
74 |

Missed Questions

75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | % while( my $row = $STH->{missed_questions}->fetchrow_hashref ) { 86 | 87 | 88 | 89 | 90 | 91 | 92 | % } 93 | 94 |
QueryFirstLastCount
<%= $row->{class} %> <%= $row->{type} %> <%= $row->{name} %><%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{misses} %>
95 |
96 | 97 | 98 | 99 |
100 | 101 | 102 | 117 | -------------------------------------------------------------------------------- /templates/server/index.html.ep: -------------------------------------------------------------------------------- 1 | %title "Server Overview"; 2 | 3 |

Overview of server statistics

4 | 5 | %= include 'server/top_servers' 6 | 7 | %= include 'server/server_responses' 8 | 9 | %= javascript begin 10 | $(function() { 11 | $('#nav_servers').addClass('active'); 12 | }); 13 | %= end 14 | -------------------------------------------------------------------------------- /templates/server/server_responses.html.ep: -------------------------------------------------------------------------------- 1 |

Server Responses

2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | % while( my $row = $STH->{server_responses}->fetchrow_hashref ) { 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | % } 20 | 21 |
IPOpStatusResponsesTotal
{id} %>"><%= $row->{ip} %><%= $row->{opcode} %><%== make_badge query_status => $row->{status} %><%= $row->{queries} %><%= $row->{total} %>
22 | 23 | %= javascript begin 24 | $(function() { 25 | $('#serverResponses').dataTable({ 26 | "aaSorting": [[ 4, "desc" ], [ 3, "desc"]], 27 | "sPaginationType": "two_button", 28 | }); 29 | }); 30 | %= end 31 | -------------------------------------------------------------------------------- /templates/server/top_servers.html.ep: -------------------------------------------------------------------------------- 1 |

DNS Servers Discovered

2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | % while( my $row = $STH->{top_servers}->fetchrow_hashref ) { 12 | 13 | 16 | 17 | 18 | 19 | 20 | 21 | % } 22 | 23 |
IPFirstLastClientsSessions
-sign"> 14 | {id} %>"><%= $row->{ip} %> 15 | <%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{clients} %><%= $row->{conversations} %>
24 | 25 | %= javascript begin 26 | $(function() { 27 | $('#topServers').dataTable({ 28 | "aaSorting": [[ 4, "desc" ], [ 3, "desc"]], 29 | "sPaginationType": "two_button", 30 | }); 31 | }); 32 | %= end 33 | -------------------------------------------------------------------------------- /templates/server/view.html.ep: -------------------------------------------------------------------------------- 1 | %title "Server Details :: $server->{ip}"; 2 | 3 |

Overview of server statistics

4 | 5 |
6 |

Clients

7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | % while ( my $row = $clients_sth->fetchrow_hashref) { 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | % } 29 | 30 |
ClientFirstLastSessionsTotal 
<%= $row->{ip} %><%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{conversation_count} %><%= $row->{total_count} %>view details
31 | 32 |
33 | 34 | %= javascript begin 35 | $( function() { 36 | $('#nav_servers').addClass('active'); 37 | $('#clients').dataTable({ 38 | "aaSorting": [[ 2, "desc" ]], 39 | }); 40 | }); 41 | %= end 42 | -------------------------------------------------------------------------------- /templates/utility/client_server_map.html.ep: -------------------------------------------------------------------------------- 1 | %title "Client Server Map"; 2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 | %= include '/graph/sigma'; 10 | 11 | %= javascript begin 12 | 13 | $(function() { 14 | /* Set Active Nav Item */ 15 | $('#nav_map').addClass('active'); 16 | }); 17 | // Draw the graph on load 18 | $(function() { 19 | var sigInst = sigma.init(document.getElementById('sigma-map')).drawingProperties({ 20 | labelThreshold: 4, 21 | defaultEdgeType: 'curve', 22 | defaultLabelColor: '#fff', 23 | defaultLabelBGColor: '#fff', 24 | defaultLabelSize: 14, 25 | defaultLabelHoverColor: '#000', 26 | }).mouseProperties({ 27 | maxRatio: 32 28 | }).graphProperties({ 29 | minNodeSize: 1, 30 | maxNodeSize: 10, 31 | minEdgeSize: 1, 32 | maxEdgeSize: 5, 33 | sideMargin: 50, 34 | scalingMode: 'outside' 35 | }); 36 | % foreach my $n (keys %{ $nodes }) { 37 | sigInst.addNode('<%= $n %>', { 'label': '<%= $n %>', 'color': '<%= $nodes->{$n}{color} %>', 'x': <%= $nodes->{$n}{x} %>, 'y': <%= $nodes->{$n}{y} %>, 'size': <%= $nodes->{$n}{size} %> }); 38 | % } 39 | 40 | % foreach my $c (@{ $conversations }) { 41 | % next unless exists $nodes->{$c->{server}} && exists $nodes->{$c->{client}}; 42 | sigInst.addEdge('<%= $c->{id} %>', '<%= $c->{server} %>', '<%= $c->{client} %>', { 'color': '#BDBDBD', 'size': <%= $c->{size} %> } ); 43 | % } 44 | 45 | // Finally, draw. 46 | sigInst.draw(); 47 | }); 48 | %end 49 | -------------------------------------------------------------------------------- /templates/utility/clients_asking.html.ep: -------------------------------------------------------------------------------- 1 | %title 'Clients Asking'; 2 | 3 |

Utility to see which clients have been discovered asking certain questions.

4 | 5 | %= include 'utility/form_clients_asking'; 6 | 7 | % if ($found && $STH->{"clients_$by"}->rows > 0 ) { 8 |
9 |

Clients who asked <%= $by eq 'question' ? "$class $type $name" : "*.$name" %>

10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | % while( my $row = $STH->{"clients_$by"}->fetchrow_hashref ) { 21 | 22 | 23 | 24 | 25 | 26 | 27 | % } 28 | 29 |
ClientFirstLastCount
<%= $row->{client} %><%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{reference_count} %>
30 |
31 | 32 | % } else { 33 |

No results found for question: <%= $question %>

34 | % } 35 | 36 | 37 | 38 | 47 | -------------------------------------------------------------------------------- /templates/utility/form_clients_asking.html.ep: -------------------------------------------------------------------------------- 1 | 2 |
3 |
" class="form-inline"> 4 |
Clients Asking
5 | 6 | 7 |
8 |
9 | 10 | -------------------------------------------------------------------------------- /templates/utility/index.html.ep: -------------------------------------------------------------------------------- 1 | %title 'Utilities'; 2 | 3 |

A list of utilities which may be useful for DNS investigations.

4 | 5 | %= include 'utility/reverse_form'; 6 | 7 | %= include 'utility/form_clients_asking'; 8 | 9 | 10 | 16 | -------------------------------------------------------------------------------- /templates/utility/reverse.html.ep: -------------------------------------------------------------------------------- 1 | %title 'Fake Reverse Utility'; 2 | 3 |

Utility which uses information gleaned from forward DNS records to assemble 4 | a dynamic, "fake" list of DNS reverses. This can be especially useful if you 5 | have network alerts about malicious activity from external IP's and you'd like 6 | to know which DNS forwards are responsible for those IP's.

7 | 8 | %= include 'utility/reverse_form'; 9 | 10 | % if ( defined $ip && $STH->{reverse_lookup}->rows > 0 ) { 11 |
12 |

Forward Queries for: <%= $ip %>

13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | % while( my $row = $STH->{reverse_lookup}->fetchrow_hashref ) { 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | % } 35 | 36 |
QueryAnswerFirstLastCountFunctions
<%= $row->{class} %> <%= $row->{type} %> <%= $row->{name} %><%= $row->{value} %><%= $row->{first_ts} %><%= $row->{last_ts} %><%= $row->{reference_count} %> 
37 |
38 | 39 | % } else { 40 |

No results found for IP:<%= $ip %>.

41 | % } 42 | 43 | 44 | 45 | 54 | -------------------------------------------------------------------------------- /templates/utility/reverse_form.html.ep: -------------------------------------------------------------------------------- 1 | 2 |
3 |
" class="form-inline"> 4 |
Reverse IP Lookup
5 | 6 | 7 |
8 |
9 | 10 | -------------------------------------------------------------------------------- /templates/zone/top_zones.html.ep: -------------------------------------------------------------------------------- 1 |

Top Zones

2 | 3 | 4 | 5 | 6 | 7 | 8 | % while( my $row = $STH->{top_zones}->fetchrow_hashref ) { 9 | 10 | 11 | 12 | 13 | % } 14 | 15 |
ZoneReferences
<%= $row->{name} %><%= $row->{reference_count} %>
16 | 17 | %= javascript begin 18 | $(function() { 19 | $("#topZones").dataTable( { 20 | "aaSorting": [[ 1, "desc" ]], 21 | "sPaginationType": "two_button", 22 | }); 23 | }); 24 | %= end 25 | -------------------------------------------------------------------------------- /weaver.ini: -------------------------------------------------------------------------------- 1 | [@Default] 2 | [Collect::FromOther] 3 | [-Transformer] 4 | transformer = List 5 | [Contributors] 6 | [Support] 7 | perldoc = 0 8 | bugs = none 9 | websites = metacpan, rt 10 | repository_link = web 11 | repository_content = This module's source code is available by visiting: 12 | --------------------------------------------------------------------------------