├── .gitignore ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── NEWS ├── README ├── autogen.sh ├── cif-router ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── LICENSE ├── Makefile.am ├── NEWS ├── README ├── bin │ └── http_api.pl ├── configure.ac ├── lib │ └── CIF │ │ ├── Router.pm │ │ └── Router │ │ ├── HTTP.pm │ │ └── HTTP │ │ └── Json.pm ├── rebuild.sh └── sbin │ └── test-dependencies ├── cif-smrt ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── NEWS ├── README ├── bin │ ├── cif_crontool │ └── cif_smrt ├── configure.ac ├── lib │ └── CIF │ │ ├── Smrt.pm │ │ └── Smrt │ │ ├── ParseCifFeed.pm │ │ ├── ParseCsv.pm │ │ ├── ParseDelim.pm │ │ ├── ParseJson.pm │ │ ├── ParseJsonIodef.pm │ │ ├── ParseRss.pm │ │ ├── ParseTxt.pm │ │ ├── ParseXml.pm │ │ └── Plugin │ │ ├── Decode │ │ ├── Gtar.pm │ │ ├── Gzip.pm │ │ └── Zip.pm │ │ ├── Postprocessor.pm │ │ ├── Postprocessor │ │ ├── Fqdn.pm │ │ ├── Fqdn │ │ │ ├── Nameserver.pm │ │ │ ├── Resolve.pm │ │ │ └── Spamhaus.pm │ │ ├── Ip.pm │ │ ├── Ip │ │ │ ├── Bgp.pm │ │ │ ├── PrefixWhitelist.pm │ │ │ └── Spamhaus.pm │ │ ├── Malware.pm │ │ ├── Malware │ │ │ └── TCMalwareRegistry.pm │ │ ├── Url.pm │ │ └── Url │ │ │ └── Fqdn.pm │ │ ├── Preprocessor │ │ ├── Address.pm │ │ ├── Description.pm │ │ ├── Email.pm │ │ ├── Portlist.pm │ │ └── Uri.pm │ │ └── Pull │ │ ├── Cif.pm │ │ ├── File.pm │ │ └── Http.pm ├── rebuild.sh ├── rules │ ├── AUTHORS │ ├── COPYING │ ├── Makefile.am │ ├── NEWS │ ├── README │ ├── configure.ac │ ├── etc │ │ ├── 00_alexa_whitelist.cfg │ │ ├── 00_mirc_whitelist.cfg │ │ ├── 00_whitelist.cfg │ │ ├── alienvault.cfg │ │ ├── aper.cfg │ │ ├── bambenekconsulting.cfg │ │ ├── cleanmx.cfg │ │ ├── custom.cfg.example │ │ ├── drg.cfg │ │ ├── feodotracker.cfg │ │ ├── malc0de.cfg │ │ ├── malwaredomainlist.cfg │ │ ├── malwaredomains.cfg │ │ ├── misc.cfg │ │ ├── mpatrol.cfg.example │ │ ├── pastebin.cfg.example │ │ ├── phishtank.cfg │ │ ├── shadowserver.cfg │ │ ├── spamhaus.cfg │ │ ├── spyeyetracker.cfg │ │ └── zeustracker.cfg │ └── rebuild.sh └── sbin │ └── test-dependencies ├── cif-test-framework ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── NEWS ├── README ├── autogen.sh ├── bin │ ├── cif_test_framework │ └── cif_test_regression ├── configure.ac └── lib │ └── CIF │ ├── TESTFRAMEWORK.pm │ └── TESTFRAMEWORK │ ├── Plugin.pm │ └── Plugin │ ├── 00_Submit.pm │ ├── 01_Submit_SimpleHTTP.pm │ ├── 10_Query.pm │ └── 11_Query_SimpleHTTP.pm ├── configure.ac ├── experimental ├── install_rhel6.sh └── legacy_example.pl ├── libcif-dbi ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── NEWS ├── README ├── bin │ ├── cif_apikeys │ ├── cif_apikeys_export │ ├── cif_feed │ ├── cif_skel │ └── cif_vacuum ├── configure.ac ├── etc │ └── upgrade │ │ ├── lib │ │ └── CIF │ │ │ ├── Legacy.pm │ │ │ └── Legacy │ │ │ ├── Bgp.pm │ │ │ ├── Domain.pm │ │ │ ├── Email.pm │ │ │ ├── Group.pm │ │ │ ├── Ipv4.pm │ │ │ ├── Malware.pm │ │ │ ├── Service.pm │ │ │ ├── ShareWith.pm │ │ │ └── Url.pm │ │ └── schema │ │ ├── 00_remove.sql │ │ ├── 01_alter_archive.sql │ │ └── 02_alter_apikeys.sql ├── lib │ └── CIF │ │ ├── APIKey.pm │ │ ├── APIKeyGroups.pm │ │ ├── APIKeyRestrictions.pm │ │ ├── Archive.pm │ │ ├── Archive │ │ ├── Plugin.pm │ │ └── Plugin │ │ │ ├── Asn.pm │ │ │ ├── Cc.pm │ │ │ ├── Domain.pm │ │ │ ├── Domain │ │ │ ├── Botnet.pm │ │ │ ├── Fastflux.pm │ │ │ ├── Malware.pm │ │ │ ├── Passive.pm │ │ │ ├── Phishing.pm │ │ │ ├── Spam.pm │ │ │ ├── Spamvertising.pm │ │ │ ├── Suspicious.pm │ │ │ └── Whitelist.pm │ │ │ ├── Email.pm │ │ │ ├── Email │ │ │ ├── Phishing.pm │ │ │ ├── Registrant.pm │ │ │ ├── Suspicious.pm │ │ │ └── Whitelist.pm │ │ │ ├── Feed.pm │ │ │ ├── Hash.pm │ │ │ ├── Hash │ │ │ ├── Md5.pm │ │ │ ├── Sha1.pm │ │ │ └── Uuid.pm │ │ │ ├── Infrastructure.pm │ │ │ ├── Infrastructure │ │ │ ├── Botnet.pm │ │ │ ├── Fastflux.pm │ │ │ ├── Malware.pm │ │ │ ├── Phishing.pm │ │ │ ├── Scan.pm │ │ │ ├── Spam.pm │ │ │ ├── Spamvertising.pm │ │ │ ├── Suspicious.pm │ │ │ └── Whitelist.pm │ │ │ ├── Malware.pm │ │ │ ├── Malware │ │ │ ├── Md5.pm │ │ │ ├── Sha1.pm │ │ │ └── Uuid.pm │ │ │ ├── Rir.pm │ │ │ ├── Search.pm │ │ │ ├── Url.pm │ │ │ └── Url │ │ │ ├── Botnet.pm │ │ │ ├── Malware.pm │ │ │ ├── Phishing.pm │ │ │ ├── Spam.pm │ │ │ ├── Spamvertising.pm │ │ │ └── Whitelist.pm │ │ ├── DBI.pm │ │ ├── Feed.pm │ │ ├── Feed │ │ ├── Plugin.pm │ │ └── Plugin │ │ │ ├── Address.pm │ │ │ ├── Domain.pm │ │ │ ├── Email.pm │ │ │ ├── Infrastructure.pm │ │ │ ├── Malware.pm │ │ │ ├── Search.pm │ │ │ └── Url.pm │ │ └── Profile.pm ├── rebuild.sh ├── sbin │ ├── migrate-data.pl │ └── test-dependencies └── schemas │ ├── Pg │ ├── 00_archive.sql │ ├── apikeys.sql │ ├── domains.sql │ ├── email.sql │ ├── feed.sql │ ├── hash.sql │ ├── index │ │ └── hash.sql │ ├── infrastructure.sql │ ├── malware.sql │ ├── search.sql │ └── url.sql │ └── pg_purge.sql ├── libcif ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── NEWS ├── README ├── bin │ └── cif ├── configure.ac ├── lib │ ├── CIF.pm │ └── CIF │ │ ├── Client.pm │ │ ├── Client │ │ ├── Query.pm │ │ ├── Query │ │ │ ├── Asn.pm │ │ │ ├── Cc.pm │ │ │ ├── Feed.pm │ │ │ ├── Ipv4.pm │ │ │ ├── Rir.pm │ │ │ └── Url.pm │ │ └── Transport │ │ │ └── HTTP.pm │ │ ├── Msg.pm │ │ └── Msg │ │ └── Feed.pm ├── protocol │ ├── AUTHORS │ ├── COPYING │ ├── ChangeLog │ ├── INSTALL │ ├── Makefile.am │ ├── NEWS │ ├── README │ ├── README.md │ ├── configure.ac │ ├── rebuild.sh │ └── src │ │ ├── feed.proto │ │ └── msg.proto ├── rebuild.sh └── sbin │ ├── gen_protocol_bindings.pl │ └── test-dependencies ├── rebuild.sh └── sbin └── test-dependencies /.gitignore: -------------------------------------------------------------------------------- 1 | provision/ 2 | Vagrantfile 3 | .project 4 | .vagrant 5 | *.DS_Store 6 | missing 7 | *.tar.gz 8 | Makefile 9 | Makefile.in 10 | *.m4 11 | *.log 12 | config.status 13 | configure 14 | autom4te.cache/ 15 | install-sh 16 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/AUTHORS -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = libcif libcif-dbi cif-router cif-smrt 2 | EXTRA_DIST = sbin 3 | 4 | ME=cif-v1 5 | BUILD=$(shell git describe --tags) 6 | SNAPSHOT=$(ME)-$(BUILD) 7 | 8 | all: default 9 | 10 | default: 11 | $(PERL) libcif/sbin/gen_protocol_bindings.pl 12 | 13 | fixperms: 14 | chmod 0755 $(prefix) 15 | chown -R $(CIF_USER) $(prefix) 16 | chgrp -R $(CIF_GROUP) $(prefix) 17 | 18 | install: files-install fixperms 19 | 20 | files-install: libcif-install libcif-dbi-install cif-router-install cif-smrt-install 21 | 22 | libcif-install: 23 | -( cd libcif && make install ) 24 | 25 | libcif-dbi-install: 26 | -( cd libcif-dbi && make install ) 27 | 28 | purgedb: 29 | -( cd libcif-dbi && make purgedb ) 30 | 31 | initdb: 32 | -( cd libcif-dbi && make initdb && make tables) 33 | 34 | tables: 35 | -( cd libcif-dbi && make tables ) 36 | 37 | rebuilddb: 38 | -( cd libcif-dbi && make rebuilddb ) 39 | 40 | upgradedb: 41 | -( cd libcif-dbi && make upgradedb ) 42 | 43 | cif-router-install: 44 | -( cd cif-router && make install ) 45 | 46 | cif-smrt-install: 47 | -( cd cif-smrt && make install ) 48 | 49 | depends: fixdeps 50 | 51 | fixdeps: 52 | $(PERL) ./sbin/test-dependencies --verbose --install --prefix $(prefix) 53 | 54 | testdeps: 55 | $(PERL) ./sbin/test-dependencies --verbose --prefix $(prefix) 56 | 57 | clean: 58 | rm Makefile 59 | rm config.log 60 | rm config.status 61 | 62 | realclean: 63 | clean 64 | rm Makefile.in 65 | rm configure 66 | rm aclocal.m4 67 | 68 | reconf: 69 | autoreconf -vf 70 | 71 | snapshot: 72 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 73 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/NEWS -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | The Collective Intelligence Framework v1 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | Resources 12 | --------- 13 | 14 | Website: collectiveintel.org 15 | Repository: github.com/collectiveintel/cif-v1 16 | Installation: https://code.google.com/p/collective-intelligence-framework/wiki/ServerInstall_v1 17 | 18 | 19 | COPYRIGHT AND LICENCE 20 | --------------------- 21 | Copyright (C) 2013 Wes Young (wesyoung.me) 22 | Copyright (C) 2013 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 23 | 24 | Free use of this software is granted under the terms of the GNU Lesser General 25 | Public License (LGPL) unless otherwise specifically stated. For details see the 26 | files `COPYING` included with the distribution. 27 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e # exit on errors 4 | 5 | srcdir=`dirname $0` 6 | test -z "$srcdir" && srcdir=. 7 | git submodule update --init --recursive 8 | 9 | autoreconf -vfi 10 | automake --add-missing 11 | ./configure 12 | -------------------------------------------------------------------------------- /cif-router/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-router/AUTHORS -------------------------------------------------------------------------------- /cif-router/ChangeLog: -------------------------------------------------------------------------------- 1 | see parent ChangeLog 2 | -------------------------------------------------------------------------------- /cif-router/LICENSE: -------------------------------------------------------------------------------- 1 | # COPYRIGHT: 2 | # 3 | # Copyright 2011 The Trustees of Indiana University, indiana.edu 4 | # Copyright 2011 REN-ISAC, ren-isac.net 5 | # Copyright 2011 Wes Young, claimid.com/wesyoung 6 | # 7 | # LICENSE: 8 | # 9 | # This work is made available to you under the terms of Version 2 of 10 | # the GNU General Public License. A copy of that license should have 11 | # been provided with this software, but in any event can be snarfed 12 | # from www.gnu.org. 13 | # 14 | # This work is distributed in the hope that it will be useful, but 15 | # WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 | # General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program; if not, write to the Free Software 21 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 22 | # 02110-1301 or visit their web page on the internet at 23 | # http://www.gnu.org/licenses/old-licenses/gpl-2.0.html. 24 | -------------------------------------------------------------------------------- /cif-router/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = lib bin sbin 2 | 3 | ME=cif-router-perl 4 | BUILD=$(shell git describe --tags) 5 | SNAPSHOT=$(ME)-$(BUILD) 6 | 7 | all: default 8 | 9 | default: 10 | @echo "Please read the README before beginning your installation." 11 | 12 | fixperms: 13 | chmod 0755 $(prefix) 14 | chown -R $(CIF_USER) $(prefix) 15 | chgrp -R $(CIF_GROUP) $(prefix) 16 | 17 | install: files-install fixperms 18 | 19 | files-install: libs-install bin-install 20 | 21 | depends: fixdeps 22 | 23 | fixdeps: 24 | $(PERL) ./sbin/test-dependencies --verbose --install --prefix $(prefix) 25 | 26 | testdeps: 27 | $(PERL) ./sbin/test-dependencies --verbose --prefix $(prefix) 28 | 29 | libs-install: 30 | [ -d $(libdir) ] || $(INSTALL) -m 0755 -d $(libdir) 31 | -( cd lib && find . -type d -print ) | while read dir ; do \ 32 | $(INSTALL) -m 0755 -d "$(libdir)/$$dir" ; \ 33 | done 34 | -( cd lib && find . -type f -print ) | while read file ; do \ 35 | $(INSTALL) -m 0644 "lib/$$file" "$(libdir)/$$file" ; \ 36 | done 37 | 38 | bin-install: 39 | [ -d $(bindir) ] || $(INSTALL) -m 0755 -d $(bindir) 40 | -( cd bin && find . -type f -print ) | while read file ; do \ 41 | $(INSTALL) -m 0755 "bin/$$file" "$(bindir)/$$file" ; \ 42 | done 43 | 44 | clean: 45 | rm Makefile 46 | rm config.log 47 | rm config.status 48 | 49 | realclean: 50 | clean 51 | rm Makefile.in 52 | rm configure 53 | rm aclocal.m4 54 | 55 | reconf: 56 | autoreconf -vf 57 | 58 | snapshot: 59 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 60 | -------------------------------------------------------------------------------- /cif-router/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-router/NEWS -------------------------------------------------------------------------------- /cif-router/README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | The CIF Routing Framework 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | Resources 12 | --------- 13 | 14 | Website: collectiveintel.org 15 | Repository: github.com/collectiveintel/cif-router 16 | 17 | 18 | COPYRIGHT AND LICENCE 19 | --------------------- 20 | Copyright (C) 2013 Wes Young (wesyoung.me) 21 | Copyright (C) 2013 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 22 | 23 | Free use of this software is granted under the terms of the GNU Lesser General 24 | Public License (LGPL) unless otherwise specifically stated. For details see the 25 | files `COPYING` included with the distribution. -------------------------------------------------------------------------------- /cif-router/bin/http_api.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | # this just sets up the lib paths for apache 7 | 8 | # fix lib paths, some may be relative 9 | BEGIN { 10 | require File::Spec; 11 | my @libs = ( 12 | "lib", 13 | "local/lib", 14 | 15 | # dev mode stuff 16 | '../libcif/lib', 17 | '../libcif-dbi/lib', 18 | ); 19 | my $bin_path; 20 | 21 | for my $lib (@libs) { 22 | unless ( File::Spec->file_name_is_absolute($lib) ) { 23 | unless ($bin_path) { 24 | if ( File::Spec->file_name_is_absolute(__FILE__) ) { 25 | $bin_path = ( File::Spec->splitpath(__FILE__) )[1]; 26 | } 27 | else { 28 | require FindBin; 29 | no warnings "once"; 30 | $bin_path = $FindBin::Bin; 31 | } 32 | } 33 | $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib ); 34 | } 35 | unshift @INC, $lib; 36 | } 37 | } 38 | 39 | 1; 40 | -------------------------------------------------------------------------------- /cif-router/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.64]) 5 | AC_INIT(cif-router, m4_esyscmd_s([git describe --tags]), [ci-framework@googlegroups.com]) 6 | AM_INIT_AUTOMAKE 7 | 8 | # Checks for programs. 9 | AC_ARG_VAR([PERL],[Perl interpreter command]) 10 | AC_PATH_PROG([PERL], [perl], [not found]) 11 | if test "$PERL" = 'not found'; then 12 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without perl]) 13 | fi 14 | 15 | AC_ARG_VAR([PSQL],[Postgres psql command]) 16 | AC_PATH_PROG([PSQL],[psql],[not found]) 17 | if test "$PSQL" = 'not found'; then 18 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without postgres psql]) 19 | fi 20 | 21 | dnl Defaults paths for installation 22 | AC_PREFIX_DEFAULT([/opt/cif]) 23 | 24 | AC_CONFIG_FILES([ 25 | Makefile 26 | ]) 27 | 28 | 29 | # Checks for libraries. 30 | 31 | ## apache stuff 32 | ## to be split out in v2? 33 | #PKG_CHECK_MODULES([apr],[apr-1 >= 1.4.2]) 34 | #PKG_CHECK_MODULES([apr_util],[apr-util-1 >= 1.3.9]) 35 | 36 | # Checks for header files. 37 | 38 | # Checks for typedefs, structures, and compiler characteristics. 39 | 40 | # Checks for library functions. 41 | 42 | #dnl CIF_USER 43 | AC_ARG_WITH(user, 44 | AC_HELP_STRING([--with-user=USER], 45 | [user the cif scripts will run as (default: cif)]), 46 | CIF_USER=$withval, 47 | CIF_USER=cif) 48 | AC_SUBST(CIF_USER) 49 | 50 | #dnl CIF_GROUP 51 | AC_ARG_WITH(group, 52 | AC_HELP_STRING([--with-group=GROUP], 53 | [group the cif scripts will run as (default: cif)]), 54 | CIF_GROUP=$withval, 55 | CIF_GROUP=$CIF_USER 56 | ) 57 | AC_SUBST(CIF_GROUP) 58 | 59 | AC_OUTPUT 60 | -------------------------------------------------------------------------------- /cif-router/lib/CIF/Router/HTTP.pm: -------------------------------------------------------------------------------- 1 | package CIF::Router::HTTP; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | ## TODO -- split this out in CIF v2 7 | ## leaving it here for now, simplier 8 | 9 | use APR::Table (); 10 | use Apache2::Request (); 11 | use Apache2::RequestRec (); 12 | use Apache2::RequestIO (); 13 | use Apache2::RequestUtil (); 14 | use Apache2::Response; 15 | use Apache2::Const qw(:common :http); 16 | 17 | require CIF::Router; 18 | require CIF::Router::HTTP::Json; 19 | use CIF qw/init_logging/; 20 | use Data::Dumper; 21 | 22 | ## NOTE: we do it this way cause mod_perl calls us by name 23 | ## not CIF::Router 24 | ## required by ::Router 25 | sub new { 26 | my $class = shift; 27 | my $self = {}; 28 | bless($self,$class); 29 | return($self); 30 | } 31 | 32 | sub handler { 33 | my $req = shift; 34 | 35 | my $config = $req->dir_config->get('CIFRouterConfig') || '/home/cif/.cif'; 36 | 37 | my ($err,$router) = CIF::Router->new({ 38 | config => $config, 39 | }); 40 | if($err){ 41 | ## TODO -- set debugging variable 42 | debug($err); 43 | return Apache2::Const::SERVER_ERROR(); 44 | } 45 | my $debug = $router->get_config->{'debug'} || 0; 46 | init_logging($debug); 47 | 48 | my $reply; 49 | 50 | # test for legacy [v0] support first 51 | # this basically does a recursive lookup and translates out for us 52 | my $agent = lc($req->headers_in->{'User-Agent'}); 53 | if($agent =~ /(mozilla|msie|chrome|safari)/ || ($req->headers_in->{'Accept'} && $req->headers_in->{'Accept'} =~ /application\/json/)){ 54 | if($router->get_config->{'disable_legacy'}){ 55 | $req->content_type('text/plain'); 56 | $reply = "For performance reasons, it appears that legacy JSON support has been disabled by your administrator"; 57 | } else { 58 | $req->content_type('application/json'); 59 | $reply = CIF::Router::HTTP::Json::handler($config,$req,$router); 60 | if($reply =~ /^\d+$/){ 61 | $req->status($reply); 62 | return $reply; 63 | } 64 | } 65 | } else { 66 | return unless($req->method() eq 'POST'); 67 | my $len = $req->headers_in->{'content-length'}; 68 | unless($len > 0){ 69 | return Apache2::Const::FORBIDDEN(); 70 | } 71 | my $buffer; 72 | $req->read($buffer,$req->headers_in->{'content-length'}); 73 | $req->content_type('application/x-protobuf'); 74 | $reply = $router->process($buffer); 75 | } 76 | $req->headers_out()->add('Content-length',length($reply)); 77 | 78 | $req->status(Apache2::Const::HTTP_OK()); 79 | 80 | binmode STDOUT; 81 | print $reply; 82 | 83 | return Apache2::Const::OK(); 84 | } 85 | 86 | 1; -------------------------------------------------------------------------------- /cif-router/rebuild.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make 5 | make dist 6 | -------------------------------------------------------------------------------- /cif-smrt/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-smrt/AUTHORS -------------------------------------------------------------------------------- /cif-smrt/ChangeLog: -------------------------------------------------------------------------------- 1 | see parent ChangeLog 2 | -------------------------------------------------------------------------------- /cif-smrt/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = lib bin sbin rules/etc 2 | 3 | ME=cif-smrt 4 | BUILD=$(shell git describe --tags) 5 | SNAPSHOT=$(ME)-$(BUILD) 6 | 7 | all: default 8 | 9 | default: 10 | @echo "Please read the README before beginning your installation." 11 | 12 | fixperms: 13 | chmod 0755 $(prefix) 14 | chown -R $(CIF_USER) $(prefix) 15 | chgrp -R $(CIF_GROUP) $(prefix) 16 | 17 | install: files-install etc-install local-whitelist fixperms etc-link 18 | 19 | files-install: libs-install bin-install 20 | 21 | depends: fixdeps 22 | 23 | fixdeps: 24 | $(PERL) ./sbin/test-dependencies --verbose --install --prefix $(prefix) 25 | 26 | testdeps: 27 | $(PERL) ./sbin/test-dependencies --verbose --prefix $(prefix) 28 | 29 | local-whitelist: 30 | [ -e $(prefix)/etc/whitelist_domains ] || touch "$(prefix)/etc/whitelist_domains" 31 | [ -e $(prefix)/etc/whitelist_infrastructure ] || touch "$(prefix)/etc/whitelist_infrastructure" 32 | [ -e $(prefix)/etc/whitelist_urls ] || touch "$(prefix)/etc/whitelist_urls" 33 | 34 | etc-link: 35 | [ -d /etc/cif ] || ln -sf $(prefix)/etc /etc/cif 36 | 37 | etc-install: 38 | [ -d $(sysconfdir) ] || $(INSTALL) -m 0755 -d $(sysconfdir) 39 | -( cd rules/etc && find . -type d -print ) | while read dir ; do \ 40 | $(INSTALL) -m 0750 -d "$(sysconfdir)/$$dir" ; \ 41 | done 42 | -( cd rules/etc && find . -type f -print ) | while read file ; do \ 43 | $(INSTALL) -m 0640 "rules/etc/$$file" "$(sysconfdir)/$$file" ; \ 44 | done 45 | 46 | libs-install: 47 | [ -d $(libdir) ] || $(INSTALL) -m 0755 -d $(libdir) 48 | -( cd lib && find . -type d -print ) | while read dir ; do \ 49 | $(INSTALL) -m 0755 -d "$(libdir)/$$dir" ; \ 50 | done 51 | -( cd lib && find . -type f -print ) | while read file ; do \ 52 | $(INSTALL) -m 0644 "lib/$$file" "$(libdir)/$$file" ; \ 53 | done 54 | 55 | bin-install: 56 | [ -d $(bindir) ] || $(INSTALL) -m 0755 -d $(bindir) 57 | -( cd bin && find . -type f -print ) | while read file ; do \ 58 | $(INSTALL) -m 0755 "bin/$$file" "$(bindir)/$$file" ; \ 59 | done 60 | 61 | clean: 62 | rm Makefile 63 | rm config.log 64 | rm config.status 65 | 66 | realclean: 67 | clean 68 | rm Makefile.in 69 | rm configure 70 | rm aclocal.m4 71 | 72 | reconf: 73 | autoreconf -vf 74 | 75 | snapshot: 76 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 77 | -------------------------------------------------------------------------------- /cif-smrt/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-smrt/NEWS -------------------------------------------------------------------------------- /cif-smrt/README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | This is the SMRT intelligence parser 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | Resources 12 | --------- 13 | 14 | Website: collectiveintel.org 15 | Repository: github.com/collectiveintel/cif-smrt 16 | 17 | 18 | COPYRIGHT AND LICENCE 19 | --------------------- 20 | Copyright (C) 2013 Wes Young (wesyoung.me) 21 | Copyright (C) 2013 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 22 | 23 | Free use of this software is granted under the terms of the GNU Lesser General 24 | Public License (LGPL) unless otherwise specifically stated. For details see the 25 | files `COPYING` included with the distribution. -------------------------------------------------------------------------------- /cif-smrt/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.64]) 5 | AC_INIT(cif-smrt, m4_esyscmd_s([git describe --tags]), [ci-framework@googlegroups.com]) 6 | AM_INIT_AUTOMAKE 7 | 8 | # Checks for programs. 9 | AC_ARG_VAR([PERL],[Perl interpreter command]) 10 | AC_PATH_PROG([PERL], [perl], [not found]) 11 | if test "$PERL" = 'not found'; then 12 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without perl]) 13 | fi 14 | 15 | AC_ARG_VAR([PSQL],[Postgres psql command]) 16 | AC_PATH_PROG([PSQL],[psql],[not found]) 17 | if test "$PSQL" = 'not found'; then 18 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without postgres psql]) 19 | fi 20 | 21 | AC_ARG_VAR([PGDUMP],[Postgres pg_dump command]) 22 | AC_PATH_PROG([PGDUMP],[pg_dump],[not found]) 23 | if test "$PGDUMP" = 'not found'; then 24 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without postgres pg_dump]) 25 | fi 26 | 27 | dnl Defaults paths for installation 28 | AC_PREFIX_DEFAULT([/opt/cif]) 29 | 30 | AC_CONFIG_FILES([ 31 | Makefile 32 | ]) 33 | 34 | 35 | # Checks for libraries. 36 | 37 | ## ZeroMQ specific -- coming in beta2 38 | ##PKG_CHECK_MODULES([libzmq],[libzmq >= 2.1.11]) 39 | ##PKG_CHECK_MODULES([libzmq],[libzmq < 2.2]) 40 | 41 | # Checks for header files. 42 | 43 | # Checks for typedefs, structures, and compiler characteristics. 44 | 45 | # Checks for library functions. 46 | 47 | #dnl CIF_USER 48 | AC_ARG_WITH(user, 49 | AC_HELP_STRING([--with-user=USER], 50 | [user the cif scripts will run as (default: cif)]), 51 | CIF_USER=$withval, 52 | CIF_USER=cif) 53 | AC_SUBST(CIF_USER) 54 | 55 | #dnl CIF_GROUP 56 | AC_ARG_WITH(group, 57 | AC_HELP_STRING([--with-group=GROUP], 58 | [group the cif scripts will run as (default: cif)]), 59 | CIF_GROUP=$withval, 60 | CIF_GROUP=$CIF_USER 61 | ) 62 | AC_SUBST(CIF_GROUP) 63 | 64 | AC_OUTPUT 65 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseCifFeed.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseCifFeed; 2 | 3 | use Iodef::Pb::Format; 4 | use CIF; 5 | use MIME::Base64; 6 | use Compress::Snappy; 7 | 8 | sub parse { 9 | my $f = shift; 10 | my $content = shift; 11 | 12 | return unless($content =~ /^application\/cif\n([\S\s]+)$/); 13 | 14 | # this could all be done a better way, and will be in the future 15 | 16 | my @array = split("\n\n",$1); 17 | my @return; 18 | 19 | foreach (@array){ 20 | my $ret = decompress(decode_base64($1)); 21 | 22 | $ret = FeedType->decode($ret); 23 | next unless($ret->get_data()); 24 | 25 | my @blobs = @{$ret->get_data()}; 26 | 27 | @blobs = map { IODEFDocumentType->decode(decompress(decode_base64($_))) } @blobs; 28 | 29 | @blobs = @{Iodef::Pb::Format->new({ 30 | data => \@blobs, 31 | format => 'Raw', 32 | })}; 33 | if($f->{'guid_overwrite'}){ 34 | foreach my $b (@blobs){ 35 | $b->{'guid'} = $f->{'guid_overwrite'}; 36 | } 37 | } 38 | push(@return,@blobs); 39 | } 40 | return(\@return); 41 | } 42 | 43 | 1; 44 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseCsv.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseCsv; 2 | 3 | use strict; 4 | use warnings; 5 | use Text::CSV; 6 | 7 | sub parse { 8 | my $f = shift; 9 | my $content = shift; 10 | 11 | my @lines = split(/[\r\n]/,$content); 12 | my @array; 13 | 14 | if(my $l = $f->{'feed_limit'}){ 15 | my ($start,$end); 16 | if(ref($l) eq 'ARRAY'){ 17 | ($start,$end) = @{$l}; 18 | } else { 19 | ($start,$end) = (0,$l-1); 20 | } 21 | @lines = @lines[$start..$end]; 22 | 23 | # A feed limit may have already been applied to 24 | # this data. If so, don't apply it again. 25 | if ($#lines > ($end - $start)){ 26 | @lines = @lines[$start..$end]; 27 | } 28 | } 29 | 30 | my $csv = Text::CSV->new({binary => 1}); 31 | my @cols = split(',',$f->{'values'}); 32 | 33 | shift @lines if($f->{'skipfirst'}); 34 | 35 | foreach(@lines){ 36 | next if(/^(#|<|$)/); 37 | my $row = $csv->parse($_); 38 | next unless($csv->parse($_)); 39 | my $h; 40 | my @m = $csv->fields(); 41 | foreach (0 ... $#cols){ 42 | next if($cols[$_] eq 'null'); 43 | $h->{$cols[$_]} = $m[$_]; 44 | } 45 | map { $h->{$_} = $f->{$_} } keys %$f; 46 | push(@array,$h); 47 | } 48 | return(\@array); 49 | 50 | } 51 | 52 | 1; 53 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseDelim.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseDelim; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub parse { 7 | my $f = shift; 8 | my $content = shift; 9 | my $split = shift; 10 | 11 | my @lines = split(/[\r\n]/,$content); 12 | my @cols = split(',',$f->{'values'}); 13 | my @array; 14 | 15 | if(my $l = $f->{'feed_limit'}){ 16 | my ($start,$end); 17 | if(ref($l) eq 'ARRAY'){ 18 | ($start,$end) = @{$l}; 19 | } else { 20 | ($start,$end) = (0,$l-1); 21 | } 22 | @lines = @lines[$start..$end]; 23 | 24 | # A feed limit may have already been applied to 25 | # this data. If so, don't apply it again. 26 | if ($#lines > ($end - $start)){ 27 | @lines = @lines[$start..$end]; 28 | } 29 | } 30 | 31 | shift @array if($f->{'skipfirst'}); 32 | 33 | foreach(@lines){ 34 | next if(/^(#|$|<)/); 35 | my @m = split($split,$_); 36 | my $h; 37 | map { $h->{$_} = $f->{$_} } keys %$f; 38 | foreach (0 ... $#cols){ 39 | $h->{$cols[$_]} = $m[$_]; 40 | } 41 | push(@array,$h); 42 | } 43 | return(\@array); 44 | } 45 | 46 | 1; 47 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseJson.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseJson; 2 | 3 | use JSON; 4 | 5 | sub parse { 6 | my $f = shift; 7 | my $content = shift; 8 | 9 | my @feed = @{from_json($content)}; 10 | my @fields = split(',',$f->{'fields'}); 11 | my @fields_map = split(',',$f->{'fields_map'}); 12 | my @array; 13 | foreach my $a (@feed){ 14 | foreach (0 ... $#fields_map){ 15 | if($a->{$fields[$_]}){ 16 | $a->{$fields_map[$_]} = lc($a->{$fields[$_]}); 17 | } 18 | } 19 | map { $a->{$_} = $f->{$_} } keys %$f; 20 | push(@array,$a); 21 | } 22 | return(\@array); 23 | } 24 | 25 | 1; 26 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseJsonIodef.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseJsonIodef; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub parse { 7 | my $f = shift; 8 | my $content = shift; 9 | 10 | require JSON; 11 | my $ret = JSON::from_json($content); 12 | require CIF::Client::Plugin::Iodef; 13 | my @array; 14 | foreach my $r (@$ret){ 15 | my @a; 16 | my $h = CIF::Client::Plugin::Iodef->hash_simple($r); 17 | foreach my $rr (@$h){ 18 | if($f->{'detection'}){ 19 | delete($rr->{'detecttime'}); 20 | $rr->{'detection'} = $f->{'detection'}; 21 | } 22 | push(@a,$rr); 23 | } 24 | push(@array,@a); 25 | } 26 | return(\@array); 27 | } 28 | 29 | 1; 30 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseRss.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseRss; 2 | 3 | use strict; 4 | use warnings; 5 | use XML::RSS; 6 | 7 | sub parse { 8 | my $f = shift; 9 | my $content = shift; 10 | 11 | # fix malformed RSS 12 | unless($content =~ /^<\?xml version/){ 13 | $content = ''."\n".$content; 14 | } 15 | 16 | my $rss = XML::RSS->new(); 17 | my @lines = split(/[\r\n]/,$content); 18 | # work-around for any < > & that is in the feed as part of a url 19 | # http://stackoverflow.com/questions/5199463/escaping-in-perl-generated-xml/5899049#5899049 20 | # needs some work, the parser still pukes. 21 | foreach(@lines){ 22 | s/(\S+)<(?!\!\[CDATA)(.*<\/\S+>)$/$1<$2/g; 23 | s/^(<.*>.*)(?(.*<\/\S+>)$/$1>$2/g; 24 | } 25 | $content = join("\n",@lines); 26 | $rss->parse($content); 27 | my @array; 28 | foreach my $item (@{$rss->{items}}){ 29 | my $h; 30 | foreach my $key (keys %$item){ 31 | if(my $r = $f->{'regex_'.$key}){ 32 | my @m = ($item->{$key} =~ /$r/); 33 | my @cols = split(',',$f->{'regex_'.$key.'_values'}); 34 | foreach (0 ... $#cols){ 35 | $h->{$cols[$_]} = $m[$_]; 36 | } 37 | } 38 | } 39 | map { $h->{$_} = $f->{$_} } keys %$f; 40 | push(@array,$h); 41 | } 42 | return(\@array); 43 | 44 | } 45 | 46 | 1; 47 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseTxt.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseTxt; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub parse { 7 | my $f = shift; 8 | my $content = shift; 9 | 10 | return unless($f->{'regex'}); 11 | 12 | my @lines = split(/[\r\n]/,$content); 13 | my @array; 14 | foreach(@lines){ 15 | next if(/^(#|<|$)/); 16 | my @m = ($_ =~ /$f->{'regex'}/); 17 | next unless(@m); 18 | my $h; 19 | my @cols = (ref($f->{'regex_values'}) eq 'ARRAY') ? @{$f->{'regex_values'}} : split(',',$f->{'regex_values'}); 20 | foreach (0 ... $#cols){ 21 | $m[$_] = '' unless($m[$_]); 22 | for($m[$_]){ 23 | s/^\s+//; 24 | s/\s+$//; 25 | } 26 | $h->{$cols[$_]} = $m[$_]; 27 | } 28 | # a work-around, we do some of this in iodef::pb::simple too 29 | # adding this here makes the debugging messages a little less complicated 30 | if($h->{'address_mask'}){ 31 | $h->{'address'} .= '/'.$h->{'address_mask'}; 32 | } 33 | map { $h->{$_} = $f->{$_} } keys %$f; 34 | push(@array,$h); 35 | } 36 | return(\@array); 37 | 38 | } 39 | 40 | 1; 41 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/ParseXml.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::ParseXml; 2 | 3 | use strict; 4 | use warnings; 5 | require XML::LibXML; 6 | 7 | sub parse { 8 | my $f = shift; 9 | my $content = shift; 10 | 11 | my $parser = XML::LibXML->new(); 12 | my $doc = $parser->load_xml(string => $content); 13 | my @nodes = $doc->findnodes('//'.$f->{'node'}); 14 | my @subnodes = $doc->findnodes('//'.$f->{'subnode'}) if($f->{'subnode'}); 15 | 16 | return unless(@nodes); 17 | 18 | my @array; 19 | my @elements = split(',', $f->{'elements'}) if($f->{'elements'}); 20 | my @elements_map = split(',', $f->{'elements_map'}) if($f->{'elements_map'}); 21 | my @attributes_map = split(',', $f->{'attributes_map'}) if($f->{'attributes_map'}); 22 | my @attributes = split(',', $f->{'attributes'}) if($f->{'attributes'}); 23 | 24 | my %regex; 25 | foreach my $k (keys %$f){ 26 | # pull out any custom regex 27 | for($k){ 28 | if(/^regex_(\S+)$/){ 29 | $regex{$1} = qr/$f->{$k}/; 30 | delete($f->{$k}); 31 | last; 32 | } 33 | # clean up the hash, so we can re-map the default values later 34 | if(/^(elements_?|attributes_?|node|subnode)/){ 35 | delete($f->{$k}); 36 | last; 37 | } 38 | } 39 | } 40 | 41 | foreach my $node (@nodes){ 42 | my $h = {}; 43 | map { $h->{$_} = $f->{$_} } keys %$f; 44 | my $found = 0; 45 | if(@elements_map){ 46 | foreach my $e (0 ... $#elements_map){ 47 | my $x = $node->findvalue('./'.$elements[$e]); 48 | next unless($x); 49 | if(my $r = $regex{$elements[$e]}){ 50 | if($x =~ $r){ 51 | $h->{$elements_map[$e]} = $x; 52 | $found = 1; 53 | } else { 54 | $found = 0; 55 | } 56 | } else { 57 | $h->{$elements_map[$e]} = $x; 58 | $found = 1; 59 | } 60 | } 61 | } else { 62 | foreach my $e (0 ... $#attributes_map){ 63 | my $x = $node->getAttribute($attributes[$e]); 64 | next unless($x); 65 | if(my $r = $regex{$attributes[$e]}){ 66 | if($x =~ $r){ 67 | $h->{$attributes_map[$e]} = $x; 68 | $found = 1; 69 | } else { 70 | $found = 0; 71 | } 72 | } else { 73 | $h->{$attributes_map[$e]} = $x; 74 | $found = 1; 75 | } 76 | } 77 | } 78 | push(@array,$h) if($found); 79 | } 80 | return(\@array); 81 | } 82 | 83 | 1; 84 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Decode/Gtar.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Decode::Gtar; 2 | 3 | use Compress::Zlib; 4 | use Archive::Tar; 5 | use IO::Scalar; 6 | 7 | sub decode { 8 | my $class = shift; 9 | my $data = shift; 10 | my $type = shift; 11 | my $f = shift; 12 | my $uncompressed; 13 | return unless($type =~ /gzip/ || $type =~ /gtar/); #support tar.gz or tar 14 | 15 | if ($type =~ /gzip/){ #if tar.gz, extract to tar 16 | $uncompressed = Compress::Zlib::memGunzip($data); 17 | my $ft = File::Type->new(); 18 | my $t = $ft->mime_type($uncompressed); 19 | return unless($t =~ /gtar/); #return if not a tar inside 20 | } else { 21 | $uncompressed = $data; #already a tar 22 | } 23 | 24 | my $file; 25 | if($f->{'zip_filename'}){ 26 | $file = $f->{'zip_filename'}; 27 | } else { 28 | die $f->{'feed'}; 29 | $f->{'feed'} =~ m/\/([a-zA-Z0-9_]+).zip$/; 30 | $file = $1; 31 | } 32 | return unless($file); 33 | 34 | my $fh = IO::Scalar->new(\$uncompressed); 35 | 36 | my $tar = Archive::Tar->new; 37 | $tar->read($fh) or die "Cannot read from \$fh"; 38 | my $untarred = $tar->get_content($file); 39 | 40 | die("couldn't extract $file from archive") unless($untarred); 41 | return $untarred; 42 | 43 | } 44 | 45 | 1; 46 | 47 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Decode/Gzip.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Decode::Gzip; 2 | 3 | use Compress::Zlib; 4 | 5 | sub decode { 6 | my $class = shift; 7 | my $data = shift; 8 | my $type = shift; 9 | return unless($type =~ /gzip/); 10 | 11 | my $uncompressed = Compress::Zlib::memGunzip($data); 12 | my $ft = File::Type->new(); 13 | my $t = $ft->mime_type($uncompressed); 14 | 15 | return unless($t =~ /octet-stream/); #only return octect streams(aka text) 16 | return $uncompressed; 17 | } 18 | 19 | 1; 20 | 21 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Decode/Zip.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Decode::Zip; 2 | 3 | use strict; 4 | use warnings; 5 | use IO::Uncompress::Unzip qw(unzip $UnzipError); 6 | 7 | sub decode { 8 | my $class = shift; 9 | my $data = shift; 10 | my $type = shift; 11 | my $f = shift; 12 | return unless($type =~ /zip/ && $type !~ /gzip/); 13 | 14 | my $file; 15 | if($f->{'zip_filename'}){ 16 | $file = $f->{'zip_filename'}; 17 | } else { 18 | $f->{'feed'} =~ m/\/([a-zA-Z0-9_]+).zip$/; 19 | $file = $1; 20 | } 21 | return unless($file); 22 | 23 | my $unzipped; 24 | unzip \$data => \$unzipped, Name => $file || die('unzip failed: '.$UnzipError); 25 | return $unzipped; 26 | } 27 | 28 | 1; 29 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor; 2 | use base 'CIF::Smrt'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use CIF qw/generate_uuid_random generate_uuid_ns/; 8 | use Iodef::Pb ':all'; 9 | use Net::Abuse::Utils qw(get_as_description get_asn_info); 10 | 11 | sub process {} 12 | 13 | sub degrade_confidence { 14 | my $class = shift; 15 | my $confidence = shift; 16 | 17 | for(lc($confidence)){ 18 | if(/^\d+/){ 19 | my $log = log($confidence) / log(500); 20 | $confidence = sprintf('%.3f',($confidence * $log)); 21 | return($confidence); 22 | } 23 | if(/^high$/){ 24 | return 'medium'; 25 | } 26 | if(/^medium$/){ 27 | return 'low'; 28 | } 29 | } 30 | } 31 | 32 | sub resolve_bgp { 33 | my $class = shift; 34 | my $a = shift; 35 | return undef unless($a); 36 | 37 | # aggregate our cache, we could miss a more specific route, but unlikely 38 | ## TODO -- improve for ipv6 39 | my @bits = split(/\./,$a); 40 | return if($#bits < 0); 41 | 42 | $bits[$#bits] = '0'; 43 | $a = join('.',@bits); 44 | 45 | my ($as,$network,$ccode,$rir,$date) = get_asn_info($a); 46 | my $as_desc; 47 | $as_desc = get_as_description($as) if($as); 48 | 49 | $as = undef if($as && $as eq 'NA'); 50 | $network = undef if($network && $network eq 'NA'); 51 | $ccode = undef if($ccode && $ccode eq 'NA'); 52 | $rir = undef if($rir && $rir eq 'NA'); 53 | $date = undef if($date && $date eq 'NA'); 54 | $as_desc = undef if($as_desc && $as_desc eq 'NA'); 55 | $a = undef if($a eq ''); 56 | 57 | return({ 58 | asn => $as, 59 | prefix => $network, 60 | cc => $ccode, 61 | date => $date, 62 | asn_desc => $as_desc, 63 | rir => $rir, 64 | }); 65 | } 66 | 67 | 1; 68 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor/Fqdn.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor::Fqdn; 2 | use base 'CIF::Smrt::Plugin::Postprocessor'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple ':all'; 8 | require Net::DNS::Resolver; 9 | 10 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 11 | 12 | my @plugins = __PACKAGE__->plugins(); 13 | 14 | use constant RE_REGEX => qr/^(?:[0-9a-zA-Z-]{1,63}\.)+[a-zA-Z]{2,63}$/; 15 | 16 | sub process { 17 | my $class = shift; 18 | my $smrt = shift; 19 | my $data = shift; 20 | 21 | my $addresses = iodef_addresses($data); 22 | return unless($#{$addresses} > -1); 23 | 24 | my $array; 25 | foreach (@plugins){ 26 | my $r = $_->process($smrt,$data); 27 | push(@$array,@$r) if($r && @$r); 28 | } 29 | return $array; 30 | } 31 | 32 | sub is_fqdn { 33 | my $class = shift; 34 | my $addr = shift; 35 | 36 | return unless($addr && $addr->get_content()); 37 | return unless($addr->get_category() == AddressType::AddressCategory::Address_category_ext_value()); 38 | return unless($addr->get_ext_category() =~ /^(fqdn|domain)$/); 39 | return 1 if($addr->get_content() =~ RE_REGEX); 40 | } 41 | 42 | sub resolve { 43 | my $class = shift; 44 | my $addr = shift; 45 | 46 | my $r = Net::DNS::Resolver->new(recursive => 0); 47 | $r->udp_timeout(2); 48 | $r->tcp_timeout(2); 49 | 50 | my $pkt = $r->send($addr); 51 | return unless($pkt); 52 | my @rdata = $pkt->answer(); 53 | return(\@rdata); 54 | } 55 | 56 | sub resolve_ns { 57 | my $class = shift; 58 | my $addr = shift; 59 | 60 | my $r = Net::DNS::Resolver->new(recursive => 0); 61 | $r->udp_timeout(2); 62 | $r->tcp_timeout(2); 63 | 64 | my $pkt = $r->send($addr,'NS'); 65 | return unless($pkt); 66 | my @rdata = $pkt->answer(); 67 | return(\@rdata); 68 | } 69 | 70 | 71 | 1; 72 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor/Ip.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor::Ip; 2 | use base 'CIF::Smrt::Plugin::Postprocessor'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple ':all'; 8 | use Regexp::Common qw/net/; 9 | 10 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 11 | 12 | my @plugins = __PACKAGE__->plugins(); 13 | 14 | sub process { 15 | my $class = shift; 16 | my $smrt = shift; 17 | my $data = shift; 18 | 19 | my $addresses = iodef_addresses($data); 20 | return unless($#{$addresses} > -1); 21 | 22 | my $array; 23 | foreach (@plugins){ 24 | my $r = $_->process($smrt,$data); 25 | push(@$array,@$r) if($r && @$r); 26 | } 27 | return $array; 28 | } 29 | 30 | sub is_ipv4 { 31 | my $class = shift; 32 | my $addr = shift; 33 | 34 | return 0 unless($addr->get_content() =~ /^$RE{'net'}{'IPv4'}/); 35 | 36 | return 0 unless($addr && $addr->get_content()); 37 | return 1 if($addr->get_category() == AddressType::AddressCategory::Address_category_ipv4_addr()); 38 | return 1 if($addr->get_category() == AddressType::AddressCategory::Address_category_ipv4_net()); 39 | return 1 if($addr->get_category() == AddressType::AddressCategory::Address_category_ipv4_net_mask()); 40 | } 41 | 42 | sub is_ipv6 { 43 | my $class = shift; 44 | my $addr = shift; 45 | 46 | return 1 if($addr->get_category() == AddressType::AddressCategory::Address_category_ipv6_addr()); 47 | return 1 if($addr->get_category() == AddressType::AddressCategory::Address_category_ipv6_net()); 48 | return 1 if($addr->get_category() == AddressType::AddressCategory::Address_category_ipv6_net_mask()); 49 | } 50 | 51 | 1; -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor/Ip/PrefixWhitelist.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor::Ip::PrefixWhitelist; 2 | use base 'CIF::Smrt::Plugin::Postprocessor::Ip'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use CIF qw/generate_uuid_random/; 8 | use Regexp::Common qw/net/; 9 | use Iodef::Pb::Simple ':all'; 10 | 11 | sub process { 12 | my $class = shift; 13 | my $smrt = shift; 14 | my $data = shift; 15 | 16 | my @new_ids; 17 | foreach my $i (@{$data->get_Incident()}){ 18 | next unless($i->get_EventData()); 19 | 20 | my $impact = iodef_impacts_first($i); 21 | ## TODO -- this is a work-around 22 | return unless($impact->get_content()->get_content() =~ /^whitelist$/); 23 | 24 | my $assessment = $i->get_Assessment(); 25 | my $confidence = @{$assessment}[0]->get_Confidence(); 26 | $confidence = $confidence->get_content(); 27 | $confidence = $class->degrade_confidence($confidence); 28 | 29 | next unless($confidence > 25); 30 | 31 | my $restriction = $i->get_restriction(); 32 | my $description = @{$i->get_Description()}[0]->get_content(); 33 | 34 | my $altids = $i->get_RelatedActivity(); 35 | $altids = $altids->get_IncidentID() if($altids); 36 | 37 | my $bgp = iodef_bgp($i) || next(); 38 | foreach (@$bgp){ 39 | next unless($_->{'prefix'}); 40 | my $new_id = IncidentIDType->new({ 41 | content => generate_uuid_random(), 42 | instance => $smrt->get_instance(), 43 | name => $smrt->get_name(), 44 | restriction => $restriction, 45 | }); 46 | my $new = Iodef::Pb::Simple->new({ 47 | address => $_->{'prefix'}, 48 | prefix => $_->{'prefix'}, 49 | cc => $_->{'cc'}, 50 | rir => $_->{'rir'}, 51 | asn => $_->{'asn'}, 52 | asn_desc => $_->{'asn_desc'}, 53 | IncidentID => $new_id, 54 | assessment => 'whitelist', 55 | description => $description.' prefix', 56 | confidence => $confidence, 57 | restriction => $restriction, 58 | RelatedActivity => RelatedActivityType->new({ 59 | IncidentID => [ $i->get_IncidentID() ], 60 | restrcition => $restriction, 61 | }), 62 | guid => iodef_guid($i), 63 | AlternativeID => $i->get_AlternativeID(), 64 | 65 | }); 66 | push(@new_ids,@{$new->get_Incident()}[0]); 67 | push(@$altids, $new_id); 68 | } 69 | if($altids){ 70 | $i->set_RelatedActivity( 71 | RelatedActivityType->new({ 72 | IncidentID => $altids, 73 | }) 74 | ); 75 | } 76 | } 77 | return(\@new_ids); 78 | } 79 | 80 | 1; -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor::Malware; 2 | use base 'CIF::Smrt::Plugin::Postprocessor'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_malware); 8 | 9 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 10 | 11 | my @plugins = __PACKAGE__->plugins(); 12 | 13 | sub process { 14 | my $class = shift; 15 | my $smrt = shift; 16 | my $data = shift; 17 | 18 | my $malware = iodef_malware($data); 19 | return unless($malware); 20 | 21 | my $array; 22 | foreach (@plugins){ 23 | my $r = $_->process($smrt,$data); 24 | push(@$array,@$r) if($r && @$r); 25 | } 26 | return $array; 27 | } 28 | 29 | 1; -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor/Malware/TCMalwareRegistry.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor::Malware::TCMalwareRegistry; 2 | use base 'CIF::Smrt::Plugin::Postprocessor::Malware'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Regexp::Common qw/URI net/; 8 | use Iodef::Pb::Simple qw(iodef_malware); 9 | use Net::Abuse::Utils; 10 | 11 | sub process { 12 | my $class = shift; 13 | my $smrt = shift; 14 | my $data = shift; 15 | 16 | foreach my $i (@{$data->get_Incident()}){ 17 | my $malware = iodef_malware($i); 18 | foreach my $m (@$malware){ 19 | next unless($m->get_meaning() eq 'malware hash'); 20 | my $ret = get_malware($m->get_content()); 21 | if($ret){ 22 | my $timestamp = DateTime->from_epoch(epoch => $ret->{'last_seen'}); 23 | my $drate = ExtensionType->new({ 24 | meaning => 'tc malware registry detection rate', 25 | formatid => 'percentage', 26 | content => $ret->{'detection_rate'}, 27 | dtype => ExtensionType::DtypeType::dtype_type_real(), 28 | restriction => RestrictionType::restriction_type_public(), 29 | }); 30 | my $drate_timestamp = ExtensionType->new({ 31 | meaning => 'tc malware registry detection rate timestamp', 32 | formatid => 'timestamp', 33 | content => $timestamp->ymd().'T'.$timestamp->hms().'Z', 34 | dtype => ExtensionType::DtypeType::dtype_type_string(), 35 | restriction => RestrictionType::restriction_type_public(), 36 | }); 37 | push(@{$i->get_AdditionalData},($drate,$drate_timestamp)); 38 | } 39 | } 40 | } 41 | } 42 | 43 | ## TODO -- replace when Net::Abuse::Utils is updated to v0.13 44 | sub get_malware { 45 | my $hash = shift; 46 | return unless($hash && lc($hash) =~ /^[a-z0-9]{32}$/); 47 | 48 | my $lookup = $hash.'.malware.hash.cymru.com'; 49 | 50 | my $res = Net::Abuse::Utils::_return_rr($lookup, 'TXT') or return; 51 | my ($last_seen,$detection_rate) = split(/ /,$res); 52 | return({ 53 | last_seen => $last_seen, 54 | detection_rate => $detection_rate, 55 | }); 56 | } 57 | 1; -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Postprocessor/Url.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Postprocessor::Url; 2 | use base 'CIF::Smrt::Plugin::Postprocessor'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Regexp::Common qw/URI net/; 8 | use Iodef::Pb::Simple ':all'; 9 | 10 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 11 | 12 | my @plugins = __PACKAGE__->plugins(); 13 | 14 | sub process { 15 | my $class = shift; 16 | my $smrt = shift; 17 | my $data = shift; 18 | 19 | my $addresses = iodef_addresses($data); 20 | return unless($#{$addresses} > -1); 21 | 22 | my $found = 0; 23 | foreach (@$addresses){ 24 | next unless($class->is_url($_)); 25 | $found = 1; 26 | } 27 | return unless($found); 28 | 29 | my $array; 30 | foreach (@plugins){ 31 | my $r = $_->process($smrt,$data); 32 | push(@$array,@$r) if($r && @$r); 33 | } 34 | return $array; 35 | } 36 | 37 | sub is_url { 38 | my $class = shift; 39 | my $addr = shift; 40 | 41 | return unless($addr->get_category() == AddressType::AddressCategory::Address_category_ext_value()); 42 | return unless($addr->get_ext_category() eq 'url'); 43 | return 1 if($addr->get_content() =~ /^$RE{'URI'}{'HTTP'}/ || $addr->get_content() =~ /^$RE{'URI'}{'HTTP'}{-scheme => 'https'}$/); 44 | } 45 | 46 | 1; -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Preprocessor/Address.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Preprocessor::Address; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use Regexp::Common qw/net/; 7 | use Regexp::Common::net::CIDR; 8 | 9 | sub process { 10 | my $class = shift; 11 | my $rules = shift; 12 | my $rec = shift; 13 | 14 | return $rec unless($rec->{'address'}); 15 | return $rec if ($rec->{'atype'} && $rec->{'atype'} ne 'ipv4'); 16 | $rec->{'address'} = lc($rec->{'address'}); 17 | 18 | if($rec->{'address'} =~ /^$RE{'net'}{'CIDR'}{'IPv4'}{'-keep'}$/){ 19 | my $min = $rec->{'min_prefix'} || 14; 20 | if ($2 < $min){ 21 | $rec->{'address'} = $1.'/'.$min; 22 | } 23 | } 24 | return $rec; 25 | } 26 | 27 | 1; 28 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Preprocessor/Description.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Preprocessor::Description; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub process { 7 | my $class = shift; 8 | my $rules = shift; 9 | my $rec = shift; 10 | 11 | $rec->{'assessment'} = lc($rec->{'assessment'}) if($rec->{'assessment'}); 12 | unless($rec->{'description'}){ 13 | $rec->{'description'} = 'unknown'; 14 | } else { 15 | $rec->{'description'} = lc($rec->{'description'}); 16 | } 17 | return $rec; 18 | } 19 | 20 | 1; 21 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Preprocessor/Email.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Preprocessor::Email; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub process { 7 | my $class = shift; 8 | my $rules = shift; 9 | my $rec = shift; 10 | 11 | my $address = $rec->{'address'}; 12 | return $rec unless($address); 13 | 14 | # Regexp::Common qw/URI/ chokes on large urls 15 | return $rec if($address =~ /^(ftp|https?):\/\//); 16 | 17 | if($address =~ /([a-z0-9_.-]+\@[a-z0-9.-]+\.[a-z0-9.-]{2,6})/){ 18 | $rec->{'address'} = $1; 19 | } 20 | 21 | return $rec; 22 | } 23 | 24 | 1; 25 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Preprocessor/Portlist.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Preprocessor::Portlist; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub process { 7 | my $class = shift; 8 | my $rules = shift; 9 | my $rec = shift; 10 | 11 | return $rec unless($rec->{'portlist'}); 12 | return $rec if($rec->{'portlist'} =~ /^\d+$/); 13 | 14 | for($rec->{'portlist'}){ 15 | if(/_/){ 16 | $rec->{'portlist'} =~ s/_/,/g; 17 | last; 18 | } 19 | } 20 | 21 | return $rec; 22 | } 23 | 24 | 1; 25 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Preprocessor/Uri.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Preprocessor::Uri; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use Regexp::Common qw/net/; 7 | use Regexp::Common::net::CIDR; 8 | 9 | sub process { 10 | my $class = shift; 11 | my $rules = shift; 12 | my $rec = shift; 13 | 14 | my $address = $rec->{'address'}; 15 | return $rec unless($address); 16 | return $rec if($rec->{'atype'} && $rec->{'atype'} ne 'url'); 17 | if ($address =~ /^$RE{'net'}{'IPv4'}$/ || $address =~ /^$RE{'net'}{'CIDR'}{'IPv4'}$/){ 18 | return $rec unless($rec->{'atype'} && $rec->{'atype'} eq 'url'); 19 | } 20 | 21 | # Regexp::Common qw/URI/ chokes on large urls 22 | return $rec if($address =~ /^(ftp|https?):\/\//); 23 | 24 | if($rec->{'address'} =~ /^([a-z0-9.-]+[a-z]{2,6}|\b(?:\d{1,3}\.){3}\d{1,3}\b)(:\d+)?\/+/){ 25 | $rec->{'address'} = 'http://'.$rec->{'address'}; 26 | } 27 | return $rec; 28 | } 29 | 30 | 1; 31 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Pull/Cif.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Pull::Cif; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use CIF; 7 | use CIF::Client; 8 | use Iodef::Pb::Simple; 9 | 10 | sub pull { 11 | my $class = shift; 12 | my $f = shift; 13 | return unless($f->{'cif'} && $f->{'cif'} eq 'true'); 14 | 15 | my ($err,$ret) = CIF::Client->new({ 16 | config => $f->{'config'} || $f->{'client_config'} || $ENV{'/home/cif/.cif'}, 17 | }); 18 | return($err) if($err); 19 | 20 | my $confidence = $f->{'confidence'} || 85; 21 | 22 | ($err,$ret) = $ret->search({ 23 | query => $f->{'feed'}, 24 | confidence => $confidence, 25 | nolog => 1, 26 | no_decode => 1, 27 | }); 28 | return($err) if($err); 29 | # multiple... 30 | 31 | $ret = 'application/cif'."\n".join("\n",@$ret); 32 | 33 | return(undef,$ret); 34 | } 35 | 36 | 1; 37 | -------------------------------------------------------------------------------- /cif-smrt/lib/CIF/Smrt/Plugin/Pull/File.pm: -------------------------------------------------------------------------------- 1 | package CIF::Smrt::Plugin::Pull::File; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | sub pull { 7 | my $class = shift; 8 | my $f = shift; 9 | 10 | return unless($f->{'feed'} =~ /^(\/\S+|[a-zA-Z]+\/\S+)/); 11 | my $file = $1; 12 | if($file =~ /^([a-zA-Z]+)/){ 13 | ## TODO -- work-around, path should be passed to me by the higher level lib 14 | # || /opt/cif/bin is in case we run $ cif_crontool as is with no preceeding path 15 | my $bin_path = $FindBin::Bin || '/opt/cif/bin'; 16 | # see if we're working out of a -dev directory 17 | if(-e './rules'){ 18 | $file = $bin_path.'/../rules/'.$file; 19 | } else { 20 | $file = $bin_path.'/../'.$file; 21 | } 22 | } 23 | open(F,$file) || return($!.': '.$file); 24 | my @lines = ; 25 | close(F); 26 | 27 | if(my $l = $f->{'feed_limit'}){ 28 | my ($start,$end); 29 | if(ref($l) eq 'ARRAY'){ 30 | ($start,$end) = @{$l}; 31 | } else { 32 | ($start,$end) = (0,$l-1); 33 | } 34 | $end = $#lines if ($end > $#lines); 35 | @lines = @lines[$start..$end]; 36 | } 37 | my $content = join('',@lines) || ''; 38 | return(undef,$content); 39 | } 40 | 41 | 1; 42 | -------------------------------------------------------------------------------- /cif-smrt/rebuild.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make 5 | make dist 6 | -------------------------------------------------------------------------------- /cif-smrt/rules/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-smrt/rules/AUTHORS -------------------------------------------------------------------------------- /cif-smrt/rules/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = etc 2 | 3 | ME=cif-smrt-rules 4 | BUILD=$(shell git describe --tags) 5 | SNAPSHOT=$(ME)-$(BUILD) 6 | 7 | all: default 8 | 9 | default: 10 | @echo "Please read the README before beginning your installation." 11 | 12 | fixperms: 13 | chmod 0755 $(prefix) 14 | chown -R $(CIF_USER) $(prefix) 15 | chgrp -R $(CIF_GROUP) $(prefix) 16 | 17 | install: etc-install fixperms 18 | 19 | etc-link: 20 | [ -d /etc/cif ] || ln -sf $(prefix)/etc /etc/cif 21 | 22 | etc-install: 23 | [ -d $(sysconfdir) ] || $(INSTALL) -m 0755 -d $(sysconfdir) 24 | -( cd etc && find . -type d -print ) | while read dir ; do \ 25 | $(INSTALL) -m 0750 -d "$(sysconfdir)/$$dir" ; \ 26 | done 27 | -( cd etc && find . -type f -print ) | while read file ; do \ 28 | $(INSTALL) -m 0640 "etc/$$file" "$(sysconfdir)/$$file" ; \ 29 | done 30 | 31 | clean: 32 | rm Makefile 33 | rm config.log 34 | rm config.status 35 | 36 | realclean: 37 | clean 38 | rm Makefile.in 39 | rm configure 40 | rm aclocal.m4 41 | 42 | reconf: 43 | autoreconf -vf 44 | 45 | snapshot: 46 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 47 | -------------------------------------------------------------------------------- /cif-smrt/rules/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-smrt/rules/NEWS -------------------------------------------------------------------------------- /cif-smrt/rules/README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | This is the SMRT intelligence rule-set. 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | 12 | Resources 13 | --------- 14 | 15 | Website: collectiveintel.net 16 | Git repository: github.com/collectiveintel/cif-smrt-rules 17 | 18 | 19 | COPYRIGHT AND LICENCE 20 | --------------------- 21 | Copyright (C) 2012 Wes Young (claimid.com/wesyoung) 22 | Copyright (C) 2012 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 23 | 24 | Free use of this software is granted under the terms of the GNU Lesser General 25 | Public License (LGPL). For details see the files `COPYING` included with the 26 | distribution. 27 | -------------------------------------------------------------------------------- /cif-smrt/rules/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.64]) 5 | AC_INIT(cif-smrt-rules, m4_esyscmd_s([git describe --tags]), [ci-framework@googlegroups.com]) 6 | AM_INIT_AUTOMAKE 7 | 8 | dnl Defaults paths for installation 9 | AC_PREFIX_DEFAULT([/opt/cif]) 10 | 11 | AC_CONFIG_FILES([ 12 | Makefile 13 | ]) 14 | 15 | 16 | AC_OUTPUT 17 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/00_alexa_whitelist.cfg: -------------------------------------------------------------------------------- 1 | feed = 'http://s3.amazonaws.com/alexa-static/top-1m.csv.zip' 2 | delimiter = ',' 3 | values = 'rank,address' 4 | description = 'alexa #' 5 | assessment = 'whitelist' 6 | alternativeid = 'http://www.alexa.com/siteinfo/
' 7 | alternativeid_restriction = public 8 | source = 'alexa.com' 9 | restriction = need-to-know 10 | guid = everyone 11 | mirror = '/tmp' 12 | zip_filename = 'top-1m.csv' 13 | 14 | # remember arrays start at 0 15 | # 1-10 is really 0-9 16 | [top10000] 17 | feed_limit = 1000,9999 18 | confidence = 25 19 | 20 | [top1000] 21 | feed_limit = 100,999 22 | confidence = 50 23 | 24 | [top100] 25 | feed_limit = 10,99 26 | confidence = 75 27 | 28 | [top10] 29 | feed_limit = 10 30 | confidence = 95 31 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/00_mirc_whitelist.cfg: -------------------------------------------------------------------------------- 1 | feed = http://www.mirc.com/servers.ini 2 | alternativeid = 'http://www.mirc.com/servers.ini' 3 | alternativeid_restriction = public 4 | source = 'mirc.com' 5 | restriction = need-to-know 6 | confidence = 85 7 | description = 'known irc host' 8 | protocol = 6 9 | guid = everyone 10 | 11 | [domains] 12 | regex = 'SERVER:([a-zA-Z0-9-.]+\.[a-z]{2,3}):(\S+)GROUP' 13 | regex_values = 'address,portlist' 14 | assessment = 'whitelist' 15 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/00_whitelist.cfg: -------------------------------------------------------------------------------- 1 | restriction = 'need-to-know' 2 | guid = everyone 3 | alternativeid_restriction = 'public' 4 | source = 'localhost' 5 | description = 'unknown local' 6 | assessment = 'whitelist' 7 | 8 | [url_shortners] 9 | feed = 'http://mirror3.malwaredomains.com/files/url_shorteners.zip' 10 | description = 'url shortening service' 11 | regex = '^(\S+)' 12 | regex_values = 'address' 13 | confidence = 85 14 | alternativeid = 'http://mirror1.malwaredomains.com/files/url_shorteners.txt' 15 | alternativeid_restriction = need-to-know 16 | mirror = /tmp 17 | zip_filename = 'url_shorteners.txt' 18 | 19 | [domains] 20 | feed = etc/whitelist_domains 21 | regex = '^(\S+)$' 22 | regex_values = 'address' 23 | confidence = 95 24 | 25 | [infrastructure] 26 | feed = etc/whitelist_infrastructure 27 | regex = '^(\S+)$' 28 | regex_values = 'address' 29 | confidence = 95 30 | 31 | [urls] 32 | feed = etc/whitelist_urls 33 | regex = '^(\S+)$' 34 | regex_values = 'address' 35 | confidence = 95 36 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/alienvault.cfg: -------------------------------------------------------------------------------- 1 | feed = http://reputation.alienvault.com/reputation.data 2 | source = 'reputation.alienvault.com' 3 | guid = everyone 4 | confidence = 65 5 | severity = medium 6 | restriction = 'need-to-know' 7 | alternativeid = "http://reputation.alienvault.com/reputation.generic" 8 | alternativeid_restriction = 'public' 9 | mirror = /tmp 10 | 11 | [botnet] 12 | regex = '^(\b(?:\d{1,3}\.){3}\d{1,3}\b).*#.*(C&C|APT);?.*' 13 | regex_values= 'address,description' 14 | assessment = 'botnet' 15 | severity = high 16 | 17 | # disabled for now 18 | # feed as grown to 500k, could overwhelm systems with low memory 19 | # copy to custom.cfg if you want to use with the top level meta data too 20 | [scan] 21 | regex = '^(\b(?:\d{1,3}\.){3}\d{1,3}\b).*#.*Scanning Host;?\S+' 22 | regex_values= 'address' 23 | assessment = 'scanner' 24 | period = daily 25 | disabled = true 26 | 27 | [phishing] 28 | regex = '^(\b(?:\d{1,3}\.){3}\d{1,3}\b).*#.*Phishing;?.*' 29 | regex_values= 'address' 30 | assessment = 'phishing' 31 | period = daily 32 | 33 | [spam] 34 | regex = '^(\b(?:\d{1,3}\.){3}\d{1,3}\b).*#.*Spamming;?.*' 35 | regex_values= 'address' 36 | assessment = 'spam' 37 | 38 | [malware] 39 | regex = '^(\b(?:\d{1,3}\.){3}\d{1,3}\b).*#.*(Malware IP|Malware Domain|Malicious Host|Malware distribution|RBN);?.*' 40 | regex_values = 'address,description' 41 | assessment = 'malware' -------------------------------------------------------------------------------- /cif-smrt/rules/etc/aper.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | restriction = 'need-to-know' 3 | guid = everyone 4 | source = 'code.google.com/p/anti-phishing-email-reply' 5 | feed = 'http://aper.svn.sourceforge.net/svnroot/aper/phishing_reply_addresses' 6 | assessment = 'phishing' 7 | alternativeid = 'http://svn.code.sf.net/p/aper/code/phishing_reply_addresses' 8 | alternativeid_restriction = 'public' 9 | 10 | [content] 11 | description = 'address contained in content' 12 | severity = 'medium' 13 | confidence = 75 14 | regex = '^(\S+),[C|D|E],(\S+)$' 15 | regex_values = "address,detecttime" 16 | 17 | [header] 18 | description = 'address used in replyto/from header' 19 | alternativeid = 'http://svn.code.sf.net/p/aper/code/phishing_reply_addresses' 20 | regex = '^(\S+),[A|B],(\S+)$' 21 | regex_values = "address,detecttime" 22 | confidence = 85 23 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/bambenekconsulting.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | source = 'bambenekconsulting.com' 3 | alternativeid_restriction = 'public' 4 | restriction = 'need-to-know' 5 | guid = everyone 6 | mirror = '/tmp' 7 | period = hourly 8 | 9 | [goz_domain] 10 | feed = 'http://osint.bambenekconsulting.com/feeds/goz-domlist.txt' 11 | values = 'address,description,detecttime,alternativeid' 12 | delimiter = ',' 13 | severity = high 14 | confidence = 85 15 | assessment = 'botnet' 16 | 17 | [goz_ip] 18 | feed = 'http://osint.bambenekconsulting.com/feeds/goz-iplist.txt' 19 | values = 'address,description,detecttime,alternativeid' 20 | delimiter = ',' 21 | severity = high 22 | confidence = 85 23 | assessment = 'botnet' 24 | 25 | [dga_feed] 26 | feed = 'http://osint.bambenekconsulting.com/feeds/dga-feed.csv' 27 | values = 'address,description,detecttime,alternativeid' 28 | delimiter = ',' 29 | severity = medium 30 | confidence = 75 31 | assessment = 'suspicious' 32 | period = daily 33 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/cleanmx.cfg: -------------------------------------------------------------------------------- 1 | # re-enable at your own risk 2 | # their XML breaks constantly 3 | severity = 'medium' 4 | confidence = 85 5 | restriction = 'need-to-know' 6 | alternativeid_restriction = 'public' 7 | source = 'clean-mx.de' 8 | guid = everyone 9 | 10 | [malware] 11 | feed = 'http://support.clean-mx.de/clean-mx/xmlviruses.php?' 12 | assessment = 'malware' 13 | node = entry 14 | elements = 'id,first,malware_md5,virusname,url' 15 | elements_map = 'id,detecttime,malware_md5,description,address' 16 | alternativeid = 'http://support.clean-mx.de/clean-mx/viruses.php?id=' 17 | disabled = true 18 | 19 | # re-enable at your own risk 20 | # their XML breaks constantly 21 | [phishing] 22 | feed = 'http://support.clean-mx.de/clean-mx/xmlphishing.php?' 23 | assessment = 'phishing' 24 | source = 'clean-mx.de' 25 | node = entry 26 | elements = 'id,first,malware_md5,virusname,url' 27 | elements_map = 'id,detecttime,malware_md5,description,address' 28 | alternativeid = 'http://support.clean-mx.de/clean-mx/viruses.php?id=' 29 | disabled = true 30 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/custom.cfg.example: -------------------------------------------------------------------------------- 1 | [phishtank] 2 | guid = everyone 3 | # phishtank asks but not requires that you get your own apikey for their API 4 | feed = 'http://data.phishtank.com/data/XXXXXXXXXXXXXXXXXXXXXXXXXXXX INSERT YOUR KEY HERE XXXXXXXXXXXXXXXXXXXXXX/online-valid.json.gz' 5 | # if you don't get one, cif-smrt will default to this feed url instead of the one above, but PT might block you eventually 6 | feed = http://data.phishtank.com/data/online-valid.json.gz 7 | assessment = 'phishing' 8 | source = 'phishtank.com' 9 | fields = 'submission_time,url,target,phish_detail_url' 10 | fields_map = 'detecttime,address,description,alternativeid' 11 | severity = 'medium' 12 | confidence = 85 13 | restriction = 'need-to-know' 14 | alternativeid_restriction = 'public' 15 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/drg.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | confidence = 85 3 | restriction = 'need-to-know' 4 | alternativeid_restriction = 'public' 5 | guid = everyone 6 | source = 'dragonresearchgroup.org' 7 | 8 | [ssh] 9 | feed = 'http://dragonresearchgroup.org/insight/sshpwauth.txt' 10 | regex = '^\d+\s+\|\s+[\S|\s]+\|\s+(\S+)\s+\|\s+(\S+\s\S+)\s+\|' 11 | regex_values = 'address,detecttime' 12 | source = 'dragonresearchgroup.org' 13 | assessment = 'scanner' 14 | description = 'ssh' 15 | portlist = 22 16 | protocol = tcp 17 | alternativeid = 'http://dragonresearchgroup.org/insight/sshpwauth.txt' 18 | period = hourly 19 | 20 | [vnc] 21 | feed = 'http://dragonresearchgroup.org/insight/vncprobe.txt' 22 | regex = '^\d+\s+\|\s+[\S|\s]+\|\s+(\S+)\s+\|\s+(\S+\s\S+)\s+\|' 23 | regex_values = 'address,detecttime' 24 | assessment = 'scanner' 25 | description = vnc 26 | portlist = 5900-5904 27 | protocol = tcp 28 | alternativeid = 'https://dragonresearchgroup.org/insight/vncprobe.txt' 29 | period = hourly 30 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/feodotracker.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | source = 'feodotracker.abuse.ch' 3 | alternativeid_restriction = 'public' 4 | regex_guid = '(\S+)' 5 | regex_guid_values = 'alternativeid' 6 | restriction = 'need-to-know' 7 | description = 'feodo' 8 | guid = everyone 9 | 10 | [domainblocklist] 11 | # Domains used soley for Feodo C&C; chance of blocking legitimate content 12 | # very low 13 | feed = 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist' 14 | regex = '^([A-Za-z0-9.-]+\.[a-zA-Z]{2,6})$' 15 | regex_values = 'address' 16 | assessment = 'botnet' 17 | description = 'feodo' 18 | severity = high 19 | confidence = 65 20 | alternativeid = 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist' 21 | 22 | [ipblocklist] 23 | # Includes Type A and Type B C&C hosts; chance of blocking legitimate 24 | # content if compromised server included (Type A) 25 | feed = 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist' 26 | regex = '^(\S+)$' 27 | regex_values = 'address' 28 | assessment = 'botnet' 29 | description = 'feodo' 30 | severity = high 31 | confidence = 40 32 | alternativeid = 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist' 33 | 34 | [badips] 35 | # Includes only Type B C&C hosts; chance of blocking legitimate content 36 | # very low 37 | feed = 'https://feodotracker.abuse.ch/blocklist/?download=badips' 38 | regex = '^(\S+)$' 39 | regex_values = 'address' 40 | assessment = 'botnet' 41 | description = 'feodo' 42 | severity = high 43 | confidence = 40 44 | alternativeid = 'https://feodotracker.abuse.ch/blocklist/?download=badips' 45 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/malc0de.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | severity = 'medium' 3 | confidence = 85 4 | restriction = 'need-to-know' 5 | alternativeid_restriction = 'public' 6 | guid = everyone 7 | source = 'malc0de.com' 8 | feed = 'http://malc0de.com/rss' 9 | 10 | [url] 11 | atype = 'url' 12 | assessment = 'malware' 13 | description = 'unknown' 14 | regex_description = '^URL: ([\S\s]+)?, IP Address: \S+?, Country: \S+?, ASN: \S+?, MD5: (\S+)?' 15 | regex_description_values = 'address,malware_md5' 16 | regex_link = '(\S+)' 17 | regex_link_values = 'alternativeid' 18 | 19 | [malware] 20 | atype = 'url' 21 | assessment = 'malware' 22 | severity = high 23 | regex_description = '^URL: [\S+|\s]+?, IP Address: \S+?, Country: \S+?, ASN: \S+?, MD5: (\S+)?' 24 | regex_description_values = 'malware_md5' 25 | regex_link = '(\S+)' 26 | regex_link_values = 'alternativeid' 27 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/malwaredomainlist.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | severity = 'medium' 3 | restriction = 'need-to-know' 4 | alternativeid_restriction = 'public' 5 | alternativeid = 'http://www.malwaredomainlist.com/mdl.php?search=
' 6 | guid = everyone 7 | source = 'malwaredomainlist.com' 8 | 9 | [malwaredomainlist] 10 | # '"2011/02/14_19:35","-","91.213.29.45/~samui/jhgth/fgdsfdty/panos.php","-","zeus v2 drop zone","-","51786"' 11 | #feed = 'http://www.malwaredomainlist.com/mdlcsv.php' 12 | feed = 'http://www.malwaredomainlist.com/updatescsv.php' 13 | values = 'null,address,null,null,description,null,null,null' 14 | assessment = 'malware' 15 | confidence = 85 16 | 17 | [malwaredomainlist_registrants] 18 | feed = 'http://www.malwaredomainlist.com/updatescsv.php' 19 | regex = '(\d{4}/\d{2}/\d{2})[_\d:]+.*["\s:]([a-z0-9_.%-]+\@[a-z0-9-.]+\.[a-z0-9.-]{2,6})\b' 20 | regex_values = 'detecttime,address' 21 | assessment = 'registrant address' 22 | severity = low 23 | confidence = 95 24 | 25 | [malwaredomainlist_infra] 26 | # "2011/05/09_14:20","mathres.com/data/gate.php","216.239.139.112","216-239-139.112.cluster.omnis.com.","zeus v1 drop zone","mathres.com@guardedwhois.com","19237" 27 | #feed = 'http://www.malwaredomainlist.com/mdlcsv.php' 28 | feed = 'http://www.malwaredomainlist.com/updatescsv.php' 29 | values = 'detecttime,null,address,null,description,null,null,null' 30 | assessment = 'malware' 31 | confidence = 65 32 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/malwaredomains.cfg: -------------------------------------------------------------------------------- 1 | severity = medium 2 | confidence = 75 3 | feed = http://mirror3.malwaredomains.com/files/domains.zip 4 | assessment = 'malware' 5 | source = 'malwaredomains.com' 6 | restriction = need-to-know 7 | guid = everyone 8 | alternativeid = 'http://mirror3.malwaredomains.com/files/domains.txt' 9 | alternativeid_restriction = public 10 | zip_filename = 'domains.txt' 11 | mirror = '/tmp' 12 | 13 | [domains] 14 | values = 'null,null,address,description,null,null' 15 | delimiter = '[\t|\f]' 16 | 17 | [sbl] 18 | regex = '^\t\t(\S+)\t(\S+)\t(www.spamhaus.org/sbl/sbl.lasso\?query=sbl\d+)' 19 | regex_values = 'address,description,alternativeid' 20 | confidence = 85 21 | 22 | [rbn] 23 | regex = '^\t\t(\S+)\trbn\t(\S+)' 24 | description = 'russian business network' 25 | regex_values = 'address,alternativeid' 26 | confidence = 85 27 | 28 | [tdl] 29 | regex = '^\t\t(\S+)\t(tdl|TDL3|TDL4|tdl3\/tdss|tld4|tld3)\t(\S+)' 30 | description = 'TDL' 31 | assessment = 'botnet' 32 | regex_values = 'address,description,alternativeid' 33 | confidence = 85 34 | severity = high 35 | 36 | [registrars] 37 | feed = http://mirror3.malwaredomains.com/files/bulk_registrars.zip 38 | regex = '([a-zA-Z-]+\.[a-z]{2,3})' 39 | regex_values = 'address' 40 | assessment = 'suspicious' 41 | description = 'free/bulk domain registration services' 42 | confidence = 60 43 | alternativeid = http://mirror1.malwaredomains.com/files/bulk_registrars.txt 44 | zip_filename = 'bulk_registrars.txt' 45 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/misc.cfg: -------------------------------------------------------------------------------- 1 | # these top-level configs are the defaults applied to each section 2 | severity = 'medium' 3 | confidence = 85 4 | restriction = 'need-to-know' 5 | alternativeid_restriction = 'public' 6 | source = 'localhost' 7 | guid = everyone 8 | 9 | [sshbl.org] 10 | feed = 'http://www.openbl.org/lists/date_all.txt' 11 | regex = '^(\S+)[\s]+(\d+)$' 12 | regex_values = 'address,detecttime' 13 | source = sshbl.org 14 | assessment = 'scanner' 15 | description = 'ssh' 16 | portlist = 22 17 | protocol = tcp 18 | alternativeid = 'http://www.openbl.org/lists/date_all.txt' 19 | period = hourly 20 | 21 | [bruteforceblocker] 22 | feed = 'http://danger.rulez.sk/projects/bruteforceblocker/blist.php' 23 | regex = '^(\S+)[\s|\t]+#\s(\S+\s\S+)' 24 | regex_values = 'address,detecttime' 25 | source = 'danger.rulez.sk' 26 | portlist = 22 27 | protocol = tcp 28 | assessment = 'scanner' 29 | description = 'ssh' 30 | alternativeid = 'http://danger.rulez.sk/projects/bruteforceblocker/blist.php' 31 | period = hourly 32 | 33 | [threatexpert] 34 | feed = 'http://www.threatexpert.com/latest_threat_reports.aspx' 35 | regex_link = "md5=([a-f0-9]{32})$" 36 | regex_link_values = 'hash' 37 | regex_title = '([\s\S]+)' 38 | regex_title_values = 'description' 39 | regex_guid = '(\S+)' 40 | regex_guid_values = 'alternativeid' 41 | regex_pubDate = '([\s\S]+)' 42 | regex_pubDate_values = 'detecttime' 43 | assessment = 'malware' 44 | source = 'threatexpert.com' 45 | severity = high 46 | confidence = 95 47 | # until we can figure out what they're doing 48 | disabled = true 49 | 50 | [malware_blacklist] 51 | feed = 'http://www.malwareblacklist.com/mbl.xml' 52 | regex_title = '^(\S+)' 53 | regex_title_values = 'address' 54 | regex_link = '(\S+)' 55 | regex_link_values = 'alternativeid' 56 | assessment = 'malware' 57 | confidence = 85 58 | source = 'malwareblacklist.com' 59 | alternativeid_restriction = public 60 | severity = medium 61 | description = unknown 62 | # they seem to have gone to an application model 63 | disabled = true 64 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/mpatrol.cfg.example: -------------------------------------------------------------------------------- 1 | # mpatriol recently changed their business model 2 | # see: https://www.malwarepatrol.net/lists.shtml 3 | 4 | # these top-level configs are the defaults applied to each section 5 | confidence = 85 6 | restriction = 'need-to-know' 7 | guid = everyone 8 | source = malware.com.br 9 | alternativeid_restriction = public 10 | assessment = malware 11 | 12 | [md5] 13 | feed = 'http://www.malware.com.br/cgi/submit?action=list_hashes' 14 | regex = '^(\S+)[\t]+([a-f0-9]{32})[\t]+[a-f0-9]{40}$' 15 | regex_values = 'description,malware_md5' 16 | confidence = 95 17 | severity = high 18 | alternativeid = 'http://www.malware.com.br/cgi/submit?action=list_hashes' 19 | 20 | [sha1] 21 | feed = 'http://www.malware.com.br/cgi/submit?action=list_hashes' 22 | regex = '^(\S+)[\t]+[a-f0-9]{32}[\t]+([a-f0-9]{40})$' 23 | regex_values = 'description,malware_sha1' 24 | confidence = 95 25 | severity = high 26 | alternativeid = 'http://www.malware.com.br/cgi/submit?action=list_hashes' 27 | 28 | [urls] 29 | feed = 'http://www.malware.com.br/cgi/submit?action=list_xml' 30 | node = 'url' 31 | elements = 'uri,id,date,av_info' 32 | elements_map = 'address,id,detecttime,description' 33 | alternativeid = 'http://www.malware.com.br/cgi/search.pl?id=' 34 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/pastebin.cfg.example: -------------------------------------------------------------------------------- 1 | confidence = 50 2 | guid = everyone 3 | severity = low 4 | source = pastebin.com 5 | restriction = need-to-know 6 | alternativeid_restriction = public 7 | 8 | [rsa_dump] 9 | feed = 'http://pastebin.com/raw.php?i=yKSQd5Z5' 10 | period = daily 11 | assessment = 'botnet' 12 | description = 'suspected apt' 13 | confidence = 75 14 | severity = high 15 | regex = '^([a-zA-Z-\.]+\.[a-z]{2,3})' 16 | regex_values = 'address' 17 | alternativeid = http://pastebin.com/raw.php?i=yKSQd5Z5 18 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/phishtank.cfg: -------------------------------------------------------------------------------- 1 | [urls] 2 | guid = everyone 3 | feed = 'http://data.phishtank.com/data/XXXXXXXXXXXXXXXXXXXXXXXXXXXX INSERT YOUR KEY HERE XXXXXXXXXXXXXXXXXXXXXX/online-valid.json.gz' 4 | feed = http://data.phishtank.com/data/online-valid.json.gz 5 | assessment = 'phishing' 6 | source = 'phishtank.com' 7 | fields = 'submission_time,url,target,phish_detail_url' 8 | fields_map = 'detecttime,address,description,alternativeid' 9 | severity = 'medium' 10 | confidence = 85 11 | restriction = 'need-to-know' 12 | alternativeid_restriction = 'public' 13 | mirror = /tmp 14 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/shadowserver.cfg: -------------------------------------------------------------------------------- 1 | # disabled by default, please visit 2 | # http://www.shadowserver.org/wiki/pmwiki.php/Services/Downloads 3 | # for access 4 | 5 | source = 'shadowserver.org' 6 | severity = 'high' 7 | restriction = 'need-to-know' 8 | confidence = 60 9 | guid = everyone 10 | 11 | [domains] 12 | feed = 'http://www.shadowserver.org/ccdns.php' 13 | regex = '^([a-zA-Z0-9.-]+[a-zA-Z0-9]{2,5})$' 14 | regex_values = 'address' 15 | assessment = 'botnet' 16 | description = 'unknown' 17 | alternativeid = 'http://www.shadowserver.org/ccdns.php' 18 | alternativeid_restriction = 'need-to-know' 19 | disabled = true 20 | 21 | [infrastructure] 22 | feed = 'http://www.shadowserver.org/ccfull.php' 23 | regex = '^(\S+):(\S+):(\S+):\S+:\S+$' 24 | regex_values = 'address,portlist,description' 25 | assessment = 'botnet' 26 | description = 'unknown' 27 | alternativeid = 'http://www.shadowserver.org/ccfull.php' 28 | alternativeid_restriction = 'need-to-know' 29 | protocol = tcp 30 | disabled = true -------------------------------------------------------------------------------- /cif-smrt/rules/etc/spamhaus.cfg: -------------------------------------------------------------------------------- 1 | source = 'spamhaus.org' 2 | confidence = 95 3 | guid = everyone 4 | alternativeid_restriction = public 5 | 6 | [spamhaus_drop] 7 | feed = 'http://www.spamhaus.org/drop/drop.lasso' 8 | regex = '^(\S+)\s;\s(\S+)$' 9 | regex_values = 'address,reference' 10 | assessment = 'suspicious' 11 | description = 'hijacked prefix' 12 | alternativeid = 'http://www.spamhaus.org/sbl/sbl.lasso?query=' 13 | 14 | [edrop] 15 | feed = 'http://www.spamhaus.org/drop/edrop.txt' 16 | regex = '^([^;][.\/0-9]+) ; (SBL[0-9]+)' 17 | regex_values = 'address,reference' 18 | assessment = 'suspicious' 19 | description = 'hijacked suballocation prefix' 20 | alternativeid = 'http://www.spamhaus.org/sbl/sbl.lasso?query=' -------------------------------------------------------------------------------- /cif-smrt/rules/etc/spyeyetracker.cfg: -------------------------------------------------------------------------------- 1 | source = 'spyeyetracker.abuse.ch' 2 | alternativeid_restriction = 'public' 3 | regex_guid = '(\S+)' 4 | regex_guid_values = 'alternativeid' 5 | restriction = 'need-to-know' 6 | description = 'spyeye' 7 | guid = everyone 8 | 9 | [domainblocklist] 10 | feed = 'https://spyeyetracker.abuse.ch/blocklist.php?download=domainblocklist' 11 | regex = '^([A-Za-z0-9.-]+\.[a-zA-Z]{2,6})$' 12 | regex_values = 'address' 13 | assessment = 'botnet' 14 | description = 'spyeye' 15 | severity = high 16 | confidence = 65 17 | alternativeid = https://spyeyetracker.abuse.ch/monitor.php?search=
18 | 19 | [ipblocklist] 20 | feed = 'https://spyeyetracker.abuse.ch/blocklist.php?download=ipblocklist' 21 | regex = '^(\S+)$' 22 | regex_values = 'address' 23 | assessment = 'botnet' 24 | description = 'spyeye' 25 | severity = high 26 | confidence = 40 27 | alternativeid = https://spyeyetracker.abuse.ch/monitor.php?search=
28 | # we know these are coming from HTTP GETS 29 | protocol = 6 30 | 31 | [binaries] 32 | feed = 'https://spyeyetracker.abuse.ch/monitor.php?rssfeed=dropurls' 33 | assessment = 'botnet' 34 | description = 'spyeye binary' 35 | severity = 'high' 36 | confidence = 85 37 | regex_link = '(\S+)' 38 | regex_link_values = 'alternativeid' 39 | regex_description = '^SpyEye DropURL: ([\s\S]*), Status: ' 40 | regex_description_values = 'address,malware_md5' 41 | regex_title = '^\S+\s\((\S+)\)$' 42 | regex_title_values = 'detecttime' 43 | 44 | [binaries_malware] 45 | feed = 'https://spyeyetracker.abuse.ch/monitor.php?rssfeed=binaryurls' 46 | assessment = 'malware' 47 | description = 'spyeye binary' 48 | severity = 'high' 49 | confidence = 85 50 | regex_description = '^SpyEye BinaryURL: [\s\S]*, Status: online, MD5 hash: (\S+),' 51 | regex_description_values = 'malware_md5' 52 | regex_link = '(\S+)' 53 | regex_link_values = 'alternativeid' 54 | regex_title = '^\S+\s\((\S+)\)$' 55 | regex_title_values = 'detecttime' 56 | 57 | 58 | [configs] 59 | feed = 'https://spyeyetracker.abuse.ch/monitor.php?rssfeed=configurls' 60 | assessment = 'botnet' 61 | description = 'spyeye config' 62 | severity = 'high' 63 | confidence = 85 64 | regex_description = '^SpyEye ConfigURL: ([\s\S]*), Status: online, MD5 hash: ([\s\S]*)' 65 | regex_description_values = 'address,malware_md5' 66 | regex_link = '(\S+)' 67 | regex_link_values = 'alternativeid' 68 | regex_title = '^\S+\s\((\S+)\)$' 69 | regex_title_values = 'detecttime' 70 | 71 | 72 | [configs_malware] 73 | feed = 'https://spyeyetracker.abuse.ch/monitor.php?rssfeed=configurls' 74 | regex_link = '(\S+)' 75 | regex_link_values = 'alternativeid' 76 | assessment = 'malware' 77 | description = 'spyeye config' 78 | severity = 'high' 79 | confidence = 85 80 | regex_description = '^SpyEye ConfigURL: [\S\s]*, Status: online, MD5 hash: ([\s\S]*),' 81 | regex_description_values = 'malware_md5' 82 | regex_title = '^\S+\s\((\S+)\)$' 83 | regex_title_values = 'detecttime' 84 | -------------------------------------------------------------------------------- /cif-smrt/rules/etc/zeustracker.cfg: -------------------------------------------------------------------------------- 1 | source = 'zeustracker.abuse.ch' 2 | alternativeid_restriction = 'public' 3 | restriction = 'need-to-know' 4 | regex_guid = '(\S+)' 5 | regex_guid_values = 'alternativeid' 6 | guid = everyone 7 | 8 | [domainblocklist] 9 | feed = 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist' 10 | regex = '^([A-Za-z0-9.-]+\.[a-zA-Z]{2,6})$' 11 | regex_values = 'address' 12 | assessment = 'botnet' 13 | description = 'zeus' 14 | severity = high 15 | confidence = 65 16 | alternativeid = https://zeustracker.abuse.ch/monitor.php?search=
17 | 18 | [ipblocklist] 19 | feed = 'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist' 20 | regex = "^(\\S+)$" 21 | regex_values = 'address' 22 | assessment = 'botnet' 23 | description = 'zeus' 24 | severity = high 25 | confidence = 65 26 | alternativeid = https://zeustracker.abuse.ch/monitor.php?search=
27 | 28 | [binaries] 29 | feed = 'https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries' 30 | assessment = 'botnet' 31 | description = 'zeus binary' 32 | severity = high 33 | confidence = 85 34 | regex_description = '^URL: ([\s\S]*), status: online, MD5 hash: ([\s\S]*)' 35 | regex_description_values = 'address,malware_md5' 36 | regex_title = '^\S+\s\((\S+)\)$' 37 | regex_title_values = 'detecttime' 38 | 39 | [binaries_malware] 40 | feed = 'https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries' 41 | assessment = 'malware' 42 | description = 'zeus binary' 43 | severity = 'high' 44 | confidence = 85 45 | regex_description = '^URL: \S+?, status: online, MD5 hash: (\S+)' 46 | regex_description_values = 'malware_md5' 47 | regex_title = '^\S+\s\((\S+)\)$' 48 | regex_title_values = 'detecttime' 49 | 50 | [configs] 51 | feed = 'https://zeustracker.abuse.ch/monitor.php?urlfeed=configs' 52 | assessment = 'botnet' 53 | description = 'zeus config' 54 | severity = 'high' 55 | confidence = 85 56 | regex_description = '^URL: (\S+)?, status: online, version: \S+?, MD5 hash: (\S+)?' 57 | regex_description_values = 'address,malware_md5' 58 | 59 | [configs_malware] 60 | feed = 'https://zeustracker.abuse.ch/monitor.php?urlfeed=configs' 61 | assessment = 'malware' 62 | description = 'zeus config' 63 | severity = 'high' 64 | confidence = 85 65 | regex_description = '^URL: \S+?, status: online, version: \S+?, MD5 hash: (\S+)' 66 | regex_description_values = 'malware_md5' 67 | regex_description = '^URL: \S+?, status: online, MD5 hash: (\S+)' 68 | regex_description_values = 'malware_md5' 69 | 70 | [dropzones] 71 | feed = 'https://zeustracker.abuse.ch/monitor.php?urlfeed=dropzones' 72 | description = 'zeus dropzone' 73 | assessment = 'botnet' 74 | confidence = 85 75 | regex_description = '^URL: ([\s\S]*), status: ' 76 | regex_description_values = 'address' 77 | regex_title = '\((\d{4}-\d{2}-\d{2})\)$' 78 | regex_title_values = detecttime 79 | -------------------------------------------------------------------------------- /cif-smrt/rules/rebuild.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make 5 | make dist 6 | -------------------------------------------------------------------------------- /cif-test-framework/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-test-framework/AUTHORS -------------------------------------------------------------------------------- /cif-test-framework/ChangeLog: -------------------------------------------------------------------------------- 1 | Revision history for libcif-dbi-cc 2 | 3 | 1.0.0-rc.5 2013-08 4 | - added threading (-t ) and looping (-l ) options 5 | 6 | 0.99.01 2013-01 7 | - initial release 8 | -------------------------------------------------------------------------------- /cif-test-framework/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = lib 2 | 3 | ME=cif-testing-framework 4 | BUILD=$(shell git describe --tags) 5 | SNAPSHOT=$(ME)-$(BUILD) 6 | 7 | all: default 8 | 9 | default: 10 | @echo "Please read the README before beginning your installation." 11 | 12 | fixperms: 13 | chmod 0755 $(prefix) 14 | chown -R $(CIF_USER) $(prefix) 15 | chgrp -R $(CIF_GROUP) $(prefix) 16 | 17 | install: files-install fixperms 18 | 19 | files-install: libs-install 20 | 21 | libs-install: 22 | [ -d $(libdir) ] || $(INSTALL) -m 0755 -d $(libdir) 23 | -( cd lib && find . -type d -print ) | while read dir ; do \ 24 | $(INSTALL) -m 0755 -d "$(libdir)/$$dir" ; \ 25 | done 26 | -( cd lib && find . -type f -print ) | while read file ; do \ 27 | $(INSTALL) -m 0644 "lib/$$file" "$(libdir)/$$file" ; \ 28 | done 29 | 30 | clean: 31 | rm Makefile 32 | rm config.log 33 | rm config.status 34 | 35 | realclean: 36 | clean 37 | rm Makefile.in 38 | rm configure 39 | rm aclocal.m4 40 | 41 | reconf: 42 | autoreconf -vf 43 | 44 | snapshot: 45 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 46 | -------------------------------------------------------------------------------- /cif-test-framework/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/cif-test-framework/NEWS -------------------------------------------------------------------------------- /cif-test-framework/README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | This is the regression testing framework for CIF. Used to test the basics of an installation. 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | Resources 12 | --------- 13 | 14 | Website: collectiveintel.org 15 | Repository: github.com/collectiveintel/cif-test-framework 16 | 17 | 18 | COPYRIGHT AND LICENCE 19 | --------------------- 20 | Copyright (C) 2013 Wes Young (wesyoung.me) 21 | Copyright (C) 2013 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 22 | 23 | Free use of this software is granted under the terms of the GNU Lesser General 24 | Public License (LGPL) unless otherwise specifically stated. For details see the 25 | files `COPYING` included with the distribution. -------------------------------------------------------------------------------- /cif-test-framework/autogen.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make 5 | -------------------------------------------------------------------------------- /cif-test-framework/bin/cif_test_regression: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | # fix lib paths, some may be relative 7 | BEGIN { 8 | require File::Spec; 9 | my @libs = ( 10 | "./lib", 11 | "../libcif/lib", # dev mode 12 | "../libcif-dbi/lib", # dev mode 13 | ); 14 | my $bin_path; 15 | 16 | for my $lib (@libs) { 17 | unless ( File::Spec->file_name_is_absolute($lib) ) { 18 | unless ($bin_path) { 19 | if ( File::Spec->file_name_is_absolute(__FILE__) ) { 20 | $bin_path = ( File::Spec->splitpath(__FILE__) )[1]; 21 | } 22 | else { 23 | require FindBin; 24 | no warnings "once"; 25 | $bin_path = $FindBin::Bin; 26 | } 27 | } 28 | $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib ); 29 | } 30 | unshift @INC, $lib; 31 | } 32 | } 33 | 34 | use CIF::TESTFRAMEWORK; 35 | use Getopt::Std; 36 | use Data::Dumper; 37 | use CIF qw/debug generate_uuid_random generate_uuid_ns/; 38 | 39 | my %opts; 40 | 41 | getopts('hdv:C:',\%opts); 42 | our $debug = $opts{'d'} || 1; 43 | $debug = $opts{'v'} if($opts{'v'}); 44 | 45 | my $config = $opts{'C'} || $ENV{'HOME'}.'/.cif'; 46 | 47 | # basic tests 48 | 49 | our $tests = [ 50 | { 51 | address => '1.2.3.4', 52 | assessment => 'scanner', 53 | guid => generate_uuid_ns('everyone'), 54 | confidence => 75, 55 | id => generate_uuid_random(), 56 | }, 57 | { 58 | address => 'example.com', 59 | assessment => 'botnet', 60 | guid => generate_uuid_ns('group1'), 61 | confidence => 35, 62 | id => generate_uuid_random(), 63 | }, 64 | ]; 65 | 66 | # create new test framework object 67 | 68 | my ($err,$ret) = CIF::TESTFRAMEWORK->new({ 69 | config => $config, 70 | tests => $tests, 71 | }); 72 | 73 | die($err) if($err); 74 | 75 | my $obj = $ret; 76 | 77 | # generate temp apikey 78 | 79 | debug('generating apikey...'); 80 | my $apikey = $obj->init_apikey(); 81 | 82 | # set the apikey to the client 83 | $obj->get_client->set_apikey($apikey); 84 | 85 | # run tests 86 | 87 | ($err,$ret) = $obj->process({ 88 | client => $obj->get_client(), 89 | }); 90 | 91 | debug('all tests successful...') if($ret); 92 | 93 | # cleanup 94 | 95 | debug('removing key: '.$apikey); 96 | $obj->purge_apikey($apikey); 97 | -------------------------------------------------------------------------------- /cif-test-framework/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.64]) 5 | AC_INIT(cif-test-framework, m4_esyscmd_s([git describe --tags]), [ci-framework@googlegroups.com]) 6 | AM_INIT_AUTOMAKE 7 | 8 | # Checks for programs. 9 | AC_ARG_VAR([PERL],[Perl interpreter command]) 10 | AC_PATH_PROG([PERL], [perl], [not found]) 11 | if test "$PERL" = 'not found'; then 12 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without perl]) 13 | fi 14 | 15 | AC_ARG_VAR([PSQL],[Postgres psql command]) 16 | AC_PATH_PROG([PSQL],[psql],[not found]) 17 | if test "$PSQL" = 'not found'; then 18 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without postgres psql]) 19 | fi 20 | 21 | dnl Defaults paths for installation 22 | AC_PREFIX_DEFAULT([/opt/cif]) 23 | 24 | AC_CONFIG_FILES([ 25 | Makefile 26 | ]) 27 | 28 | 29 | # Checks for libraries. 30 | 31 | # Checks for header files. 32 | 33 | # Checks for typedefs, structures, and compiler characteristics. 34 | 35 | # Checks for library functions. 36 | 37 | #dnl CIF_USER 38 | AC_ARG_WITH(user, 39 | AC_HELP_STRING([--with-user=USER], 40 | [user the cif scripts will run as (default: cif)]), 41 | CIF_USER=$withval, 42 | CIF_USER=cif) 43 | AC_SUBST(CIF_USER) 44 | 45 | #dnl CIF_GROUP 46 | AC_ARG_WITH(group, 47 | AC_HELP_STRING([--with-group=GROUP], 48 | [group the cif scripts will run as (default: cif)]), 49 | CIF_GROUP=$withval, 50 | CIF_GROUP=$CIF_USER 51 | ) 52 | AC_SUBST(CIF_GROUP) 53 | 54 | AC_OUTPUT 55 | -------------------------------------------------------------------------------- /cif-test-framework/lib/CIF/TESTFRAMEWORK/Plugin.pm: -------------------------------------------------------------------------------- 1 | package CIF::TESTFRAMEWORK::Plugin; 2 | use base 'CIF::TESTFRAMEWORK'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | sub run { } 8 | 9 | 1; -------------------------------------------------------------------------------- /cif-test-framework/lib/CIF/TESTFRAMEWORK/Plugin/00_Submit.pm: -------------------------------------------------------------------------------- 1 | package CIF::TESTFRAMEWORK::Plugin::00_Submit; 2 | use base 'CIF::TESTFRAMEWORK::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple; 8 | use CIF qw/debug generate_uuid_ns/; 9 | use Try::Tiny; 10 | 11 | sub run { 12 | my $self = shift; 13 | my $args = shift; 14 | 15 | my @tests = @{$args->{'tests'}}; 16 | my $cli = $args->{'client'}; 17 | 18 | my $err; 19 | foreach my $t (@tests){ 20 | try { 21 | $t = Iodef::Pb::Simple->new({%$t}); 22 | } catch { 23 | $err = shift; 24 | }; 25 | if($err){ 26 | return('test failed: '.$err); 27 | } 28 | $t = $t->encode(); 29 | } 30 | 31 | my $ret = $cli->new_submission({ 32 | guid => generate_uuid_ns('everyone'), 33 | data => \@tests, 34 | }); 35 | 36 | ($err,$ret) = $cli->submit($ret); 37 | if($err){ 38 | debug($err); 39 | return($err); 40 | } 41 | 42 | foreach (@{$ret->get_data()}){ 43 | debug('submission sucessful: '.$_); 44 | } 45 | 46 | return 1; 47 | } 48 | 49 | 50 | 51 | 1; -------------------------------------------------------------------------------- /cif-test-framework/lib/CIF/TESTFRAMEWORK/Plugin/01_Submit_SimpleHTTP.pm: -------------------------------------------------------------------------------- 1 | package CIF::TESTFRAMEWORK::Plugin::01_Submit_SimpleHTTP; 2 | use base 'CIF::TESTFRAMEWORK::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use LWP::UserAgent; 8 | use JSON::XS; 9 | 10 | use CIF qw/debug/; 11 | 12 | sub run { 13 | my $self = shift; 14 | my $args = shift; 15 | 16 | my @tests = @{$args->{'tests'}}; 17 | my $cli = $args->{'client'}; 18 | 19 | my $loop = $args->{'loop'} || 0; 20 | 21 | my $ua = LWP::UserAgent->new(); 22 | $ua->ssl_opts(SSL_verify_mode => 'SSL_VERIFY_NONE'); 23 | $ua->ssl_opts(verify_hostname => 0); 24 | $ua->default_header('Accept' => 'application/json'); 25 | 26 | my $url = $cli->get_config->{'host'}; 27 | $url .= '?apikey='.$cli->get_config->{'apikey'}; 28 | 29 | my ($ret,$err); 30 | for (my $i = 0; $i < $loop; $i++){ 31 | foreach my $t (@tests){ 32 | debug('query: '.$t->{'address'}); 33 | $ret = $ua->post($url,Content => encode_json($t)); 34 | 35 | return ($ret->content()) unless($ret->status_line() eq '200'); 36 | return('query failed: '.$t->{'address'}) unless($ret); 37 | } 38 | } 39 | debug('tests successful...'); 40 | 41 | return 1; 42 | } 43 | 44 | sub basic {} 45 | 46 | sub feed {} 47 | 48 | 1; -------------------------------------------------------------------------------- /cif-test-framework/lib/CIF/TESTFRAMEWORK/Plugin/10_Query.pm: -------------------------------------------------------------------------------- 1 | package CIF::TESTFRAMEWORK::Plugin::10_Query; 2 | use base 'CIF::TESTFRAMEWORK::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use CIF qw/debug/; 8 | 9 | sub run { 10 | my $self = shift; 11 | my $args = shift; 12 | 13 | my @tests = @{$args->{'tests'}}; 14 | my $cli = $args->{'client'}; 15 | 16 | my $loop = $args->{'loop'} || 0; 17 | 18 | my ($ret,$err); 19 | for (my $i = 0; $i < $loop; $i++){ 20 | foreach my $t (@tests){ 21 | debug('query: '.$t->{'address'}); 22 | ($err,$ret) = $cli->search({ 23 | query => $t->{'address'}, 24 | nolog => 1, 25 | }); 26 | return($err) if($err); 27 | return('query failed: '.$t->{'address'}) unless($ret); 28 | } 29 | } 30 | debug('query tests successful...'); 31 | return 1; 32 | } 33 | 34 | sub basic {} 35 | 36 | sub feed {} 37 | 38 | 1; -------------------------------------------------------------------------------- /cif-test-framework/lib/CIF/TESTFRAMEWORK/Plugin/11_Query_SimpleHTTP.pm: -------------------------------------------------------------------------------- 1 | package CIF::TESTFRAMEWORK::Plugin::11_Query_SimpleHTTP; 2 | use base 'CIF::TESTFRAMEWORK::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use LWP::UserAgent; 8 | use JSON::XS; 9 | 10 | use CIF qw/debug/; 11 | 12 | sub run { 13 | my $self = shift; 14 | my $args = shift; 15 | 16 | my @tests = @{$args->{'tests'}}; 17 | my $cli = $args->{'client'}; 18 | 19 | my $loop = $args->{'loop'} || 0; 20 | 21 | my $ua = LWP::UserAgent->new(); 22 | $ua->ssl_opts(SSL_verify_mode => 'SSL_VERIFY_NONE'); 23 | $ua->ssl_opts(verify_hostname => 0); 24 | $ua->default_header('Accept' => 'application/json'); 25 | 26 | my $url = $cli->get_config->{'host'}; 27 | $url .= '?apikey='.$cli->get_config->{'apikey'}; 28 | 29 | my ($ret,$err); 30 | for (my $i = 0; $i < $loop; $i++){ 31 | foreach my $t (@tests){ 32 | debug('query: '.$t->{'address'}); 33 | $ret = $ua->get($url."&query=".$t->{'address'}); 34 | 35 | return ($ret->content()) unless($ret->status_line() eq '200'); 36 | return('query failed: '.$t->{'address'}) unless($ret); 37 | } 38 | } 39 | debug('query tests successful...'); 40 | 41 | return 1; 42 | } 43 | 44 | sub basic {} 45 | 46 | sub feed {} 47 | 48 | 1; -------------------------------------------------------------------------------- /experimental/install_rhel6.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | yum install sudo rng-tools postgresql-server httpd httpd-devel mod_ssl gcc make expat expat-devel mod_perl mod_perl-devel perl-Digest-SHA perl-Digest-SHA libxml2 libxml2-devel perl-XML-LibXML uuid-perl perl-DBD-Pg bind perl-XML-RSS perl-JSON rsync perl-Unicode-String perl-Config-Simple perl-Module-Pluggable perl-MIME-Lite perl-CPAN perl-Class-Accessor perl-YAML perl-XML-Parser uuid uuid-devel uuid-perl perl-Net-DNS perl-DateTime-Format-DateParse perl-IO-Socket-INET6 openssl-devel perl-Module-Install wget perl-Net-SSLeay perl-Class-Trigger perl-Date-Manip libuuid-devel 4 | 5 | wget http://dl.fedoraproject.org/pub/epel/6/x86_64/libapreq2-2.13-1.el6.x86_64.rpm 6 | wget http://dl.fedoraproject.org/pub/epel/6/x86_64/libapreq2-devel-2.13-1.el6.x86_64.rpm 7 | wget http://dl.fedoraproject.org/pub/epel/6/x86_64/perl-libapreq2-2.13-1.el6.x86_64.rpm 8 | wget http://dl.fedoraproject.org/pub/epel/6/x86_64/zeromq-2.1.9-1.el6.x86_64.rpm 9 | wget http://dl.fedoraproject.org/pub/epel/6/x86_64/zeromq-devel-2.1.9-1.el6.x86_64.rpm 10 | sudo rpm -i libapreq2-2.13-1.el6.x86_64.rpm libapreq2-devel-2.13-1.el6.x86_64.rpm perl-libapreq2-2.13-1.el6.x86_64.rpm zeromq-2.1.9-1.el6.x86_64.rpm zeromq-devel-2.1.9-1.el6.x86_64.rpm 11 | 12 | sudo PERL_MM_USE_DEFAULT=1 perl -MCPAN -e 'install Net::Abuse::Utils,Linux::Cpuinfo,Google::ProtocolBuffers,Iodef::Pb::Simple,Compress::Snappy,Net::Abuse::Utils::Spamhaus,Net::DNS::Match,Snort::Rule,Parse::Range,Log::Dispatch,ZeroMQ,Sys::MemInfo, JSON::XS,File::Type,LWP::UserAgent,Class::Trigger,LWP::Protocol::https,Class::DBI,Net::Patricia,Text::Table,Regexp::Common::net, Regexp::Common::net::CIDR' 13 | 14 | wget http://search.cpan.org/CPAN/authors/id/M/MI/MIKEM/Net-SSLeay-1.49.tar.gz 15 | tar -zxvf Net-SSLeay-1.49.tar.gz 16 | cd Net-SSLeay-1.49 17 | PERL_MM_USE_DEFAULT=1 perl Makefile.PL 18 | sudo make install 19 | cd ../ 20 | 21 | wget http://search.cpan.org/CPAN/authors/id/S/SH/SHLOMIF/IO-Socket-INET6-2.69.tar.gz 22 | tar -zxvf IO-Socket-INET6-2.69.tar.gz 23 | cd IO-Socket-INET6-2.69 24 | PERL_MM_USE_DEFAULT=1 perl Makefile.PL && sudo make install 25 | cd ../ 26 | 27 | echo "DNS1=127.0.01" >> /etc/sysconfig/network-scripts/ifcfg-eth0 28 | sudo service network restart 29 | 30 | sudo service postgresql initdb 31 | 32 | sudo mkdir -p /etc/postgresql/8.4/main 33 | sudo chown -R postgres:postgres /etc/postgresql 34 | sudo chmod 760 -R /etc/postgresql 35 | sudo ln -sf /var/lib/pgsql/data/postgresql.conf /etc/postgresql/8.4/main/postgresql.conf 36 | sudo ln -sf /var/lib/pgsql/data/pg_hba.conf /etc/postgresql/8.4/main/pg_hba.conf 37 | sudo service postgresql start 38 | 39 | sudo adduser cif 40 | sudo chmod 770 /home/cif 41 | 42 | # apache stuff, add the rest 43 | 44 | sudo usermod -a -G cif apache 45 | 46 | # rng options here 47 | 48 | # get things into startup 49 | 50 | sudo chkconfig --levels 345 postgresql on 51 | sudo chkconfig --levels 345 named on 52 | sudo chkconfig --levels 345 rngd on 53 | sudo chkconfig --levels 345 httpd on -------------------------------------------------------------------------------- /experimental/legacy_example.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use warnings; 4 | use strict; 5 | 6 | use LWP::UserAgent; 7 | use JSON::XS; 8 | use Data::Dumper; 9 | 10 | my $url = 'https://localhost?apikey=XXXX'; 11 | 12 | my $ua = LWP::UserAgent->new(); 13 | $ua->ssl_opts(verify_hostname => 0); 14 | $ua->default_header('Accept' => 'application/json'); 15 | 16 | my $hash = { 17 | address => 'example.com', 18 | assessment => 'botnet', 19 | confidence => 86, 20 | description => 'zeus', 21 | }; 22 | 23 | my $ret = $ua->post($url,Content => encode_json($hash)); 24 | warn Dumper($ret); 25 | -------------------------------------------------------------------------------- /libcif-dbi/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/libcif-dbi/AUTHORS -------------------------------------------------------------------------------- /libcif-dbi/ChangeLog: -------------------------------------------------------------------------------- 1 | see parent ChangeLog 2 | -------------------------------------------------------------------------------- /libcif-dbi/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/libcif-dbi/NEWS -------------------------------------------------------------------------------- /libcif-dbi/README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | This is the database interface for cif. 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | Resources 12 | --------- 13 | 14 | Website: collectiveintel.org 15 | Repository: github.com/collectiveintel/libcif-dbi 16 | 17 | 18 | COPYRIGHT AND LICENCE 19 | --------------------- 20 | Copyright (C) 2013 Wes Young (wesyoung.me) 21 | Copyright (C) 2013 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 22 | 23 | Free use of this software is granted under the terms of the GNU Lesser General 24 | Public License (LGPL) unless otherwise specifically stated. For details see the 25 | files `COPYING` included with the distribution. -------------------------------------------------------------------------------- /libcif-dbi/bin/cif_skel: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | # fix lib paths, some may be relative 7 | BEGIN { 8 | require File::Spec; 9 | my @libs = ( 10 | "lib", 11 | "local/lib", 12 | "../libcif/lib", # in case we're in -dev mode 13 | ); 14 | my $bin_path; 15 | 16 | for my $lib (@libs) { 17 | unless ( File::Spec->file_name_is_absolute($lib) ) { 18 | unless ($bin_path) { 19 | if ( File::Spec->file_name_is_absolute(__FILE__) ) { 20 | $bin_path = ( File::Spec->splitpath(__FILE__) )[1]; 21 | } 22 | else { 23 | require FindBin; 24 | no warnings "once"; 25 | $bin_path = $FindBin::Bin; 26 | } 27 | } 28 | $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib ); 29 | } 30 | unshift @INC, $lib; 31 | } 32 | } 33 | 34 | 35 | use Getopt::Std; 36 | 37 | my %opts; 38 | getopts('dh',\%opts); 39 | 40 | our $debug = $opts{'d'}; 41 | 42 | if($opts{'h'}){ 43 | print usage(); 44 | exit(0); 45 | } 46 | sub usage { 47 | return < ['CIF::Legacy'], require => 1; 4 | require JSON::XS; 5 | 6 | my @plugins = __PACKAGE__->plugins(); 7 | 8 | sub hash_simple { 9 | my $data = shift; 10 | my $uuid = shift; 11 | 12 | $data = JSON::XS::decode_json($data); 13 | 14 | $data = $data->{'Incident'}; 15 | 16 | my $impact = $data->{'Assessment'}->{'Impact'}; 17 | $impact = $data->{'Assessment'}->{'Impact'}->{'content'} if(ref($impact) eq 'HASH'); 18 | my $h = { 19 | id => $uuid, 20 | relatedid => $data->{'RelatedActivity'}->{'IncidentID'}, 21 | description => $data->{'Description'}, 22 | assessment => $impact, 23 | severity => $data->{'Assessment'}->{'Impact'}->{'severity'}, 24 | confidence => $data->{'Assessment'}->{'Confidence'}->{'content'}, 25 | source => $data->{'IncidentID'}->{'name'}, 26 | restriction => $data->{'restriction'}, 27 | alternativeid => $data->{'AlternativeID'}->{'IncidentID'}->{'content'}, 28 | alternativeid_restriction => $data->{'AlternativeID'}->{'IncidentID'}->{'restriction'}, 29 | detecttime => $data->{'DetectTime'}, 30 | reporttime => $data->{'ReportTime'} || $data->{'DetectTime'}, 31 | purpose => $data->{'purpose'}, 32 | }; 33 | 34 | foreach my $p (@plugins){ 35 | my $ret = $p->hash_simple($data); 36 | next unless($ret); 37 | map { $h->{$_} = $ret->{$_} } keys (%$ret); 38 | } 39 | return ($h); 40 | } 41 | 42 | sub _throttle { 43 | my $throttle = shift; 44 | 45 | require Linux::Cpuinfo; 46 | my $cpu = Linux::Cpuinfo->new(); 47 | return(DEFAULT_THROTTLE_FACTOR()) unless($cpu); 48 | 49 | my $cores = $cpu->num_cpus(); 50 | return(DEFAULT_THROTTLE_FACTOR()) unless($cores && $cores =~ /^\d+$/); 51 | return(DEFAULT_THROTTLE_FACTOR()) if($cores eq 1); 52 | 53 | return($cores * (DEFAULT_THROTTLE_FACTOR() * 2)) if($throttle eq 'high'); 54 | return($cores * DEFAULT_THROTTLE_FACTOR()) if($throttle eq 'medium'); 55 | return($cores / 2) if($throttle eq 'low'); 56 | } -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Bgp.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Bgp; 2 | 3 | sub hash_simple { 4 | my $class = shift; 5 | my $hash = shift; 6 | my $sh = shift; 7 | 8 | my ($asn,$prefix,$rir,$cc); 9 | 10 | my $ad = $hash->{'EventData'}->{'Flow'}->{'System'}->{'AdditionalData'}; 11 | my @array; 12 | if(ref($ad) eq 'ARRAY'){ 13 | @array = @$ad; 14 | } else { 15 | push(@array,$ad); 16 | } 17 | foreach my $a (@array){ 18 | next unless($a->{'meaning'}); 19 | for(lc($a->{'meaning'})){ 20 | $asn = $a->{'content'} if(/asn/); 21 | $prefix = $a->{'content'} if(/prefix/); 22 | $rir = $a->{'content'} if(/rir/); 23 | $cc = $a->{'content'} if(/cc/); 24 | } 25 | } 26 | return unless($asn || $prefix || $rir || $cc); 27 | 28 | if($asn){ 29 | $asn =~ /^(\d+) ([\s|\S]+)$/; 30 | $sh->{'asn'} = $1; 31 | $sh->{'asn_desc'} = $2; 32 | } 33 | $sh->{'prefix'} = $prefix; 34 | $sh->{'rir'} = $rir; 35 | $sh->{'cc'} = $cc; 36 | return($sh); 37 | } 38 | 39 | 1; 40 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Domain.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Domain; 2 | 3 | sub hash_simple { 4 | my $class = shift; 5 | my $hash = shift; 6 | 7 | my $address = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Node'}->{'Address'}; 8 | return unless($address); 9 | for(ref($address)){ 10 | if(/HASH/){ 11 | $address = $address->{'content'}; 12 | last; 13 | } 14 | if(/ARRAY/){ 15 | my @ary = @{$address}; 16 | $address = $ary[$#ary]->{'content'}; 17 | last; 18 | } 19 | } 20 | return unless($address =~ /^[a-zA-Z0-9.-]+\.[a-z]{2,5}$/); 21 | 22 | my ($rdata,$type); 23 | my $ad = $hash->{'EventData'}->{'Flow'}->{'System'}->{'AdditionalData'}; 24 | my @array; 25 | if(ref($ad) eq 'ARRAY'){ 26 | @array = @$ad; 27 | } else { 28 | push(@array,$ad); 29 | } 30 | foreach my $a (@array){ 31 | for(lc($a->{'meaning'})){ 32 | $rdata = $a->{'content'} if(/rdata/); 33 | $type = $a->{'content'} if(/type/); 34 | } 35 | } 36 | 37 | my $portlist = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Service'}->{'Portlist'}; 38 | my $protocol = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Service'}->{'ip_protocol'}; 39 | 40 | return({ 41 | rdata => $rdata, 42 | type => $type, 43 | address => $address, 44 | portlist => $portlist, 45 | protocol => $protocol, 46 | }); 47 | } 48 | 1; 49 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Email.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Email; 2 | 3 | use Regexp::Common qw/URI/; 4 | 5 | sub isEmail { 6 | my $e = shift; 7 | return unless($e); 8 | return if($e =~ /^$RE{'URI'}/ || $e =~ /^$RE{'URI'}{'HTTP'}{-scheme => 'https'}$/); 9 | return unless(lc($e) =~ /^[a-z0-9_.-]+\@[a-z0-9.-]+\.[a-z0-9.-]{2,5}$/); 10 | return(1); 11 | } 12 | 13 | sub hash_simple { 14 | my $class = shift; 15 | my $hash = shift; 16 | 17 | my $address = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Node'}->{'Address'}; 18 | return unless($address); 19 | for(ref($address)){ 20 | if(/HASH/){ 21 | $address = $address->{'content'}; 22 | last; 23 | } 24 | if(/ARRAY/){ 25 | my @ary = @{$address}; 26 | $address = $ary[$#ary]->{'content'}; 27 | last; 28 | } 29 | } 30 | return unless(isEmail($address)); 31 | 32 | return({ 33 | address => $address, 34 | }); 35 | } 36 | 1; 37 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Group.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Group; 2 | 3 | sub hash_simple { 4 | my $class = shift; 5 | my $hash = shift; 6 | 7 | my ($rdata,$type); 8 | my $ad = $hash->{'AdditionalData'}; 9 | return unless($ad); 10 | my @array; 11 | if(ref($ad) eq 'ARRAY'){ 12 | @array = @$ad; 13 | } else { 14 | push(@array,$ad); 15 | } 16 | foreach my $a (@array){ 17 | for(lc($a->{'meaning'})){ 18 | next unless(/guid/); 19 | return({ 20 | guid => $a->{'content'} 21 | }); 22 | } 23 | } 24 | } 25 | 1; 26 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Ipv4.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Ipv4; 2 | 3 | use Regexp::Common qw/net/; 4 | 5 | sub hash_simple { 6 | my $clas = shift; 7 | my $hash = shift; 8 | 9 | my $address = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Node'}->{'Address'}; 10 | for(ref($address)){ 11 | if(/HASH/){ 12 | $address = $address->{'content'}; 13 | last; 14 | } 15 | if(/ARRAY/){ 16 | my @ary = @{$address}; 17 | $address = $ary[$#ary]->{'content'}; 18 | last; 19 | } 20 | } 21 | return unless($address && $address =~ /^$RE{'net'}{'IPv4'}/); 22 | 23 | return({ 24 | address => $address, 25 | }); 26 | } 27 | 1; 28 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Malware; 2 | 3 | use warnings; 4 | use strict; 5 | 6 | sub hash_simple { 7 | my $class = shift; 8 | my $hash = shift; 9 | 10 | my ($sha1,$md5); 11 | my $impact = $hash->{'Assessment'}->{'Impact'}; 12 | if(ref($impact) eq 'HASH'){ 13 | $impact = $impact->{'content'}; 14 | } 15 | my $ad = $hash->{'EventData'}->{'AdditionalData'}; 16 | return unless($ad); 17 | my @array; 18 | if(ref($ad) eq 'ARRAY'){ 19 | @array = @$ad; 20 | } else { 21 | push(@array,$ad); 22 | } 23 | foreach my $a (@array){ 24 | for(lc($a->{'meaning'})){ 25 | $md5 = $a->{'content'} if(/md5/); 26 | $sha1 = $a->{'content'} if(/sha1/); 27 | } 28 | } 29 | if($impact eq 'malware' || $impact eq 'search'){ 30 | return({ 31 | md5 => $md5, 32 | sha1 => $sha1 33 | }); 34 | } else { 35 | return({ 36 | malware_sha1 => $sha1, 37 | malware_md5 => $md5, 38 | }); 39 | } 40 | } 41 | 1; 42 | 43 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Service.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Service; 2 | 3 | sub hash_simple { 4 | my $clas = shift; 5 | my $hash = shift; 6 | 7 | my $portlist = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Service'}->{'Portlist'}; 8 | my $protocol = $hash->{'EventData'}->{'Flow'}->{'System'}->{'Service'}->{'ip_protocol'}; 9 | 10 | return unless($portlist || $protocol); 11 | 12 | $portlist =~ s/\s+//g if($portlist); 13 | 14 | return({ 15 | portlist => $portlist, 16 | protocol => $protocol, 17 | }); 18 | } 19 | 1; 20 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/ShareWith.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::ShareWith; 2 | 3 | sub hash_simple { 4 | my $class = shift; 5 | my $hash = shift; 6 | 7 | my ($rdata,$type); 8 | my $ad = $hash->{'AdditionalData'}; 9 | return unless($ad); 10 | my @array; 11 | if(ref($ad) eq 'ARRAY'){ 12 | @array = @$ad; 13 | } else { 14 | push(@array,$ad); 15 | } 16 | my @share; 17 | foreach my $a (@array){ 18 | for(lc($a->{'meaning'})){ 19 | next unless(/sharewith/); 20 | push(@share,$a->{'content'}); 21 | } 22 | } 23 | return unless($#share > -1); 24 | return({ 25 | carboncopy => join(',',@share), 26 | }); 27 | } 28 | 1; 29 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/lib/CIF/Legacy/Url.pm: -------------------------------------------------------------------------------- 1 | package CIF::Legacy::Url; 2 | 3 | use Regexp::Common qw/URI/; 4 | 5 | sub hash_simple { 6 | my $class = shift; 7 | my $data = shift; 8 | 9 | my $address = $data->{'EventData'}->{'Flow'}->{'System'}->{'Node'}->{'Address'}; 10 | return unless($address); 11 | for(ref($address)){ 12 | if(/HASH/){ 13 | $address = $address->{'content'}; 14 | last; 15 | } 16 | if(/ARRAY/){ 17 | my @ary = @{$address}; 18 | $address = $ary[$#ary]->{'content'}; 19 | last; 20 | } 21 | } 22 | 23 | return unless($address =~ /^(ftp|https?):\/\//); 24 | 25 | my $ad = $data->{'EventData'}->{'Flow'}->{'System'}->{'AdditionalData'}; 26 | return unless($ad); 27 | my @array; 28 | if(ref($ad) eq 'ARRAY'){ 29 | @array = @$ad; 30 | } else { 31 | push(@array,$ad); 32 | } 33 | my ($md5,$sha1); 34 | foreach my $a (@array){ 35 | for(lc($a->{'meaning'})){ 36 | $md5 = $a->{'content'} if(/^md5/); 37 | $sha1 = $a->{'content'} if(/^sha1/); 38 | } 39 | } 40 | 41 | return({ 42 | address => $address, 43 | md5 => $md5, 44 | sha1 => $sha1, 45 | }); 46 | } 47 | 1; 48 | -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/schema/00_remove.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS analytic CASCADE; 2 | DROP TABLE IF EXISTS domain CASCADE; 3 | DROP TABLE IF EXISTS infrastructure CASCADE; 4 | DROP TABLE IF EXISTS url CASCADE; 5 | DROP TABLE IF EXISTS email CASCADE; 6 | DROP TABLE IF EXISTS malware CASCADE; 7 | DROP TABLE IF EXISTS feed CASCADE; 8 | DROP TABLE IF EXISTS hash CASCADE; 9 | DROP TABLE IF EXISTS rir CASCADE; 10 | DROP TABLE IF EXISTS asn CASCADE; 11 | DROP TABLE IF EXISTS countrycode CASCADE; 12 | DROP TABLE IF EXISTS search CASCADE; -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/schema/01_alter_archive.sql: -------------------------------------------------------------------------------- 1 | set default_tablespace = archive; 2 | 3 | DROP TYPE IF EXISTS severity; 4 | DROP TYPE IF EXISTS restriction; 5 | 6 | ALTER TABLE archive DROP COLUMN source; 7 | ALTER TABLE archive DROP COLUMN restriction; 8 | ALTER TABLE archive DROP COLUMN description; 9 | ALTER TABLE archive RENAME COLUMN created TO reporttime; 10 | ALTER TABLE archive ADD COLUMN created timestamp with time zone default now(); 11 | 12 | set default_tablespace = index; 13 | 14 | ALTER INDEX archive_uuid_key RENAME TO idx_archive_uuid; -------------------------------------------------------------------------------- /libcif-dbi/etc/upgrade/schema/02_alter_apikeys.sql: -------------------------------------------------------------------------------- 1 | set default_tablespace = archive; 2 | 3 | ALTER TABLE apikeys DROP COLUMN access; 4 | ALTER TABLE apikeys ADD COLUMN restricted_access boolean default false; 5 | ALTER TABLE apikeys ADD COLUMN expires timestamp with time zone; 6 | 7 | CREATE TABLE apikeys_restrictions ( 8 | id BIGSERIAL PRIMARY KEY NOT NULL, 9 | uuid uuid references apikeys(uuid) on delete cascade not null, 10 | access varchar(40) not null, 11 | created timestamp with time zone DEFAULT NOW(), 12 | UNIQUE(uuid,access) 13 | ); -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/APIKey.pm: -------------------------------------------------------------------------------- 1 | package CIF::APIKey; 2 | use base 'CIF::DBI'; 3 | 4 | __PACKAGE__->table('apikeys'); 5 | __PACKAGE__->columns(Primary => 'uuid'); 6 | __PACKAGE__->columns(All => qw/uuid uuid_alias description parentid revoked write restricted_access expires created/); 7 | __PACKAGE__->sequence('apikeys_id_seq'); 8 | __PACKAGE__->has_many(groups => 'CIF::APIKeyGroups'); 9 | __PACKAGE__->has_many(restrictions => 'CIF::APIKeyRestrictions'); 10 | 11 | use CIF qw/is_uuid generate_uuid_random generate_uuid_url generate_uuid_ns/; 12 | 13 | # because UUID's are really primary keys too in our schema 14 | # this overrides some of the default functionality of Class::DBI and 'id' 15 | sub retrieve { 16 | my $class = shift; 17 | my %keys = @_; 18 | 19 | return $class->SUPER::retrieve(@_) unless($keys{'uuid'}); 20 | 21 | my @recs = $class->search(uuid => $keys{'uuid'}); 22 | return unless(@recs); 23 | return($recs[0]); 24 | } 25 | 26 | sub add_groups { 27 | my ($self,$default_guid,$groups) = @_; 28 | if($default_guid){ 29 | $default_guid = generate_uuid_url($default_guid) unless(is_uuid($default_guid)); 30 | } 31 | 32 | foreach (split(',',$groups)){ 33 | $_ = generate_uuid_url($_) unless(is_uuid($_)); 34 | my $isDefault = 1 if($default_guid && ($_ eq $default_guid)); 35 | my $id = eval { 36 | CIF::APIKeyGroups->insert({ 37 | uuid => $self->uuid(), 38 | guid => $_, 39 | default_guid => $isDefault, 40 | }); 41 | }; 42 | if($@){ 43 | die($@) unless($@ =~ /unique constraint/); 44 | } 45 | } 46 | } 47 | 48 | sub default_guid { 49 | my $self = shift; 50 | my @groups = $self->groups(); 51 | foreach (@groups){ 52 | return($_->guid()) if($_->default_guid()); 53 | } 54 | # this shouldn't happen... in theory. 55 | return(0); 56 | } 57 | 58 | sub inGroup { 59 | return in_group(@_); 60 | } 61 | 62 | sub in_group { 63 | my $self = shift; 64 | my $grp = shift; 65 | 66 | return unless($grp); 67 | $grp = lc($grp); 68 | $grp = generate_uuid_ns($grp) unless(is_uuid($grp)); 69 | 70 | my @groups = $self->groups(); 71 | foreach (@groups){ 72 | return(1) if($grp eq $_->guid()); 73 | } 74 | return(0); 75 | } 76 | 77 | sub mygroups { 78 | return groups(\@_); 79 | } 80 | 81 | sub my_groups { 82 | my $self = shift; 83 | 84 | my @groups = $self->groups(); 85 | return unless($#groups > -1); 86 | my $g = ''; 87 | foreach (@groups){ 88 | $g .= $_->guid().','; 89 | } 90 | $g =~ s/,$//; 91 | return $g; 92 | } 93 | 94 | ## TODO -- move this to PROFILE 95 | sub expired { 96 | my $self = shift; 97 | my $args = shift; 98 | 99 | return 0 unless($self->expires()); 100 | 101 | my $time = DateTime::Format::DateParse->parse_datetime($self->expires()); 102 | return 1 if(time() > $time); 103 | return 0; 104 | } 105 | 106 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/APIKeyGroups.pm: -------------------------------------------------------------------------------- 1 | package CIF::APIKeyGroups; 2 | use base 'CIF::DBI'; 3 | 4 | __PACKAGE__->table('apikeys_groups'); 5 | __PACKAGE__->columns(Primary => qw/uuid guid/); 6 | __PACKAGE__->columns(All => qw/uuid guid default_guid created/); 7 | __PACKAGE__->sequence('apikeys_groups_id_seq'); 8 | __PACKAGE__->has_a(uuid => 'CIF::APIKey'); 9 | 10 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/APIKeyRestrictions.pm: -------------------------------------------------------------------------------- 1 | package CIF::APIKeyRestrictions; 2 | use base 'CIF::DBI'; 3 | 4 | __PACKAGE__->table('apikeys_restrictions'); 5 | __PACKAGE__->columns(Primary => qw/uuid access/); 6 | __PACKAGE__->columns(All => qw/uuid access created/); 7 | __PACKAGE__->sequence('apikeys_restrictions_id_seq'); 8 | __PACKAGE__->has_a(uuid => 'CIF::APIKey'); 9 | 10 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin; 2 | use base 'CIF::DBI'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use Digest::SHA qw/sha1_hex/; 8 | use Iodef::Pb::Simple qw/iodef_guid iodef_confidence/; 9 | 10 | sub query {} 11 | 12 | # sub tables are auto-defined by the plugin name 13 | # eg: Domain::Phishing translates to: 14 | # domain_phishing 15 | sub sub_table { 16 | my $class = shift; 17 | my $plug = shift; 18 | 19 | $plug =~ m/Plugin::(\S+)::(\S+)$/; 20 | my ($type,$subtype) = (lc($1),lc($2)); 21 | return $type.'_'.$subtype; 22 | } 23 | 24 | sub generate_sha1 { 25 | my $self = shift; 26 | my $thing = shift || return; 27 | 28 | return $thing if(lc($thing) =~ /^[a-f0-9]{40}$/); 29 | return sha1_hex($thing); 30 | } 31 | 32 | sub test_feed { 33 | my $class = shift; 34 | my $feeds = shift; 35 | 36 | $feeds = $feeds->{'feeds'}; 37 | return unless($feeds); 38 | $feeds = [$feeds] unless(ref($feeds) eq 'ARRAY'); 39 | 40 | return unless(@$feeds); 41 | foreach my $f (@$feeds){ 42 | return 1 if(lc($class) =~ /$f$/); 43 | } 44 | } 45 | 46 | sub test_datatype { 47 | my $class = shift; 48 | my $data = shift; 49 | 50 | return unless($data); 51 | $data = $data->{'datatypes'}; 52 | return unless($data); 53 | $data = [$data] unless(ref($data) eq 'ARRAY'); 54 | 55 | foreach (@$data){ 56 | return 1 if(lc($class) =~ /$_$/); 57 | } 58 | } 59 | 60 | sub insert_hash { 61 | my $class = shift; 62 | my $data = shift; 63 | my $hash = shift; 64 | 65 | $hash = sha1_hex($hash) unless($hash =~ /^[a-f0-9]{40}$/); 66 | 67 | my $id = CIF::Archive::Plugin::Hash->insert({ 68 | uuid => $data->{'uuid'}, 69 | guid => $data->{'guid'}, 70 | confidence => $data->{'confidence'}, 71 | hash => $hash, 72 | reporttime => $data->{'reporttime'}, 73 | }); 74 | return ($id); 75 | } 76 | 77 | 1; 78 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Asn.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Asn; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Try::Tiny; 9 | use Iodef::Pb::Simple qw(iodef_confidence iodef_bgp); 10 | use Digest::SHA qw/sha1_hex/; 11 | 12 | my @plugins = __PACKAGE__->plugins(); 13 | 14 | sub insert { 15 | my $class = shift; 16 | my $data = shift; 17 | 18 | return unless($class->test_datatype($data)); 19 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 20 | 21 | my $tbl = $class->table(); 22 | my @ids; 23 | 24 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 25 | foreach(@plugins){ 26 | if($_->prepare($i)){ 27 | $class->table($class->sub_table($_)); 28 | last; 29 | } 30 | } 31 | my $uuid = $i->get_IncidentID->get_content(); 32 | 33 | my $bgp = iodef_bgp($i); 34 | next unless($bgp); 35 | 36 | my $confidence = iodef_confidence($i); 37 | $confidence = @{$confidence}[0]->get_content(); 38 | 39 | foreach my $e (@$bgp){ 40 | next unless($e->{'asn'} && $e->{'asn'} =~ /^\d+/); 41 | my $asn = 'as'.$e->{'asn'}; 42 | my $hash = sha1_hex($asn); 43 | my $id = $class->insert_hash({ 44 | uuid => $data->{'uuid'}, 45 | guid => $data->{'guid'}, 46 | confidence => $confidence, 47 | reporttime => $data->{'reporttime'}, 48 | },$hash); 49 | 50 | push(@ids,$id); 51 | } 52 | } 53 | $class->table($tbl); 54 | return(undef,\@ids); 55 | } 56 | 57 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Cc.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Cc; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Try::Tiny; 9 | use Iodef::Pb::Simple qw(iodef_confidence iodef_bgp); 10 | use Digest::SHA qw/sha1_hex/; 11 | 12 | my @plugins = __PACKAGE__->plugins(); 13 | 14 | sub query { } # handled by the address module 15 | 16 | sub insert { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->test_datatype($data)); 21 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 22 | 23 | my $tbl = $class->table(); 24 | my @ids; 25 | 26 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 27 | foreach(@plugins){ 28 | if($_->prepare($i)){ 29 | $class->table($class->sub_table($_)); 30 | last; 31 | } 32 | } 33 | 34 | my $uuid = $i->get_IncidentID->get_content(); 35 | 36 | my $bgp = iodef_bgp($i); 37 | next unless($bgp); 38 | my $confidence = iodef_confidence($i); 39 | $confidence = @{$confidence}[0]->get_content(); 40 | 41 | foreach my $e (@$bgp){ 42 | next unless($e->{'cc'} && $e->{'cc'} =~ /^[a-zA-Z]{2}$/); 43 | $e->{'cc'} = lc($e->{'cc'}); 44 | my $hash = sha1_hex($e->{'cc'}); 45 | my $id = $class->insert_hash({ 46 | uuid => $data->{'uuid'}, 47 | guid => $data->{'guid'}, 48 | confidence => $confidence, 49 | reporttime => $data->{'reporttime'}, 50 | },$hash); 51 | 52 | push(@ids,$id); 53 | } 54 | } 55 | $class->table($tbl); 56 | return(undef,\@ids); 57 | } 58 | 59 | 1; 60 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Iodef::Pb::Simple qw(iodef_addresses iodef_confidence iodef_guid); 9 | use Digest::SHA qw/sha1_hex/; 10 | use CIF qw/debug/; 11 | 12 | __PACKAGE__->table('domain'); 13 | __PACKAGE__->columns(Primary => 'id'); 14 | __PACKAGE__->columns(All => qw/id uuid guid hash address confidence reporttime created/); 15 | __PACKAGE__->sequence('domain_id_seq'); 16 | 17 | my @plugins = __PACKAGE__->plugins(); 18 | 19 | sub query { } # handled by the address module 20 | 21 | sub insert { 22 | my $class = shift; 23 | my $data = shift; 24 | 25 | return unless($class->test_datatype($data)); 26 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 27 | 28 | my $tbl = $class->table(); 29 | my @ids; 30 | 31 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 32 | foreach(@plugins){ 33 | if($_->prepare($i)){ 34 | $class->table($class->sub_table($_)); 35 | last; 36 | } 37 | } 38 | my $reporttime = $i->get_ReportTime(); 39 | my $uuid = $i->get_IncidentID->get_content(); 40 | 41 | my $addresses = iodef_addresses($i); 42 | return unless(@$addresses); 43 | 44 | my $confidence = iodef_confidence($i); 45 | $confidence = @{$confidence}[0]->get_content(); 46 | 47 | foreach my $address (@$addresses){ 48 | my $addr = lc($address->get_content()); 49 | next if($addr =~ /^(ftp|https?):\/\//); 50 | # this way we can change the regex as we go if needed 51 | next if(CIF::Archive::Plugin::Email::is_email($addr)); 52 | next unless($addr =~ /[a-z0-9.\-_]+\.[a-z]{2,6}$/); 53 | if($class->test_feed($data)){ 54 | $class->SUPER::insert({ 55 | uuid => $data->{'uuid'}, 56 | guid => $data->{'guid'}, 57 | hash => sha1_hex($addr), 58 | address => $addr, 59 | confidence => $confidence, 60 | reporttime => $reporttime, 61 | }); 62 | } 63 | 64 | my @a1 = reverse(split(/\./,$addr)); 65 | my @a2 = @a1; 66 | foreach (0 ... $#a1-1){ 67 | my $a = join('.',reverse(@a2)); 68 | pop(@a2); 69 | my $hash = $class->SUPER::generate_sha1($a); 70 | my $id = $class->insert_hash({ 71 | uuid => $data->{'uuid'}, 72 | guid => $data->{'guid'}, 73 | confidence => $confidence, 74 | reporttime => $reporttime, 75 | },$hash); 76 | push(@ids,$id); 77 | } 78 | } 79 | } 80 | $class->table($tbl); 81 | return(undef,\@ids); 82 | } 83 | 84 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Botnet.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Botnet; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_botnet'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^botnet$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Fastflux.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Fastflux; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_fastflux'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /fastflux/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Malware; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_malware'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^(malware|exploit|malicious)/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Passive.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Passive; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_passive'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^passive$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Phishing.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Phishing; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_phishing'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /phish/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Spam.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Spam; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_spam'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^spam$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Spamvertising.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Spamvertising; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_spamvertising'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^spamvertising$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Suspicious.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Suspicious; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_suspicious'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^suspicious$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Domain/Whitelist.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Domain::Whitelist; 2 | use base 'CIF::Archive::Plugin::Domain'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('domain_whitelist'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /whitelist/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Email.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Email; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Digest::SHA qw(sha1_hex); 9 | use Iodef::Pb::Simple qw(iodef_addresses iodef_confidence iodef_guid); 10 | 11 | __PACKAGE__->table('email'); 12 | __PACKAGE__->columns(Primary => 'id'); 13 | __PACKAGE__->columns(Essential => qw/id uuid guid hash confidence reporttime created/); 14 | __PACKAGE__->sequence('email_id_seq'); 15 | 16 | my @plugins = __PACKAGE__->plugins(); 17 | 18 | 19 | sub is_email { 20 | my $e = shift; 21 | return unless($e); 22 | return if($e =~ /^(ftp|https?):\/\//); 23 | return unless(lc($e) =~ /^([\w+.-_]+\@[a-z0-9.-]+\.[a-z0-9.-]{2,8})$/); 24 | return(1); 25 | } 26 | 27 | sub insert { 28 | my $class = shift; 29 | my $data = shift; 30 | 31 | return unless($class->test_datatype($data)); 32 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 33 | 34 | my $addresses = iodef_addresses($data->{'data'}); 35 | return unless(@$addresses); 36 | 37 | my $tbl = $class->table(); 38 | my @ids; 39 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 40 | foreach(@plugins){ 41 | if($_->prepare($i)){ 42 | $class->table($_->table()); 43 | last; 44 | } 45 | } 46 | my $reporttime = $i->get_ReportTime(); 47 | my $confidence = iodef_confidence($i); 48 | $confidence = @{$confidence}[0]->get_content(); 49 | 50 | foreach my $address (@$addresses){ 51 | my $addr = lc($address->get_content()); 52 | next unless(is_email($addr)); 53 | my $hash = $class->SUPER::generate_sha1($addr); 54 | if($class->test_feed($data)){ 55 | $class->SUPER::insert({ 56 | uuid => $data->{'uuid'}, 57 | guid => $data->{'guid'}, 58 | hash => $hash, 59 | confidence => $confidence, 60 | reporttime => $reporttime, 61 | }); 62 | } 63 | $addr =~ /^([\w+.-_]+\@[a-z0-9.-]+\.[a-z0-9.-]{2,8})$/; 64 | $addr = $1; 65 | my @a1 = reverse(split(/\./,$addr)); 66 | my @a2 = @a1; 67 | foreach (0 ... $#a1-1){ 68 | my $a = join('.',reverse(@a2)); 69 | pop(@a2); 70 | my $hash = $class->SUPER::generate_sha1($a); 71 | my $id = $class->insert_hash({ 72 | uuid => $data->{'uuid'}, 73 | guid => $data->{'guid'}, 74 | confidence => $confidence, 75 | reporttime => $reporttime, 76 | },$hash); 77 | push(@ids,$id); 78 | } 79 | } 80 | } 81 | $class->table($tbl); 82 | return(undef,\@ids); 83 | 84 | } 85 | 86 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Email/Phishing.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Email::Phishing; 2 | use base 'CIF::Archive::Plugin::Email'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('email_phishing'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /phish/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Email/Registrant.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Email::Registrant; 2 | use base 'CIF::Archive::Plugin::Email'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('email_registrant'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /registrant/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Email/Suspicious.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Email::Suspicious; 2 | use base 'CIF::Archive::Plugin::Email'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('email_suspicious'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /suspicious/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Email/Whitelist.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Email::Whitelist; 2 | use base 'CIF::Archive::Plugin::Email'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('email_whitelist'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /whitelist/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Feed.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Feed; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | 9 | my @plugins = __PACKAGE__->plugins(); 10 | 11 | __PACKAGE__->table('feed'); 12 | __PACKAGE__->columns(Primary => 'id'); 13 | __PACKAGE__->columns(All => qw/id uuid guid hash confidence reporttime created/); 14 | __PACKAGE__->columns(Essential => qw/id uuid guid hash confidence reporttime created/); 15 | __PACKAGE__->sequence('feed_id_seq'); 16 | __PACKAGE__->has_a(uuid => 'CIF::Archive'); 17 | __PACKAGE__->add_trigger(after_delete => \&trigger_after_delete); 18 | 19 | sub trigger_after_delete { 20 | my $class = shift; 21 | 22 | my $hash = CIF::Archive::Plugin::Hash->retrieve(uuid => $class->uuid()); 23 | $hash->delete() if($hash); 24 | 25 | my $archive = CIF::Archive->retrieve(uuid => $class->uuid()); 26 | $archive->delete() if($archive); 27 | } 28 | 29 | sub insert { 30 | my $class = shift; 31 | my $data = shift; 32 | 33 | return unless(ref($data->{'data'}) eq 'FeedType'); 34 | 35 | my $hash = $data->{'data'}->get_description(); 36 | $hash = $class->SUPER::generate_sha1($hash); 37 | 38 | $data->{'confidence'} = $data->{'data'}->get_confidence(); 39 | 40 | $class->SUPER::insert({ 41 | guid => $data->{'guid'}, 42 | uuid => $data->{'uuid'}, 43 | hash => $hash, 44 | confidence => $data->{'confidence'}, 45 | reporttime => $data->{'reporttime'}, 46 | }); 47 | my $id = $class->insert_hash($data,$hash); 48 | 49 | return(undef,$id); 50 | } 51 | 52 | __PACKAGE__->set_sql(feeds => qq{ 53 | SELECT count(hash),guid,hash,confidence 54 | FROM __TABLE__ t 55 | GROUP BY hash,guid,confidence 56 | ORDER BY count desc 57 | }); 58 | 59 | __PACKAGE__->set_sql(feed_group => qq{ 60 | SELECT * 61 | FROM __TABLE__ t 62 | WHERE hash = ? 63 | AND confidence = ? 64 | ORDER BY id ASC 65 | }); 66 | 1; 67 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Hash/Md5.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Hash::Md5; 2 | use base 'CIF::Archive::Plugin::Hash'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | __PACKAGE__->table('hash_md5'); 8 | 9 | sub prepare { 10 | my $class = shift; 11 | my $data = shift; 12 | return unless(lc($data) =~ /^[a-f0-9]{32}$/); 13 | return(1); 14 | } 15 | 16 | sub query { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->prepare($data->{'query'})); 21 | return $class->search_lookup( 22 | $data->{'query'}, 23 | $data->{'confidence'}, 24 | $data->{'limit'}, 25 | ); 26 | } 27 | 28 | 1; 29 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Hash/Sha1.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Hash::Sha1; 2 | use base 'CIF::Archive::Plugin::Hash'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | __PACKAGE__->table('hash_sha1'); 8 | 9 | sub prepare { 10 | my $class = shift; 11 | my $data = shift; 12 | return unless(lc($data) =~ /^[a-f0-9]{40}$/); 13 | return(1); 14 | } 15 | 16 | sub query { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->prepare($data->{'query'})); 21 | 22 | ## TODO -- this is a crappy work-around, feeds are only searched by sha1's 23 | ## and we need to fix the ordering based on confidence 24 | ## if we're searching for a feed, it's one way, if not it's a different way 25 | if($data->{'limit'} && $data->{'limit'} == 1){ 26 | return $class->search_lookup_feed( 27 | $data->{'query'}, 28 | $data->{'confidence'}, 29 | $data->{'source'}, 30 | ); 31 | } 32 | return $class->search_lookup( 33 | $data->{'query'}, 34 | $data->{'confidence'}, 35 | $data->{'source'}, 36 | $data->{'limit'}, 37 | ); 38 | } 39 | 40 | # since all feed lookups are sha1 based, but we only want the last record 41 | __PACKAGE__->set_sql('lookup_feed' => qq{ 42 | SELECT t.id,t.uuid,archive.data 43 | FROM __TABLE__ t 44 | LEFT JOIN apikeys_groups on t.guid = apikeys_groups.guid 45 | LEFT JOIN archive ON archive.uuid = t.uuid 46 | WHERE 47 | hash = ? 48 | AND confidence >= ? 49 | AND apikeys_groups.uuid = ? 50 | AND archive.uuid IS NOT NULL 51 | ORDER BY t.confidence ASC, t.reporttime DESC, t.created DESC, t.id DESC 52 | LIMIT 1 53 | }); 54 | 55 | 1; 56 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Hash/Uuid.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Hash::Uuid; 2 | use base 'CIF::Archive::Plugin::Hash'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | __PACKAGE__->table('hash_uuid'); 8 | 9 | sub prepare { 10 | my $class = shift; 11 | my $data = shift; 12 | return unless(lc($data) =~ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); 13 | return(1); 14 | } 15 | 16 | sub query { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->prepare($data->{'query'})); 21 | return $class->search_lookup( 22 | $data->{'query'}, 23 | $data->{'confidence'}, 24 | $data->{'limit'}, 25 | ); 26 | } 27 | 28 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Botnet.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Botnet; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_botnet'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^botnet$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Fastflux.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Fastflux; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_fastflux'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /fastflux/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Malware; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_malware'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^(malware|exploit)$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Phishing.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Phishing; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_phishing'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /phish/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Scan.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Scan; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_scan'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if(lc($_->get_content->get_content()) =~ /^scan(?:(ning|ner))/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Spam.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Spam; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_spam'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^spam$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Spamvertising.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Spamvertising; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_spamvertising'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^spamvertising$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Suspicious.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Suspicious; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_suspicious'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^suspicious$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Infrastructure/Whitelist.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Infrastructure::Whitelist; 2 | use base 'CIF::Archive::Plugin::Infrastructure'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('infrastructure_whitelist'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /whitelist/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Malware; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | 9 | __PACKAGE__->table('malware'); 10 | __PACKAGE__->columns(Primary => 'id'); 11 | __PACKAGE__->columns(All => qw/id uuid guid hash confidence reporttime created/); 12 | __PACKAGE__->sequence('malware_id_seq'); 13 | 14 | use Iodef::Pb::Simple qw(:all); 15 | 16 | my @plugins = __PACKAGE__->plugins(); 17 | 18 | sub query { } # handled by the address module 19 | 20 | sub insert { 21 | my $class = shift; 22 | my $data = shift; 23 | 24 | return unless($class->test_datatype($data)); 25 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 26 | 27 | my @ids; 28 | my $tbl = $class->table(); 29 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 30 | 31 | my $impact = iodef_impacts_first($i); 32 | return unless($impact && $impact->get_content->get_content() eq 'malware'); 33 | 34 | my $reporttime = $i->get_ReportTime(); 35 | 36 | my $addresses = iodef_addresses($i); 37 | next if(@$addresses); 38 | 39 | my $ad = iodef_additional_data($i); 40 | return unless(@$ad); 41 | 42 | my $confidence = iodef_confidence($i); 43 | $confidence = @{$confidence}[0]->get_content(); 44 | 45 | foreach my $a (@$ad){ 46 | next unless($a->get_meaning() && lc($a->get_meaning()) =~ /^malware hash$/); 47 | next unless($a->get_content()); 48 | my $hash = $a->get_content(); 49 | if($class->test_feed($data)){ 50 | if(my $t = return_table($hash)){ 51 | $class->table($t); 52 | } 53 | $class->SUPER::insert({ 54 | hash => $hash, 55 | uuid => $i->get_IncidentID->get_content(), 56 | guid => iodef_guid($i) || $data->{'guid'}, 57 | confidence => $confidence, 58 | reporttime => $reporttime, 59 | }); 60 | } 61 | my $id = $class->insert_hash({ 62 | uuid => $data->{'uuid'}, 63 | guid => $data->{'guid'}, 64 | confidence => $confidence, 65 | reporttime => $reporttime, 66 | },$hash); 67 | ## TODO: mem leak? 68 | push(@ids,@$id); 69 | } 70 | } 71 | $class->table($tbl); 72 | return(undef,\@ids); 73 | } 74 | 75 | sub return_table { 76 | my $hash = shift; 77 | foreach (@plugins){ 78 | next unless($_->prepare($hash)); 79 | return $_->table(); 80 | last; 81 | } 82 | } 83 | 84 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Malware/Md5.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Malware::Md5; 2 | use base 'CIF::Archive::Plugin::Malware'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | __PACKAGE__->table('malware_md5'); 8 | 9 | sub prepare { 10 | my $class = shift; 11 | my $data = shift; 12 | return unless(lc($data) =~ /^[a-f0-9]{32}$/); 13 | return(1); 14 | } 15 | 16 | sub query { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->prepare($data->{'query'})); 21 | return $class->search_lookup( 22 | $data->{'query'}, 23 | $data->{'confidence'}, 24 | $data->{'limit'}, 25 | ); 26 | } 27 | 28 | 1; 29 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Malware/Sha1.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Malware::Sha1; 2 | use base 'CIF::Archive::Plugin::Malware'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | __PACKAGE__->table('malware_sha1'); 8 | 9 | sub prepare { 10 | my $class = shift; 11 | my $data = shift; 12 | return unless(lc($data) =~ /^[a-f0-9]{40}$/); 13 | return(1); 14 | } 15 | 16 | sub query { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->prepare($data->{'query'})); 21 | 22 | my $ret = $class->SUPER::search_lookup( 23 | $data->{'query'}, 24 | $data->{'confidence'}, 25 | $data->{'source'}, 26 | $data->{'limit'}, 27 | ); 28 | return $ret; 29 | } 30 | 31 | 1; 32 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Malware/Uuid.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Malware::Uuid; 2 | use base 'CIF::Archive::Plugin::Malware'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | __PACKAGE__->table('malware_uuid'); 8 | 9 | sub prepare { 10 | my $class = shift; 11 | my $data = shift; 12 | return unless(lc($data) =~ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); 13 | return(1); 14 | } 15 | 16 | sub query { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->prepare($data->{'query'})); 21 | return $class->search_lookup( 22 | $data->{'query'}, 23 | $data->{'confidence'}, 24 | $data->{'limit'}, 25 | ); 26 | } 27 | 28 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Rir.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Rir; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Try::Tiny; 9 | use Iodef::Pb::Simple qw(iodef_confidence iodef_bgp); 10 | use Digest::SHA qw/sha1_hex/; 11 | 12 | my @plugins = __PACKAGE__->plugins(); 13 | 14 | sub query { } # handled by the address module 15 | 16 | sub insert { 17 | my $class = shift; 18 | my $data = shift; 19 | 20 | return unless($class->test_datatype($data)); 21 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 22 | 23 | my $tbl = $class->table(); 24 | my @ids; 25 | 26 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 27 | foreach(@plugins){ 28 | if($_->prepare($i)){ 29 | $class->table($class->sub_table($_)); 30 | last; 31 | } 32 | } 33 | 34 | my $uuid = $i->get_IncidentID->get_content(); 35 | 36 | my $bgp = iodef_bgp($i); 37 | next unless($bgp); 38 | my $confidence = iodef_confidence($i); 39 | $confidence = @{$confidence}[0]->get_content(); 40 | 41 | foreach my $e (@$bgp){ 42 | next unless($e->{'rir'} && lc($e->{'rir'}) =~ /^(afrinic|apnic|arin|lacnic|ripencc)$/); 43 | $e->{'rir'} = lc($e->{'rir'}); 44 | my $hash = sha1_hex($e->{'rir'}); 45 | my $id = $class->insert_hash({ 46 | uuid => $data->{'uuid'}, 47 | guid => $data->{'guid'}, 48 | confidence => $confidence, 49 | reporttime => $data->{'reporttime'}, 50 | },$hash); 51 | 52 | push(@ids,$id); 53 | } 54 | } 55 | $class->table($tbl); 56 | return(undef,\@ids); 57 | } 58 | 59 | 1; 60 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Search.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Search; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Digest::SHA qw/sha1_hex/; 9 | use Try::Tiny; 10 | use Iodef::Pb::Simple qw(iodef_confidence iodef_impacts iodef_additional_data iodef_guid); 11 | 12 | __PACKAGE__->table('search'); 13 | __PACKAGE__->columns(Primary => 'id'); 14 | __PACKAGE__->columns(All => qw/id uuid guid term confidence reporttime created/); 15 | __PACKAGE__->sequence('search_id_seq'); 16 | 17 | my @plugins = __PACKAGE__->plugins(); 18 | 19 | sub query { } # handled by the hash module 20 | 21 | sub is_search { 22 | my $class = shift; 23 | my $data = shift; 24 | 25 | my $impacts = iodef_impacts($data); 26 | foreach (@$impacts){ 27 | return 1 if($_->get_content->get_content() =~ /^search/); 28 | } 29 | } 30 | 31 | sub insert { 32 | my $class = shift; 33 | my $data = shift; 34 | 35 | return unless($data->{'search'}); # don't ask 36 | return unless($class->test_datatype($data)); 37 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 38 | my $tbl = $class->table(); 39 | my @ids; 40 | 41 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 42 | next unless($class->is_search($i)); 43 | my $confidence = iodef_confidence($i); 44 | $confidence = @{$confidence}[0]->get_content(); 45 | my $reporttime = $i->get_ReportTime(); 46 | 47 | foreach(@plugins){ 48 | if($_->prepare($data)){ 49 | $class->table($class->sub_table($_)); 50 | last; 51 | } 52 | } 53 | if($class->test_feed($data)){ 54 | $class->SUPER::insert({ 55 | guid => iodef_guid($i) || $data->{'guid'}, 56 | uuid => $i->get_IncidentID->get_content(), 57 | term => $data->{'search'}, 58 | confidence => $confidence, 59 | reporttime => $reporttime, 60 | }); 61 | } 62 | 63 | my $id = $class->insert_hash({ 64 | uuid => $data->{'uuid'}, 65 | guid => $data->{'guid'}, 66 | confidence => $confidence, 67 | reporttime => $reporttime, 68 | },$class->SUPER::generate_sha1($data->{'search'})); 69 | push(@ids,$id); 70 | } 71 | $class->table($tbl); 72 | return(undef,\@ids); 73 | } 74 | 75 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url; 2 | use base 'CIF::Archive::Plugin'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Iodef::Pb::Simple qw(:all); 9 | use CIF qw/debug/; 10 | 11 | my @plugins = __PACKAGE__->plugins(); 12 | 13 | __PACKAGE__->table('url'); 14 | __PACKAGE__->columns(Primary => 'id'); 15 | __PACKAGE__->columns(All => qw/id uuid guid hash confidence reporttime created/); 16 | __PACKAGE__->sequence('url_id_seq'); 17 | 18 | sub query { } # handled by hash lookup 19 | 20 | sub insert { 21 | my $class = shift; 22 | my $data = shift; 23 | 24 | return unless($class->test_datatype($data)); 25 | return unless(ref($data->{'data'}) eq 'IODEFDocumentType'); 26 | 27 | my $addresses = iodef_addresses($data->{'data'}); 28 | return unless(@$addresses); 29 | 30 | my $tbl = $class->table(); 31 | my @ids; 32 | foreach my $i (@{$data->{'data'}->get_Incident()}){ 33 | foreach(@plugins){ 34 | if($_->prepare($i)){ 35 | $class->table($_->table()); 36 | } 37 | } 38 | my $reporttime = $i->get_ReportTime(); 39 | my $confidence = iodef_confidence($i); 40 | $confidence = @{$confidence}[0]->get_content(); 41 | 42 | foreach my $address (@$addresses){ 43 | my $addr = lc($address->get_content()); 44 | next unless($addr =~ /^(ftp|https?):\/\//); 45 | ## TODO -- pull this out of the IODEF ? 46 | my $hash = $class->SUPER::generate_sha1($addr); 47 | if($class->test_feed($data)){ 48 | $class->SUPER::insert({ 49 | guid => iodef_guid($i) || $data->{'guid'}, 50 | uuid => $i->get_IncidentID->get_content(), 51 | hash => $hash, 52 | confidence => $confidence, 53 | reporttime => $reporttime, 54 | }); 55 | } 56 | 57 | my $id = $class->insert_hash({ 58 | uuid => $data->{'uuid'}, 59 | guid => $data->{'guid'}, 60 | confidence => $confidence, 61 | reporttime => $reporttime, 62 | },$hash); 63 | push(@ids,$id); 64 | } 65 | } 66 | $class->table($tbl); 67 | return(undef,\@ids); 68 | } 69 | 70 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url/Botnet.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url::Botnet; 2 | use base 'CIF::Archive::Plugin::Url'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('url_botnet'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^botnet$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url::Malware; 2 | use base 'CIF::Archive::Plugin::Url'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('url_malware'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^(malware|exploit)$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url/Phishing.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url::Phishing; 2 | use base 'CIF::Archive::Plugin::Url'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('url_phishing'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /phish/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url/Spam.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url::Spam; 2 | use base 'CIF::Archive::Plugin::Url'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('url_spam'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | 17 | foreach (@$impacts){ 18 | return 1 if($_->get_content->get_content() =~ /^spam$/); 19 | } 20 | return(0); 21 | } 22 | 23 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url/Spamvertising.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url::Spamvertising; 2 | use base 'CIF::Archive::Plugin::Url'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('url_spamvertising'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /^spamvertising$/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Archive/Plugin/Url/Whitelist.pm: -------------------------------------------------------------------------------- 1 | package CIF::Archive::Plugin::Url::Whitelist; 2 | use base 'CIF::Archive::Plugin::Url'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Iodef::Pb::Simple qw(iodef_impacts); 8 | 9 | __PACKAGE__->table('url_whitelist'); 10 | 11 | sub prepare { 12 | my $class = shift; 13 | my $data = shift; 14 | 15 | my $impacts = iodef_impacts($data); 16 | foreach (@$impacts){ 17 | return 1 if($_->get_content->get_content() =~ /whitelist/); 18 | } 19 | return(0); 20 | } 21 | 22 | 1; 23 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/DBI.pm: -------------------------------------------------------------------------------- 1 | package CIF::DBI; 2 | use base 'Class::DBI'; 3 | 4 | our $VERSION = '1.0.0'; 5 | 6 | use strict; 7 | use warnings; 8 | 9 | use CIF qw/debug/; 10 | use Config::Simple; 11 | 12 | sub new { 13 | my $class = shift; 14 | my $args = shift; 15 | 16 | my $self = {}; 17 | bless($self,$class); 18 | 19 | my ($err,$ret) = $self->init($args); 20 | return $err if($err); 21 | return(undef,$self); 22 | } 23 | 24 | sub init { 25 | my $self = shift; 26 | my $args = shift; 27 | 28 | my $config = Config::Simple->new($args->{'config'}) || return('missing config file'); 29 | 30 | my $datatypes = $config->param(-block => 'cif_archive')->{'datatypes'} || ['infrastructure','domain','url','email','malware','search']; 31 | my $feeds = $config->param(-block => 'cif_archive')->{'feeds'}; 32 | 33 | $self->{'datatypes'} = $datatypes; 34 | $self->{'feeds'} = $feeds; 35 | 36 | my ($err,$ret) = $self->init_db($args); 37 | return $err unless($ret); 38 | 39 | return (undef,$ret); 40 | } 41 | 42 | sub init_db { 43 | my $self = shift; 44 | my $args = shift; 45 | 46 | my $config = Config::Simple->new($args->{'config'}) || return('missing config file'); 47 | 48 | $config = $config->param(-block => 'db'); 49 | 50 | my $db = $config->{'database'} || 'cif'; 51 | my $user = $config->{'user'} || 'postgres'; 52 | my $password = $config->{'password'} || ''; 53 | my $host = $config->{'host'} || '127.0.0.1'; 54 | 55 | my $dbi = 'DBI:Pg:database='.$db.';host='.$host; 56 | my $ret = $self->connection($dbi,$user,$password,{ AutoCommit => 0}); 57 | return(undef,$ret); 58 | } 59 | 60 | # because UUID's are really primary keys too in our schema 61 | # this overrides some of the default functionality of Class::DBI and 'id' 62 | sub retrieve { 63 | my $class = shift; 64 | 65 | return $class->SUPER::retrieve(@_) if(@_ == 1); 66 | my %keys = @_; 67 | 68 | my @recs = $class->search_retrieve_uuid($keys{'uuid'}); 69 | return unless(defined($#recs) && $#recs > -1); 70 | return($recs[0]); 71 | } 72 | 73 | sub vacuum { 74 | my $class = shift; 75 | my $args = shift; 76 | 77 | my $ts = $args->{'timestamp'}; 78 | 79 | my ($ret,$err); 80 | 81 | ## TODO -- try catch? 82 | $ret = $class->sql_vacuum->execute($ts); 83 | $class->dbi_commit() unless($class->db_Main->{'AutoCommit'}); 84 | return (undef,$ret); 85 | } 86 | 87 | __PACKAGE__->set_sql('retrieve_uuid' => qq{ 88 | SELECT * 89 | FROM __TABLE__ 90 | WHERE uuid = ? 91 | ORDER BY id DESC 92 | LIMIT 1 93 | }); 94 | 95 | __PACKAGE__->set_sql('vacuum' => qq{ 96 | DELETE FROM __TABLE__ 97 | WHERE reporttime <= ?; 98 | }); 99 | 100 | 101 | 1; 102 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Feed/Plugin.pm: -------------------------------------------------------------------------------- 1 | package CIF::Feed::Plugin; 2 | use base 'CIF::DBI'; 3 | use base 'Class::Accessor'; 4 | 5 | use warnings; 6 | use strict; 7 | 8 | use Try::Tiny; 9 | use CIF::Msg; 10 | use CIF::Msg::Feed; 11 | use CIF qw/generate_uuid_random debug/; 12 | 13 | __PACKAGE__->columns(All => qw/id uuid/); 14 | __PACKAGE__->columns(Primary => 'id'); 15 | __PACKAGE__->has_a(uuid => 'CIF::Archive'); 16 | 17 | __PACKAGE__->follow_best_practice(); 18 | __PACKAGE__->mk_accessors(qw(restriction_map group_map)); 19 | 20 | sub generate_feeds { 21 | my $class = shift; 22 | my $args = shift; 23 | 24 | my @vars = @{$args->{'vars'}}; 25 | 26 | my $sth = $class->sql_feed(); 27 | debug('executing'); 28 | my $ret = $sth->execute(@vars); 29 | return unless($ret); 30 | ## TODO: protect against orphan keys 31 | ## if there's nothing in the data section, trigger a warning 32 | ## or a delete? 33 | debug('fetching'); 34 | return($sth->fetchall_hashref('id')); 35 | } 36 | 37 | sub encode_feed { 38 | my $class = shift; 39 | my $args = shift; 40 | 41 | my $recs = $args->{'recs'}; 42 | if(keys %$recs){ 43 | $recs = [ map { $recs->{$_}->{'data'} } keys (%$recs) ]; 44 | } else { 45 | $recs = []; 46 | } 47 | 48 | delete($args->{'recs'}); 49 | 50 | my $feed = FeedType->new({ 51 | description => $args->{'description'}, 52 | ReportTime => $args->{'report_time'}, 53 | data => $recs, 54 | version => $CIF::VERSION, 55 | confidence => $args->{'confidence'}, 56 | guid => $args->{'guid'}, 57 | group_map => $args->{'group_map'}, 58 | restriction_map => $args->{'restriction_map'}, 59 | uuid => generate_uuid_random(), 60 | restriction => $args->{'restriction'}, 61 | }); 62 | return $feed; 63 | } 64 | 65 | sub test_whitelist { 66 | my $class = shift; 67 | my $args = shift; 68 | 69 | return $args->{'recs'} if(keys %{$args->{'recs'}}); 70 | } 71 | 72 | __PACKAGE__->set_sql('feed' => qq{ 73 | SELECT DISTINCT ON (t1.hash) t1.hash, t1.id, archive.data 74 | FROM ( 75 | SELECT t.hash, t.id, t.uuid, t.guid 76 | FROM __TABLE__ t 77 | WHERE 78 | t.reporttime >= ? 79 | AND t.confidence >= ? 80 | ORDER by t.id DESC 81 | LIMIT ? 82 | ) t1 83 | LEFT JOIN archive ON t1.uuid = archive.uuid 84 | LEFT JOIN apikeys_groups ON t1.guid = apikeys_groups.guid 85 | WHERE apikeys_groups.uuid = ? 86 | }); 87 | 88 | __PACKAGE__->set_sql('feed_whitelist' => qq{ 89 | SELECT DISTINCT on (t1.hash) t1.hash 90 | FROM ( 91 | SELECT t2.hash 92 | FROM __TABLE__ t2 93 | WHERE 94 | t2.reporttime >= ? 95 | AND t2.confidence >= ? 96 | ORDER BY id DESC 97 | LIMIT ? 98 | ) t1 99 | }); 100 | 101 | 102 | 103 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Feed/Plugin/Address.pm: -------------------------------------------------------------------------------- 1 | package CIF::Feed::Plugin::Address; 2 | use base 'CIF::Feed::Plugin'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use CIF qw(debug); 8 | 9 | __PACKAGE__->columns(All => qw/id uuid guid hash address confidence reporttime created/); 10 | 11 | sub generate_feeds { return; } 12 | 13 | # specific to domain and infrastructure 14 | # where we need the address to bench against the whitelist 15 | no warnings; 16 | sub generate_feeds { 17 | my $class = shift; 18 | my $args = shift; 19 | 20 | my @vars = @{$args->{'vars'}}; 21 | 22 | my $sth = $class->sql_feed(); 23 | debug('executing'); 24 | my $ret = $sth->execute(@vars); 25 | return unless($ret); 26 | debug('fetching'); 27 | return($sth->fetchall_hashref('id')); 28 | } 29 | use warnings; 30 | 31 | # used for domains and ip's 32 | # because we can have /24's and blanket TLDs in the whitelist 33 | 34 | __PACKAGE__->set_sql('feed' => qq{ 35 | SELECT DISTINCT ON (t1.hash) t1.hash, t1.id, t1.address, archive.data 36 | FROM ( 37 | SELECT t.hash, t.id, t.uuid, t.guid, t.address 38 | FROM __TABLE__ t 39 | WHERE 40 | t.reporttime >= ? 41 | AND t.confidence >= ? 42 | ORDER by t.id DESC 43 | LIMIT ? 44 | ) t1 45 | LEFT JOIN archive ON t1.uuid = archive.uuid 46 | LEFT JOIN apikeys_groups ON t1.guid = apikeys_groups.guid 47 | WHERE apikeys_groups.uuid = ? 48 | }); 49 | 50 | __PACKAGE__->set_sql('feed_whitelist' => qq{ 51 | SELECT DISTINCT on (t1.hash) t1.hash, t1.address 52 | FROM ( 53 | SELECT t2.hash, t2.address 54 | FROM __TABLE__ t2 55 | WHERE 56 | t2.reporttime >= ? 57 | AND t2.confidence >= ? 58 | ORDER BY id DESC 59 | LIMIT ? 60 | ) t1 61 | }); 62 | 63 | # override since we're just a skel. 64 | 65 | sub vacuum { } 66 | 67 | 68 | 1; 69 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Feed/Plugin/Email.pm: -------------------------------------------------------------------------------- 1 | package CIF::Feed::Plugin::Email; 2 | use base 'CIF::Feed::Plugin'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use CIF qw/debug/; 9 | 10 | __PACKAGE__->table('email'); 11 | __PACKAGE__->columns(All => qw/id uuid guid hash confidence reporttime created/); 12 | __PACKAGE__->sequence('email_id_seq'); 13 | 14 | ## TODO: database config? 15 | my @plugins = __PACKAGE__->plugins(); 16 | push(@plugins,'phishing','registrant','spam','spamvertising','suspicious','whitelist'); 17 | 18 | sub generate_feeds { 19 | my $class = shift; 20 | my $args = shift; 21 | 22 | my $tbl = $class->table(); 23 | 24 | my @feeds; 25 | foreach my $p (@plugins){ 26 | my $t = $p; 27 | if($p =~ /Email::(\S+)$/){ 28 | $t = lc($1); 29 | } 30 | my $desc = $t.' email feed'; 31 | $t = 'email_'.$t; 32 | $class->table($t); 33 | my $feed_args = { 34 | description => $desc, 35 | report_time => $args->{'report_time'}, 36 | confidence => $args->{'confidence'}, 37 | guid => $args->{'guid'}, 38 | vars => [ 39 | $args->{'start_time'}, 40 | $args->{'confidence'}, 41 | $args->{'limit'}, 42 | $args->{'uuid'}, 43 | ], 44 | group_map => $args->{'group_map'}, 45 | restriction_map => $args->{'restriction_map'}, 46 | restriction => $args->{'restriction'}, 47 | }; 48 | debug($desc.': generating'); 49 | my $f = $class->SUPER::generate_feeds($feed_args); 50 | debug('final count: '.keys(%$f)); 51 | debug($desc.': encoding'); 52 | $f = $class->SUPER::encode_feed({ recs => $f, %$feed_args }); 53 | push(@feeds,$f); 54 | } 55 | $class->table($tbl); 56 | return(\@feeds); 57 | } 58 | 59 | 1; 60 | -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Feed/Plugin/Malware.pm: -------------------------------------------------------------------------------- 1 | package CIF::Feed::Plugin::Malware; 2 | use base 'CIF::Feed::Plugin'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use CIF qw/debug/; 9 | 10 | __PACKAGE__->table('malware'); 11 | __PACKAGE__->columns(All => qw/id uuid guid hash confidence reporttime created/); 12 | __PACKAGE__->sequence('malware_id_seq'); 13 | 14 | ## TODO: database config? 15 | my @plugins = __PACKAGE__->plugins(); 16 | push(@plugins, ('md5','sha1','uuid')); 17 | 18 | sub generate_feeds { 19 | my $class = shift; 20 | my $args = shift; 21 | 22 | my $tbl = $class->table(); 23 | 24 | my @feeds; 25 | foreach my $p (@plugins){ 26 | my $t = $p; 27 | if($p =~ /Domain::(\S+)$/){ 28 | $t = lc($1); 29 | } 30 | my $desc = $t.' malware feed'; 31 | $t = 'malware_'.$t; 32 | $class->table($t); 33 | my $feed_args = { 34 | description => $desc, 35 | report_time => $args->{'report_time'}, 36 | confidence => $args->{'confidence'}, 37 | guid => $args->{'guid'}, 38 | vars => [ 39 | $args->{'start_time'}, 40 | $args->{'confidence'}, 41 | $args->{'limit'}, 42 | $args->{'uuid'}, 43 | ], 44 | group_map => $args->{'group_map'}, 45 | restriction_map => $args->{'restriction_map'}, 46 | restriction => $args->{'restriction'}, 47 | }; 48 | debug($desc.': generating'); 49 | my $f = $class->SUPER::generate_feeds($feed_args); 50 | debug('records: '.keys %$f); 51 | debug($desc.': encoding'); 52 | $f = $class->SUPER::encode_feed({ recs => $f, %$feed_args }); 53 | push(@feeds,$f); 54 | } 55 | $class->table($tbl); 56 | return(\@feeds); 57 | } 58 | 59 | 60 | 1; -------------------------------------------------------------------------------- /libcif-dbi/lib/CIF/Feed/Plugin/Search.pm: -------------------------------------------------------------------------------- 1 | package CIF::Feed::Plugin::Search; 2 | use base 'CIF::Feed::Plugin'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use CIF qw/debug/; 9 | 10 | __PACKAGE__->table('search'); 11 | __PACKAGE__->columns(All => qw/id uuid guid term confidence reporttime created/); 12 | __PACKAGE__->sequence('search_id_seq'); 13 | 14 | ## TODO: database config? 15 | my @plugins = __PACKAGE__->plugins(); 16 | 17 | sub generate_feeds { 18 | my $class = shift; 19 | my $args = shift; 20 | 21 | my @feeds; 22 | 23 | my $desc = 'search feed'; 24 | 25 | my $feed_args = { 26 | description => $desc, 27 | report_time => $args->{'report_time'}, 28 | confidence => $args->{'confidence'}, 29 | guid => $args->{'guid'}, 30 | vars => [ 31 | $args->{'start_time'}, 32 | $args->{'confidence'}, 33 | $args->{'limit'}, 34 | $args->{'uuid'}, 35 | ], 36 | group_map => $args->{'group_map'}, 37 | restriction_map => $args->{'restriction_map'}, 38 | restriction => $args->{'restriction'}, 39 | }; 40 | debug($desc.': generating'); 41 | my $f = $class->SUPER::generate_feeds($feed_args); 42 | debug('records: '.keys %$f); 43 | debug($desc.': encoding'); 44 | $f = $class->SUPER::encode_feed({ recs => $f, %$feed_args }); 45 | push(@feeds,$f); 46 | 47 | return(\@feeds); 48 | } 49 | 50 | __PACKAGE__->set_sql('feed' => qq{ 51 | SELECT DISTINCT ON (t1.term) t1.term, t1.id, archive.data 52 | FROM ( 53 | SELECT t.term, t.id, t.uuid, t.guid 54 | FROM __TABLE__ t 55 | WHERE 56 | t.reporttime >= ? 57 | AND t.confidence >= ? 58 | ORDER by t.id DESC 59 | LIMIT ? 60 | ) t1 61 | LEFT JOIN archive ON t1.uuid = archive.uuid 62 | LEFT JOIN apikeys_groups ON t1.guid = apikeys_groups.guid 63 | WHERE apikeys_groups.uuid = ? 64 | }); 65 | 66 | 1; 67 | -------------------------------------------------------------------------------- /libcif-dbi/rebuild.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make bindings 5 | make dist 6 | -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/00_archive.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'archive'; 2 | DROP TABLE IF EXISTS archive CASCADE; 3 | 4 | CREATE TABLE archive ( 5 | id BIGSERIAL NOT NULL, 6 | uuid uuid NOT NULL, 7 | guid uuid, 8 | format text, 9 | reporttime timestamp with time zone DEFAULT NOW(), 10 | created timestamp with time zone DEFAULT NOW(), 11 | data text not null 12 | ); 13 | 14 | SET default_tablespace = 'index'; 15 | ALTER TABLE archive ADD PRIMARY KEY (id); 16 | CREATE INDEX idx_archive_uuid ON archive (uuid); 17 | CREATE INDEX idx_archive_created ON archive (created); -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/apikeys.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'archive'; 2 | DROP TABLE IF EXISTS apikeys_groups; 3 | DROP TABLE IF EXISTS apikeys_restrictions; 4 | DROP TABLE IF EXISTS apikeys; 5 | CREATE TABLE apikeys ( 6 | id BIGSERIAL PRIMARY KEY NOT NULL, 7 | uuid uuid not null, 8 | uuid_alias text NOT NULL, 9 | description text, 10 | parentid uuid null, 11 | revoked bool default null, 12 | restricted_access bool default false, 13 | write bool default null, 14 | created timestamp with time zone DEFAULT NOW(), 15 | expires timestamp with time zone, 16 | UNIQUE(uuid) 17 | ); 18 | 19 | CREATE TABLE apikeys_groups ( 20 | id BIGSERIAL PRIMARY KEY NOT NULL, 21 | uuid uuid references apikeys(uuid) on delete cascade not null, 22 | guid uuid not null, 23 | default_guid bool, 24 | created timestamp with time zone default now(), 25 | unique(uuid,guid) 26 | ); 27 | 28 | CREATE TABLE apikeys_restrictions ( 29 | id BIGSERIAL PRIMARY KEY NOT NULL, 30 | uuid uuid references apikeys(uuid) on delete cascade not null, 31 | access varchar(40) not null, 32 | created timestamp with time zone DEFAULT NOW(), 33 | UNIQUE(uuid,access) 34 | ); 35 | -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/domains.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | 3 | DROP TABLE IF EXISTS domain CASCADE; 4 | CREATE TABLE domain ( 5 | id BIGSERIAL PRIMARY KEY NOT NULL, 6 | uuid uuid NOT NULL, 7 | guid uuid, 8 | hash text, 9 | address text, 10 | confidence real, 11 | reporttime timestamp with time zone default NOW(), 12 | created timestamp with time zone DEFAULT NOW() 13 | ); 14 | 15 | DROP TABLE IF EXISTS domain_botnet; 16 | CREATE TABLE domain_botnet () INHERITS (domain); 17 | ALTER TABLE domain_botnet ADD PRIMARY KEY (id); 18 | 19 | DROP TABLE IF EXISTS domain_malware; 20 | CREATE TABLE domain_malware () INHERITS (domain); 21 | ALTER TABLE domain_malware ADD PRIMARY KEY (id); 22 | 23 | DROP TABLE IF EXISTS domain_fastflux; 24 | CREATE TABLE domain_fastflux () INHERITS (domain); 25 | ALTER TABLE domain_fastflux ADD PRIMARY KEY (id); 26 | 27 | DROP TABLE IF EXISTS domain_whitelist; 28 | CREATE TABLE domain_whitelist () INHERITS (domain); 29 | ALTER TABLE domain_whitelist ADD PRIMARY KEY (id); 30 | 31 | DROP TABLE IF EXISTS domain_suspicious; 32 | CREATE TABLE domain_suspicious () INHERITS (domain); 33 | ALTER TABLE domain_suspicious ADD PRIMARY KEY (id); 34 | 35 | DROP TABLE IF EXISTS domain_phishing; 36 | CREATE TABLE domain_phishing () INHERITS (domain); 37 | ALTER TABLE domain_phishing ADD PRIMARY KEY (id); 38 | 39 | DROP TABLE IF EXISTS domain_spamvertising; 40 | CREATE TABLE domain_spamvertising () INHERITS (domain); 41 | ALTER TABLE domain_spamvertising ADD PRIMARY KEY (id); 42 | 43 | DROP TABLE IF EXISTS domain_spam; 44 | CREATE TABLE domain_spam () INHERITS (domain); 45 | ALTER TABLE domain_spam ADD PRIMARY KEY (id); 46 | 47 | DROP TABLE IF EXISTS domain_passive; 48 | CREATE TABLE domain_passive () INHERITS (domain); 49 | ALTER TABLE domain_passive ADD PRIMARY KEY (id); -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/email.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | 3 | DROP TABLE IF EXISTS email CASCADE; 4 | CREATE TABLE email ( 5 | id BIGSERIAL PRIMARY KEY NOT NULL, 6 | uuid uuid NOT NULL, 7 | guid uuid, 8 | hash varchar(40), 9 | confidence real, 10 | reporttime timestamp with time zone default NOW(), 11 | created timestamp with time zone DEFAULT NOW() 12 | ); 13 | 14 | DROP TABLE IF EXISTS email_suspicious; 15 | CREATE TABLE email_suspicious () INHERITS (email); 16 | ALTER TABLE email_suspicious ADD PRIMARY KEY (id); 17 | 18 | DROP TABLE IF EXISTS email_phishing; 19 | CREATE TABLE email_phishing () INHERITS (email); 20 | ALTER TABLE email_phishing ADD PRIMARY KEY (id); 21 | 22 | DROP TABLE IF EXISTS email_registrant; 23 | CREATE TABLE email_registrant () INHERITS (email); 24 | ALTER TABLE email_registrant ADD PRIMARY KEY (id); 25 | 26 | DROP TABLE IF EXISTS email_whitelist; 27 | CREATE TABLE email_whitelist () INHERITS (email); 28 | ALTER TABLE email_whitelist ADD PRIMARY KEY (id); 29 | 30 | DROP TABLE IF EXISTS email_spam; 31 | CREATE TABLE email_spam () INHERITS (email); 32 | ALTER TABLE email_spam ADD PRIMARY KEY (id); 33 | 34 | DROP TABLE IF EXISTS email_spamvertising; 35 | CREATE TABLE email_spamvertising () INHERITS (email); 36 | ALTER TABLE email_spamvertising ADD PRIMARY KEY (id); -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/feed.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | DROP TABLE IF EXISTS feed CASCADE; 3 | CREATE TABLE feed ( 4 | id BIGSERIAL PRIMARY KEY NOT NULL, 5 | uuid uuid NOT NULL, 6 | guid uuid, 7 | hash text, 8 | confidence real, 9 | reporttime timestamp with time zone default NOW(), 10 | created timestamp with time zone DEFAULT NOW() 11 | ); 12 | 13 | DROP INDEX IF EXISTS idx_feed; 14 | CREATE INDEX idx_feed ON feed (hash,confidence); 15 | 16 | DROP INDEX IF EXISTS idx_feed_uuid; 17 | CREATE INDEX idx_feed_uuid ON feed (uuid); -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/hash.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | DROP TABLE IF EXISTS hash CASCADE; 3 | CREATE TABLE hash ( 4 | id BIGSERIAL, 5 | uuid uuid NOT NULL, 6 | guid uuid NOT NULL, 7 | hash text not null, 8 | confidence real, 9 | reporttime timestamp with time zone DEFAULT NOW(), 10 | created timestamp with time zone DEFAULT NOW() 11 | ); 12 | 13 | CREATE TABLE hash_sha1 () INHERITS (hash); 14 | ALTER TABLE hash_sha1 ADD PRIMARY KEY (id); 15 | 16 | CREATE TABLE hash_md5 () INHERITS (hash); 17 | ALTER TABLE hash_md5 ADD PRIMARY KEY (id); 18 | 19 | CREATE TABLE hash_uuid () INHERITS (hash); 20 | ALTER TABLE hash_uuid ADD PRIMARY KEY (id); -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/index/hash.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | 3 | -- MD5 table 4 | DROP INDEX IF EXISTS idx_hash_md5_1; 5 | CREATE INDEX idx_hash_md5 ON hash_md5 (hash,confidence); 6 | 7 | DROP INDEX IF EXISTS idx_hash_md5_2; 8 | CREATE INDEX idx_hash_md5_2 ON hash_md5 (uuid); 9 | 10 | DROP INDEX IF EXISTS idx_hash_md5_3; 11 | CREATE INDEX idx_hash_md5_3 ON hash_md5 (reporttime); 12 | 13 | -- SHA1 Table 14 | DROP INDEX IF EXISTS idx_hash_sha1_1; 15 | CREATE INDEX idx_hash_sha1_1 ON hash_sha1 (hash,confidence); 16 | 17 | DROP INDEX IF EXISTS idx_hash_sha1_2; 18 | CREATE INDEX idx_hash_sha1_2 ON hash_sha1 (uuid); 19 | 20 | DROP INDEX IF EXISTS idx_hash_sha1_3; 21 | CREATE INDEX idx_hash_sha1_3 ON hash_sha1 (reporttime); 22 | 23 | DROP INDEX IF EXISTS idx_hash_sha1_4; 24 | CREATE INDEX idx_hash_sha1_4 ON hash_sha1 (hash,id); 25 | 26 | --UUID table 27 | DROP INDEX IF EXISTS idx_hash_uuid_1; 28 | CREATE INDEX idx_hash_uuid_1 ON hash_uuid (hash,confidence); 29 | 30 | DROP INDEX IF EXISTS idx_hash_uuid_2; 31 | CREATE INDEX idx_hash_uuid_2 ON hash_uuid (uuid); 32 | 33 | DROP INDEX IF EXISTS idx_hash_uuid_3; 34 | CREATE INDEX idx_hash_uuid_3 ON hash_uuid (reporttime); 35 | -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/infrastructure.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | 3 | DROP TABLE IF EXISTS infrastructure CASCADE; 4 | CREATE TABLE infrastructure ( 5 | id BIGSERIAL PRIMARY KEY NOT NULL, 6 | uuid uuid NOT NULL, 7 | guid uuid, 8 | hash text, 9 | address text, 10 | confidence real, 11 | reporttime timestamp with time zone DEFAULT NOW(), 12 | created timestamp with time zone DEFAULT NOW() 13 | ); 14 | 15 | DROP TABLE IF EXISTS infrastructure_botnet; 16 | CREATE TABLE infrastructure_botnet () INHERITS (infrastructure); 17 | ALTER TABLE infrastructure_botnet ADD PRIMARY KEY (id); 18 | 19 | DROP TABLE IF EXISTS infrastructure_malware; 20 | CREATE TABLE infrastructure_malware () INHERITS (infrastructure); 21 | ALTER TABLE infrastructure_malware ADD PRIMARY KEY (id); 22 | 23 | DROP TABLE IF EXISTS infrastructure_fastflux; 24 | CREATE TABLE infrastructure_fastflux () INHERITS (infrastructure); 25 | ALTER TABLE infrastructure_fastflux ADD PRIMARY KEY (id); 26 | 27 | DROP TABLE IF EXISTS infrastructure_whitelist; 28 | CREATE TABLE infrastructure_whitelist () INHERITS (infrastructure); 29 | ALTER TABLE infrastructure_whitelist ADD PRIMARY KEY (id); 30 | 31 | DROP TABLE IF EXISTS infrastructure_scan; 32 | CREATE TABLE infrastructure_scan () INHERITS (infrastructure); 33 | ALTER TABLE infrastructure_scan ADD PRIMARY KEY (id); 34 | 35 | DROP TABLE IF EXISTS infrastructure_spam; 36 | CREATE TABLE infrastructure_spam () INHERITS (infrastructure); 37 | ALTER TABLE infrastructure_spam ADD PRIMARY KEY (id); 38 | 39 | DROP TABLE IF EXISTS infrastructure_spamvertising; 40 | CREATE TABLE infrastructure_spamvertising () INHERITS (infrastructure); 41 | ALTER TABLE infrastructure_spamvertising ADD PRIMARY KEY (id); 42 | 43 | DROP TABLE IF EXISTS infrastructure_suspicious; 44 | CREATE TABLE infrastructure_suspicious () INHERITS (infrastructure); 45 | ALTER TABLE infrastructure_suspicious ADD PRIMARY KEY (id); 46 | 47 | DROP TABLE IF EXISTS infrastructure_phishing; 48 | CREATE TABLE infrastructure_phishing () INHERITS (infrastructure); 49 | ALTER TABLE infrastructure_phishing ADD PRIMARY KEY (id); 50 | 51 | DROP TABLE IF EXISTS infrastructure_warez; 52 | CREATE TABLE infrastructure_warez () INHERITS (infrastructure); 53 | ALTER TABLE infrastructure_warez ADD PRIMARY KEY (id); 54 | 55 | DROP TABLE IF EXISTS infrastructure_passive; 56 | CREATE TABLE infrastructure_passive () INHERITS (infrastructure); 57 | ALTER TABLE infrastructure_passive ADD PRIMARY KEY (id); 58 | -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/malware.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | DROP TABLE IF EXISTS malware CASCADE; 3 | CREATE TABLE malware ( 4 | id BIGSERIAL PRIMARY KEY NOT NULL, 5 | uuid uuid NOT NULL, 6 | guid uuid, 7 | hash text, 8 | confidence REAL, 9 | reporttime timestamp with time zone DEFAULT NOW(), 10 | created timestamp with time zone DEFAULT NOW() 11 | ); 12 | 13 | CREATE TABLE malware_md5 () INHERITS (malware); 14 | ALTER TABLE malware_md5 ADD PRIMARY KEY (id); 15 | 16 | CREATE TABLE malware_sha1 () INHERITS (malware); 17 | ALTER TABLE malware_sha1 ADD PRIMARY KEY (id); 18 | 19 | CREATE TABLE malware_uuid () INHERITS (malware); 20 | ALTER TABLE malware_uuid ADD PRIMARY KEY (id); 21 | -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/search.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | DROP TABLE IF EXISTS search CASCADE; 3 | CREATE TABLE search ( 4 | id BIGSERIAL PRIMARY KEY NOT NULL, 5 | uuid uuid NOT NULL, 6 | guid uuid, 7 | term text, 8 | confidence REAL, 9 | reporttime timestamp with time zone DEFAULT NOW(), 10 | created timestamp with time zone DEFAULT NOW() 11 | ); 12 | 13 | -- upgrade from v1-FINAL 14 | -- ALTER TABLE search RENAME COLUMN hash TO term; 15 | -- ALTER TABLE search ALTER COLUMN term TYPE text; -------------------------------------------------------------------------------- /libcif-dbi/schemas/Pg/url.sql: -------------------------------------------------------------------------------- 1 | SET default_tablespace = 'index'; 2 | DROP TABLE IF EXISTS url CASCADE; 3 | CREATE TABLE url ( 4 | id BIGSERIAL PRIMARY KEY NOT NULL, 5 | uuid uuid NOT NULL, 6 | guid uuid, 7 | hash varchar(40), 8 | confidence REAL, 9 | reporttime timestamp with time zone DEFAULT NOW(), 10 | created timestamp with time zone DEFAULT NOW() 11 | ); 12 | 13 | CREATE TABLE url_botnet () INHERITS (url); 14 | ALTER TABLE url_botnet ADD PRIMARY KEY (id); 15 | 16 | CREATE TABLE url_malware () INHERITS (url); 17 | ALTER TABLE url_malware ADD PRIMARY KEY (id); 18 | 19 | CREATE TABLE url_phishing () INHERITS (url); 20 | ALTER TABLE url_phishing ADD PRIMARY KEY (id); 21 | 22 | CREATE TABLE url_suspicious () INHERITS (url); 23 | ALTER TABLE url_suspicious ADD PRIMARY KEY (id); 24 | 25 | CREATE TABLE url_spam () INHERITS (url); 26 | ALTER TABLE url_spam ADD PRIMARY KEY (id); 27 | 28 | CREATE TABLE url_spamvertising () INHERITS (url); 29 | ALTER TABLE url_spamvertising ADD PRIMARY KEY (id); 30 | 31 | CREATE TABLE url_whitelist () INHERITS (url); 32 | ALTER TABLE url_whitelist ADD PRIMARY KEY (id); -------------------------------------------------------------------------------- /libcif-dbi/schemas/pg_purge.sql: -------------------------------------------------------------------------------- 1 | TRUNCATE archive; 2 | TRUNCATE hash; 3 | TRUNCATE domain; 4 | TRUNCATE infrastructure; 5 | TRUNCATE url; 6 | TRUNCATE email; 7 | TRUNCATE malware; 8 | TRUNCATE search; 9 | TRUNCATE feed; 10 | -------------------------------------------------------------------------------- /libcif/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/libcif/AUTHORS -------------------------------------------------------------------------------- /libcif/ChangeLog: -------------------------------------------------------------------------------- 1 | see parent ChangeLog 2 | -------------------------------------------------------------------------------- /libcif/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = protocol 2 | EXTRA_DIST = lib sbin bin 3 | 4 | ME=libcif 5 | BUILD=$(shell git describe --tags) 6 | SNAPSHOT=$(ME)-$(BUILD) 7 | 8 | all: default 9 | 10 | default: 11 | @echo "Please read the README before beginning your installation." 12 | 13 | install: files-install 14 | 15 | files-install: libs-install bin-install 16 | 17 | depends: fixdeps 18 | 19 | fixdeps: 20 | $(PERL) ./sbin/test-dependencies --verbose --install --prefix $(prefix) 21 | 22 | testdeps: 23 | $(PERL) ./sbin/test-dependencies --verbose --prefix $(prefix) 24 | 25 | libs-install: 26 | [ -d $(libdir) ] || $(INSTALL) -m 0755 -d $(libdir) 27 | -( cd lib && find . -type d -print ) | while read dir ; do \ 28 | $(INSTALL) -m 0755 -d "$(libdir)/$$dir" ; \ 29 | done 30 | -( cd lib && find . -type f -print ) | while read file ; do \ 31 | $(INSTALL) -m 0644 "lib/$$file" "$(libdir)/$$file" ; \ 32 | done 33 | 34 | bin-install: 35 | [ -d $(bindir) ] || $(INSTALL) -m 0755 -d $(bindir) 36 | -( cd bin && find . -type f -print ) | while read file ; do \ 37 | $(INSTALL) -m 0755 "bin/$$file" "$(bindir)/$$file" ; \ 38 | done 39 | 40 | clean: 41 | rm Makefile 42 | rm config.log 43 | rm config.status 44 | 45 | realclean: 46 | clean 47 | rm Makefile.in 48 | rm configure 49 | rm aclocal.m4 50 | 51 | bindings: 52 | $(PERL) ./sbin/gen_protocol_bindings.pl 53 | 54 | reconf: 55 | autoreconf -vf 56 | 57 | snapshot: 58 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 59 | -------------------------------------------------------------------------------- /libcif/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/libcif/NEWS -------------------------------------------------------------------------------- /libcif/README: -------------------------------------------------------------------------------- 1 | Welcome 2 | ------- 3 | 4 | This is the core message protocol and client library for cif. 5 | 6 | Building and installation 7 | ------------------------- 8 | 9 | See the INSTALL file included with the distribution. 10 | 11 | Resources 12 | --------- 13 | 14 | Website: collectiveintel.org 15 | Repository: github.com/collectiveintel/cif-v1 16 | 17 | 18 | COPYRIGHT AND LICENCE 19 | --------------------- 20 | Copyright (C) 2013 Wes Young (wesyoung.me) 21 | Copyright (C) 2013 REN-ISAC and The Trustees of Indiana University (ren-isac.net) 22 | 23 | Free use of this software is granted under the terms of the GNU Lesser General 24 | Public License (LGPL) unless otherwise specifically stated. For details see the 25 | files `COPYING` included with the distribution. 26 | -------------------------------------------------------------------------------- /libcif/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.64]) 5 | AC_INIT(libcif, m4_esyscmd_s([git describe --tags]), [ci-framework@googlegroups.com]) 6 | AM_INIT_AUTOMAKE 7 | 8 | AC_CONFIG_SUBDIRS([protocol]) 9 | 10 | # Checks for programs. 11 | AC_ARG_VAR([PERL],[Perl interpreter command]) 12 | AC_PATH_PROG([PERL], [perl], [not found]) 13 | if test "$PERL" = 'not found'; then 14 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without perl]) 15 | fi 16 | 17 | dnl Defaults paths for installation 18 | AC_PREFIX_DEFAULT([/opt/cif]) 19 | 20 | AC_CONFIG_FILES([ 21 | Makefile 22 | ]) 23 | 24 | 25 | # Checks for libraries. 26 | 27 | # not needed yet 28 | ## Google Protocol Buffers 29 | #PKG_CHECK_MODULES([protobuf],[protobuf >= 2.4.1]) 30 | 31 | ## uuid 32 | #PKG_CHECK_MODULES([uuid],[uuid >= 2.17.0]) 33 | #this check breaks in CentOS 6.3 34 | # need to do some sort of conditional 35 | # http://www.flameeyes.eu/autotools-mythbuster/pkgconfig/pkg_check_modules.html 36 | # pkg-config --list-all | grep uuid 37 | # ossp-uuid OSSP uuid - Universally Unique Identifier (UUID) Library 38 | 39 | # Checks for header files. 40 | 41 | # Checks for typedefs, structures, and compiler characteristics. 42 | 43 | # Checks for library functions. 44 | 45 | AC_OUTPUT 46 | -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query; 2 | use base 'Class::Accessor'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Module::Pluggable require => 1, search_path => [__PACKAGE__]; 8 | use Digest::SHA qw/sha1_hex/; 9 | use CIF qw(is_uuid); 10 | 11 | my @plugins = __PACKAGE__->plugins(); 12 | 13 | __PACKAGE__->follow_best_practice(); 14 | __PACKAGE__->mk_accessors(qw( 15 | apikey limit confidence 16 | guid description feed 17 | )); 18 | 19 | sub new { 20 | my $class = shift; 21 | my $args = shift; 22 | 23 | my $self = {}; 24 | bless($self,$class); 25 | 26 | my ($err,$ret); 27 | foreach my $p (@plugins){ 28 | ($err,$ret) = $p->process($args); 29 | return($err) if($err); 30 | last if($ret); 31 | } 32 | 33 | $ret = \%$args unless($ret); 34 | $ret = [$ret] unless(ref($ret) eq 'ARRAY'); 35 | 36 | foreach my $qq (@{$ret}){ 37 | $qq->{'query'} = lc($qq->{'query'}); 38 | $qq->{'query'} = sha1_hex($qq->{'query'}) unless($qq->{'query'} =~ /^[a-f0-9]{40}$/ || is_uuid($qq->{'query'}) ); 39 | 40 | ## don't ask, its' all crap. 41 | $args->{'limit'} = $qq->{'limit'} if($qq->{'limit'}); 42 | $args->{'description'} = $qq->{'description'} if($qq->{'description'}); 43 | $args->{'feed'} = $qq->{'feed'} if($qq->{'feed'}); 44 | $args->{'confidence'} = (defined($qq->{'confidence'})) ? $qq->{'confidence'} : 0; 45 | 46 | $qq = MessageType::QueryStruct->new({ 47 | query => $qq->{'query'}, 48 | nolog => $qq->{'nolog'}, 49 | }); 50 | } 51 | 52 | my $msg = MessageType::QueryType->new({ 53 | apikey => $args->{'apikey'}, 54 | limit => $args->{'limit'}, 55 | confidence => $args->{'confidence'}, 56 | guid => $args->{'guid'}, 57 | description => $args->{'description'}, 58 | query => $ret, 59 | 60 | ## TODO: clean this up... 61 | feed => $args->{'feed'}, 62 | }); 63 | 64 | return (undef,$msg->encode()); 65 | } 66 | 67 | # skel 68 | sub process {} 69 | 70 | 1; -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query/Asn.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query::Asn; 2 | use base 'CIF::Client::Query'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | my $regex = qr/^(AS|as)?\d+$/; 8 | 9 | sub process { 10 | my $class = shift; 11 | my $args = shift; 12 | 13 | return unless($args->{'query'} =~ $regex); 14 | 15 | $args->{'query'} =~ s/^(as|AS)//g; 16 | $args->{'query'} = 'as'.$args->{'query'}; 17 | 18 | my $query = { 19 | description => 'search '.$args->{'query'}, 20 | %$args, 21 | }; 22 | 23 | return(undef,$query); 24 | } 25 | 26 | 1; 27 | -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query/Cc.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query::Cc; 2 | use base 'CIF::Client::Query'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use CIF qw/debug/; 8 | 9 | my $regex = qr/^[A-Za-z]{2}$/; 10 | 11 | sub process { 12 | my $class = shift; 13 | my $args = shift; 14 | 15 | return unless($args->{'query'} =~ $regex); 16 | debug('performing country code search...'); 17 | 18 | $args->{'query'} = uc($args->{'query'}); 19 | 20 | my $query = { 21 | description => 'search CC '.$args->{'query'}, 22 | %$args, 23 | }; 24 | 25 | return(undef,$query); 26 | } 27 | 28 | 1; 29 | -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query/Feed.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query::Feed; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | my $regex = qr/^([a-z]+\/[a-z]+([0-9])?)$/; 7 | 8 | sub process { 9 | my $class = shift; 10 | my $args = shift; 11 | 12 | return unless($args->{'query'} =~ $regex); 13 | 14 | my @bits = split(/\//,$args->{'query'}); 15 | $args->{'query'} = join(' ',reverse(@bits)).' feed'; 16 | $args->{'limit'} = 1; 17 | $args->{'description'} = 'search '.$args->{'query'}; 18 | $args->{'feed'} = 1; 19 | $args->{'confidence'} = defined($args->{'confidence'}) ? $args->{'confidence'} : 95; 20 | 21 | return(undef,$args); 22 | } 23 | 24 | 1; -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query/Ipv4.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query::Ipv4; 2 | use base 'CIF::Client::Query'; 3 | 4 | use strict; 5 | use warnings; 6 | 7 | use Net::Patricia; 8 | use Regexp::Common qw/net/; 9 | use Regexp::Common::net::CIDR; 10 | 11 | sub process { 12 | my $class = shift; 13 | my $args = shift; 14 | 15 | return unless($args->{'query'} =~ /^$RE{'net'}{'IPv4'}$/ || $args->{'query'} =~ /^$RE{'net'}{'CIDR'}{'IPv4'}$/); 16 | $args->{'query'} = normalize_address($args->{'query'}); 17 | 18 | my $pt = $args->{'pt'}; 19 | $pt->add_string($args->{'query'}); 20 | 21 | my @array = split(/\./,$args->{'query'}); 22 | my $queries; 23 | for($args->{'query'}){ 24 | if(/^$RE{'net'}{'IPv4'}$/){ 25 | $queries = [ 26 | { query => $args->{'query'}, nolog => $args->{'nolog'}, confidence => $args->{'confidence'} }, 27 | { query => $array[0].'.'.$array[1].'.'.$array[2].'.0/24', nolog => 1, confidence => $args->{'confidence'} }, 28 | { query => $array[0].'.'.$array[1].'.0.0/16', nolog => 1, confidence => $args->{'confidence'} }, 29 | { query => $array[0].'.0.0.0/8', nolog => 1, confidence => $args->{'confidence'} }, 30 | ]; 31 | } 32 | if(/^$RE{'net'}{'CIDR'}{'IPv4'}{-keep}$/){ 33 | my $addr = $1; 34 | my $mask = $2; 35 | return 'mask too low; minimum value is 8' unless($mask > 7); 36 | 37 | for($mask){ 38 | if($_ > 8){ 39 | push(@$queries, { query => $array[0].'.0.0.0/8', nolog => 1, confidence => $args->{'confidence'} }); 40 | } 41 | if($_ > 16){ 42 | push(@$queries, { query => $array[0].'.'.$array[1].'.0.0/16', nolog => 1, confidence => $args->{'confidence'} }); 43 | } 44 | if($_ > 24){ 45 | push(@$queries, { query => $array[0].'.'.$array[1].'.'.$array[2].'.0/24', nolog => 1, confidence => $args->{'confidence'} }); 46 | } 47 | } 48 | push(@$queries, { query => $args->{'query'}, nolog => $args->{'nolog'}, confidence => $args->{'confidence'} }); 49 | } 50 | } 51 | return(undef,$queries); 52 | } 53 | 54 | sub normalize_address { 55 | my $addr = shift; 56 | 57 | my @bits = split(/\./,$addr); 58 | foreach(@bits){ 59 | if(/^0+\/(\d+)$/){ 60 | $_ = '0/'.$1; 61 | } else { 62 | next if(/^0$/); 63 | next unless(/^0{1,2}/); 64 | $_ =~ s/^0{1,2}//; 65 | } 66 | } 67 | return join('.',@bits); 68 | } 69 | 70 | 1; 71 | -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query/Rir.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query::Rir; 2 | use base 'CIF::Client::Query'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use CIF qw/debug/; 8 | 9 | my $regex = qr/^(afrinic|apnic|arin|lacnic|ripencc)$/; 10 | 11 | sub process { 12 | my $class = shift; 13 | my $args = shift; 14 | 15 | return unless(lc($args->{'query'}) =~ $regex); 16 | debug('performing rir query...'); 17 | 18 | $args->{'query'} = lc($args->{'query'}); 19 | 20 | my $query = { 21 | query => $args->{'query'}, 22 | description => 'search RIR '.uc($args->{'query'}), 23 | %$args, 24 | }; 25 | 26 | return(undef,$query); 27 | } 28 | 29 | 1; -------------------------------------------------------------------------------- /libcif/lib/CIF/Client/Query/Url.pm: -------------------------------------------------------------------------------- 1 | package CIF::Client::Query::Url; 2 | use base 'CIF::Client::Query'; 3 | 4 | use warnings; 5 | use strict; 6 | 7 | use URI::Escape; 8 | 9 | my $regex = qr/^http(s)?:\/\//; 10 | 11 | sub process { 12 | my $class = shift; 13 | my $args = shift; 14 | 15 | return unless(lc($args->{'query'}) =~ $regex); 16 | 17 | $args->{'query'} =~ s/\/$//g; 18 | $args->{'query'} = uri_escape($args->{'query'},'\x00-\x1f\x7f-\xff'); 19 | $args->{'query'} = lc($args->{'query'}); 20 | 21 | my $query = { 22 | description => 'search '.$args->{'query'}, 23 | %$args, 24 | }; 25 | 26 | return(undef,$query); 27 | } 28 | 29 | 1; 30 | -------------------------------------------------------------------------------- /libcif/protocol/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/libcif/protocol/AUTHORS -------------------------------------------------------------------------------- /libcif/protocol/ChangeLog: -------------------------------------------------------------------------------- 1 | Revision history for cif-protocol. 2 | 3 | 0.00.01 2012-05-10 4 | - initial release 5 | -------------------------------------------------------------------------------- /libcif/protocol/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = src 2 | 3 | ME=cif-protocol 4 | BUILD=$(shell git describe --tags) 5 | SNAPSHOT=$(ME)-$(BUILD) 6 | PROTOFILES=src/msg.proto src/feed.proto 7 | THRIFTFILES=src/msg.thrift src/feed.thrift 8 | 9 | all: default 10 | 11 | default: 12 | @echo "Please read the README before beginning your installation." 13 | 14 | install: 15 | @echo "Please read the README before beginning your installation." 16 | 17 | perl: pbperl 18 | pbperl: 19 | $(PERL) sbin/generate_perl_bindings.pl 20 | 21 | cpp: protobuf_cpp 22 | pbcpp: 23 | [ -d pb-cpp ] || mkdir pb-cpp 24 | $(PROTOC) --cpp_out=pb-cpp $(PROTOFILES) 25 | 26 | python: pbpython 27 | pbpython: 28 | [ -d pb-python ] || mkdir pb-python 29 | $(PROTOC) --python_out=pb-python $(PROTOFILES) 30 | 31 | tpython: 32 | @echo "not implemented yet..." 33 | 34 | tcpp: 35 | @echo "not implemented yet..." 36 | 37 | tperl: 38 | @echo "not implemented yet..." 39 | 40 | clean: 41 | rm Makefile 42 | rm config.log 43 | rm config.status 44 | 45 | realclean: clean 46 | rm Makefile.in 47 | rm configure 48 | rm aclocal.m4 49 | 50 | reconf: 51 | autoreconf -vf 52 | 53 | snapshot: 54 | git archive --format=tar --prefix=$(BUILD)/ $(BUILD) | gzip > $(SNAPSHOT).tar.gz 55 | -------------------------------------------------------------------------------- /libcif/protocol/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csirtgadgets/cif-v1/2140eabbb775ba72fd6acfa1adcd480146f813e6/libcif/protocol/NEWS -------------------------------------------------------------------------------- /libcif/protocol/README: -------------------------------------------------------------------------------- 1 | see README.md 2 | -------------------------------------------------------------------------------- /libcif/protocol/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.64]) 5 | AC_INIT(cif-protocol, m4_esyscmd_s([git describe --tags]), [ci-framework@googlegroups.com]) 6 | AM_INIT_AUTOMAKE 7 | 8 | # Checks for programs. 9 | AC_ARG_WITH([perl], [AS_HELP_STRING([--with-perl], 10 | [build perl bindings [default=no]])], 11 | [with_perl=$withval],[with_perl=yes]) 12 | 13 | if test "x$with_perl" != "xno"; then 14 | AC_ARG_VAR([PERL],[Perl interpreter command]) 15 | AC_PATH_PROG([PERL], [perl], [not found]) 16 | if test "$PERL" = 'not found'; then 17 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without perl]) 18 | fi 19 | fi 20 | 21 | AC_ARG_WITH([python], [AS_HELP_STRING([--with-python], 22 | [build python bindings [default=no]])], 23 | [with_python=$withval],[with_python=yes]) 24 | 25 | if test "x$with_python" != "xno"; then 26 | AC_ARG_VAR([PYTHON],[Python interpreter command]) 27 | AC_PATH_PROG([PYTHON], [python], [not found]) 28 | if test "$PYTHON" = 'not found'; then 29 | AC_MSG_ERROR([cannot use $PACKAGE_NAME without python]) 30 | fi 31 | fi 32 | 33 | #AC_PATH_PROG([PROTOC], [protoc], [not found]) 34 | # if test "$PROTOC" = 'not found'; then 35 | # AC_MSG_ERROR([cannot use $PACKAGE_NAME without protoc]) 36 | # fi 37 | 38 | AC_CONFIG_FILES([ 39 | Makefile 40 | ]) 41 | 42 | 43 | # Checks for libraries. 44 | 45 | #PKG_CHECK_MODULES([protobuf],[protobuf >= 2.4.1]) 46 | 47 | # Checks for header files. 48 | 49 | # Checks for typedefs, structures, and compiler characteristics. 50 | 51 | # Checks for library functions. 52 | 53 | AC_OUTPUT 54 | -------------------------------------------------------------------------------- /libcif/protocol/rebuild.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make -------------------------------------------------------------------------------- /libcif/protocol/src/feed.proto: -------------------------------------------------------------------------------- 1 | // TODO -- version, namespace, etc 2 | 3 | message FeedType { 4 | // pulled from RFC 5070 -- IODEF 5 | enum RestrictionType { 6 | restriction_type_default = 1; 7 | restriction_type_need_to_know = 2; 8 | restriction_type_private = 3; 9 | restriction_type_public = 4; 10 | } 11 | message MapType { 12 | required string key = 1; 13 | required string value = 2; 14 | } 15 | 16 | // TODO: this is a string for now 17 | // git describe will give off string references 18 | // based on the tags, may not be optimal 19 | required string version = 1; 20 | // guid should evolve to the ContactType from IODEF 21 | optional string guid = 2; 22 | 23 | // TODO: evolve this to Assessment Class 24 | optional int32 confidence = 3; 25 | 26 | // TODO: MLStringType? or even Assessment? 27 | // combine with confidence class 28 | required string description = 4; 29 | required string ReportTime = 5; 30 | optional RestrictionType restriction = 6; 31 | repeated MapType restriction_map = 7; 32 | repeated MapType group_map = 8; 33 | // TODO: research packed = true 34 | repeated bytes data = 9; 35 | 36 | optional string uuid = 10; 37 | optional int32 query_limit = 11; 38 | 39 | repeated MapType feeds_map = 12; 40 | } 41 | -------------------------------------------------------------------------------- /libcif/protocol/src/msg.proto: -------------------------------------------------------------------------------- 1 | // TODO -- version, namespace, etc 2 | // VERSION string should be set by make script (git describe or so) 3 | 4 | message MessageType { 5 | enum StatusType { 6 | SUCCESS = 1; 7 | FAILED = 2; 8 | UNAUTHORIZED = 3; 9 | } 10 | enum MsgType { 11 | QUERY = 1; 12 | SUBMISSION = 2; 13 | REPLY = 3; 14 | } 15 | message QueryStruct { 16 | required string query = 1; 17 | optional bool nolog = 2; 18 | } 19 | message QueryType { 20 | optional string apikey = 1; 21 | optional string guid = 2; 22 | optional int32 limit = 3; 23 | optional int32 confidence = 4; 24 | repeated QueryStruct query = 5; 25 | optional string description = 6; 26 | optional bool feed = 7 [ default = false ]; 27 | } 28 | message SubmissionType { 29 | optional string guid = 1; 30 | repeated bytes data = 2; 31 | } 32 | // TODO: this is a string for now 33 | // git describe will give off string references 34 | // based on the tags, may not be optimal 35 | required string version = 1; 36 | required MsgType type = 2; 37 | optional StatusType status = 3; 38 | 39 | optional string apikey = 4; 40 | 41 | // TODO: research [ packed = true ] 42 | // src/msg.proto:37:14: [packed = true] can only be specified for repeated primitive fields. 43 | repeated bytes data = 5; 44 | } 45 | -------------------------------------------------------------------------------- /libcif/rebuild.sh: -------------------------------------------------------------------------------- 1 | autoreconf -vf 2 | automake --add-missing 3 | ./configure 4 | make bindings 5 | make dist 6 | -------------------------------------------------------------------------------- /libcif/sbin/gen_protocol_bindings.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | use strict; 4 | 5 | use Google::ProtocolBuffers; 6 | use FindBin qw/$Bin/; 7 | 8 | my $f = $Bin.'/../protocol/src/msg.proto'; 9 | 10 | Google::ProtocolBuffers->parsefile($f, 11 | { 12 | generate_code => $Bin.'/../lib/CIF/Msg.pm', 13 | create_accessors => 1, 14 | follow_best_practice => 1, 15 | } 16 | ); 17 | 18 | $f = $Bin.'/../protocol/src/feed.proto'; 19 | 20 | unless(-e $Bin.'/../lib/CIF/Msg'){ 21 | system("mkdir $Bin/../lib/CIF/Msg"); 22 | } 23 | 24 | Google::ProtocolBuffers->parsefile($f, 25 | { 26 | generate_code => $Bin.'/../lib/CIF/Msg/Feed.pm', 27 | create_accessors => 1, 28 | follow_best_practice => 1, 29 | } 30 | ); 31 | 32 | # work-around till we fix: 33 | # https://rt.cpan.org/Ticket/Display.html?id=76641 34 | 35 | open(F,$Bin.'/../lib/CIF/Msg.pm') || die($!);; 36 | my @lines = ; 37 | close(F); 38 | open(F,'>',$Bin.'/../lib/CIF/Msg.pm'); 39 | no warnings; 40 | print F "package CIF::Msg;\n"; 41 | foreach (@lines){ 42 | print F $_; 43 | } 44 | close(F); 45 | 46 | open(F,$Bin.'/../lib/CIF/Msg/Feed.pm'); 47 | my @lines = ; 48 | close(F); 49 | open(F,'>',$Bin.'/../lib/CIF/Msg/Feed.pm'); 50 | no warnings; 51 | print F "package CIF::Msg::Feed;\n"; 52 | foreach (@lines){ 53 | print F $_; 54 | } 55 | close(F); 56 | -------------------------------------------------------------------------------- /rebuild.sh: -------------------------------------------------------------------------------- 1 | make clean 2 | rm *.tar.gz 3 | autoreconf -vfi 4 | ./configure 5 | make dist 6 | --------------------------------------------------------------------------------