├── .gitignore ├── README ├── Zapret.pm ├── req_template.xml ├── zapret.conf ├── zapret.pl ├── zapret.sql └── zapret_log.conf /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | 2 | Скрипт для формирования БД ресстра запрещенных сайтов. БД может использоваться для работы фильтра запрещённых сайтов https://github.com/max197616/extfilter. 3 | 4 | Данный скрипт основан на https://github.com/ircop/zapret/. Внесены следующие изменения: 5 | 6 | - добавлена поддержка IPv6 7 | - существенно увеличена скорость определения ip адресов хостов (за счет использования AnyEvent::DNS) 8 | - поддержка русскоязычных доменов 9 | - добавлена поддержка логирования в файл 10 | - поведение скрипта скорректировано в соответствии с последними требованиями (п.5-7) 11 | - оптимизированы некоторые алгоритмы 12 | 13 | 14 | Формат файла собственной блокировки ip/url/домена: 15 | 16 | - для блокировки по ip: ip://1.1.1.1 17 | - для блокировки по ipv6: ipv6://2a00:1148:db00:0:b0b0::1 18 | - для блокировки по url: http://example.com 19 | - для блокировки по домену: www.example.com 20 | -------------------------------------------------------------------------------- /Zapret.pm: -------------------------------------------------------------------------------- 1 | package Zapret; 2 | 3 | require Exporter; 4 | 5 | @ISA = qw/Exporter/; 6 | @EXPORT = qw/$RKN_DUMP_VERSION/; 7 | 8 | use utf8; 9 | use strict; 10 | use SOAP::Lite; 11 | use MIME::Base64; 12 | 13 | our $RKN_DUMP_VERSION = 2.3; 14 | my $VERSION='0.02'; 15 | 16 | sub new 17 | { 18 | my $class = shift; 19 | my $wsdl = shift || die("WSDL not defined!\n"); 20 | my $ns = shift || 'http://vigruzki.rkn.gov.ru/OperatorRequest/'; 21 | my $self={ 22 | service => SOAP::Lite->new(proxy => $wsdl, ns => $ns) 23 | }; 24 | bless $self, $class; 25 | return $self; 26 | } 27 | 28 | sub getLastDumpDateEx 29 | { 30 | my $this=shift; 31 | my $res = $this->{service}->call("getLastDumpDateEx"); 32 | die("getLastDumpDateEx: soap error: ".$res->faultcode().": ".$res->faultstring()."(".$res->faultdetail().")") if ($res->fault()); 33 | my $resp = $res->valueof("Body/getLastDumpDateExResponse"); 34 | die("getLastDumpDateEx: Response is empty!") if (!defined($resp)); 35 | return $resp; 36 | } 37 | 38 | sub sendRequest 39 | { 40 | my $this=shift; 41 | my $requestFile=shift; 42 | my $signatureFile=shift; 43 | open XMLREQ, $requestFile; 44 | my $xmlreq = do { local $/ = undef; ; }; 45 | close XMLREQ; 46 | open XMLREQSIG, $signatureFile; 47 | my $xmlreqsig = do { local $/ = undef; ; }; 48 | close XMLREQSIG; 49 | my $res = $this->{service}->call('sendRequest', 50 | SOAP::Data->name("requestFile" => $xmlreq)->type("base64Binary"), 51 | SOAP::Data->name("signatureFile" => $xmlreqsig)->type("base64Binary"), 52 | SOAP::Data->name("dumpFormatVersion" => $RKN_DUMP_VERSION)->type("string") 53 | ); 54 | die("sendRequest: soap error: ".$res->faultcode().": ".$res->faultstring()."(".$res->faultdetail().")") if($res->fault()); 55 | my $resp = $res->valueof("Body/sendRequestResponse"); 56 | die("sendRequest: Response is empty!") if (!defined($resp)); 57 | die("sendRequest: result error: ".$resp->{resultComment}) if ($resp->{result} ne 'true'); 58 | return $resp; 59 | } 60 | 61 | sub getResult 62 | { 63 | my $this=shift; 64 | my $code=shift; 65 | my $res; 66 | eval { 67 | $res = $this->{service}->call("getResult", SOAP::Data->name("code" => $code)); 68 | }; 69 | die("getResult: soap exception: ".$@) if ($@); 70 | die("getResult: soap error: ".$res->faultcode().": ".$res->faultstring()."(".$res->faultdetail().")") if ($res->fault()); 71 | my $resp = $res->valueof("Body/getResultResponse"); 72 | die("geResult: Response is empty!") if (!defined($resp)); 73 | return $resp; 74 | } 75 | 76 | 1; 77 | -------------------------------------------------------------------------------- /req_template.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/max197616/zapret/231b9938fa3e86dcce2aa8699b57af7f9691bad2/req_template.xml -------------------------------------------------------------------------------- /zapret.conf: -------------------------------------------------------------------------------- 1 | [DB] 2 | host = 'db.provider.com' 3 | user = 'db_user' 4 | password = 'db_password' 5 | name = 'db_name' 6 | 7 | [API] 8 | url = 'http://vigruzki.rkn.gov.ru/services/OperatorRequest/?wsdl' 9 | form_request = 1 10 | 11 | [PATH] 12 | req_file = "req.xml" 13 | sig_file = "req.xml.sig" 14 | template_file = "req_template.xml" 15 | #our_blacklist = /path/to/our_blacklist 16 | # Путь к каталогу, где будет архив полученных файлов. Архив будет создаваться в виде: 17 | # yyyy 18 | # yyyy-mm 19 | # yyyy-mm-dd 20 | # arch-yyyy-mm-dd-hh_mm_ss.zip 21 | #archive = /path/to/archive 22 | 23 | [NS] 24 | resolve = 1 25 | resolvers = 10.10.10.10, 8.8.8.8 26 | # резолвить ли хосты в ipv6 адреса 27 | ipv6_support = yes 28 | # сохранять ли уже отрезолвленные записи 29 | keep_resolved = true 30 | 31 | [MAIL] 32 | to = 'noc@provider.com', 'admin@provider.com' 33 | from = 'zapret@provider.com' 34 | server = 'mail.provider.com' 35 | port = 25 36 | auth = 1 37 | login = 'zapret@provider.com' 38 | password = 'zapret_mail_pw' 39 | excludes = 1 40 | new = 1 41 | new_ips = 1 42 | removed = 1 43 | removed_ips = 1 44 | # Mail if there is only subnet without domains/urls in content block: 45 | alone = 1 46 | -------------------------------------------------------------------------------- /zapret.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use utf8; 7 | use XML::LibXML::Reader; 8 | use DBI; 9 | use File::Basename 'dirname'; 10 | use File::Spec; 11 | use lib join '/',File::Spec->splitdir(dirname(__FILE__)); 12 | use Zapret; 13 | use Config::Simple; 14 | use File::Basename; 15 | use Getopt::Long; 16 | use Log::Log4perl; 17 | use Net::IP qw(:PROC); 18 | use Net::SMTP; 19 | use Email::MIME; 20 | use PerlIO::gzip; 21 | use POSIX qw(strftime); 22 | use MIME::Base64; 23 | use File::Path qw(make_path); 24 | use File::Copy; 25 | use Digest::MD5 qw (md5_hex); 26 | use Fcntl qw(LOCK_EX LOCK_NB); 27 | 28 | use Data::Dumper; 29 | use Devel::Size qw(size total_size); 30 | 31 | binmode(STDOUT,':utf8'); 32 | binmode(STDERR,':utf8'); 33 | 34 | use constant 35 | { 36 | URL_TABLE => "zap2_urls", 37 | URL_COL_NAME => "url", 38 | 39 | IP_TABLE => "zap2_ips", 40 | IP_COL_NAME => "ip", 41 | 42 | 43 | IP_ONLY_TABLE => "zap2_only_ips", 44 | IP_ONLY_COL_NAME => "ip", 45 | 46 | DOMAIN_TABLE => "zap2_domains", 47 | DOMAIN_COL_NAME => "domain", 48 | 49 | SUBNET_TABLE => "zap2_subnets", 50 | SUBNET_COL_NAME => "subnet", 51 | 52 | SUPPORTED_DUMP_VERSION => 2.4 53 | }; 54 | 55 | 56 | 57 | ######## Config ######### 58 | 59 | my $openssl_bin_path="/usr/local/gost-ssl-new/bin"; 60 | 61 | my $dir = File::Basename::dirname($0); 62 | my $Config = {}; 63 | 64 | my $config_file=$dir.'/zapret.conf'; 65 | my $force_load=''; 66 | my $log_file=$dir."/zapret_log.conf"; 67 | 68 | GetOptions("force_load" => \$force_load, 69 | "log=s" => \$log_file, 70 | "config=s" => \$config_file) or die "Error no command line arguments\n"; 71 | 72 | Config::Simple->import_from($config_file, $Config) or die "Can't open ".$config_file." for reading!\n"; 73 | 74 | Log::Log4perl::init( $log_file ); 75 | 76 | my $logger=Log::Log4perl->get_logger(); 77 | 78 | my $api_url = $Config->{'API.url'} || die "API.url not defined."; 79 | my $req_file = $Config->{'PATH.req_file'} || die "PATH.req_file not defined."; 80 | $req_file = $dir."/".$req_file; 81 | my $sig_file = $Config->{'PATH.sig_file'} || die "PATH.sig_file not defined."; 82 | $sig_file = $dir."/".$sig_file; 83 | my $template_file = $Config->{'PATH.template_file'} || die "PATH.template_file not defined."; 84 | $template_file = $dir."/".$template_file; 85 | my $archive_path = $Config->{'PATH.archive'} || ""; 86 | 87 | my $db_host = $Config->{'DB.host'} || die "DB.host not defined."; 88 | my $db_user = $Config->{'DB.user'} || die "DB.user not defined."; 89 | my $db_pass = $Config->{'DB.password'} || die "DB.password not defined."; 90 | my $db_name = $Config->{'DB.name'} || die "DB.name not defined."; 91 | 92 | my $soap = new Zapret($api_url); 93 | 94 | 95 | #my $mail_send = $Config->{'MAIL.send'} || 0; 96 | 97 | my $mails_to = $Config->{'MAIL.to'} || die "MAIL.to not defined."; 98 | my @mail_to; 99 | if(ref($mails_to) ne "ARRAY") 100 | { 101 | push(@mail_to, $mails_to); 102 | } else { 103 | @mail_to = @{$mails_to}; 104 | } 105 | my $smtp_auth = $Config->{'MAIL.auth'} || 0; 106 | my $smtp_from = $Config->{'MAIL.from'} || die "MAIL.from not defined."; 107 | my $smtp_host = $Config->{'MAIL.server'} || die "MAIL.server not defined."; 108 | my $smtp_port = $Config->{'MAIL.port'} || die "MAIL.port not defined."; 109 | my $smtp_login = $Config->{'MAIL.login'} || ""; 110 | my $smtp_password = $Config->{'MAIL.password'} || ""; 111 | 112 | my $mail_excludes = $Config->{'MAIL.excludes'} || 0; 113 | my $mail_new = $Config->{'MAIL.new'} || 0; 114 | my $mail_new_ips = $Config->{'MAIL.new_ips'} || 0; 115 | my $mail_removed = $Config->{'MAIL.removed'} || 0; 116 | my $mail_removed_ips = $Config->{'MAIL.removed_ips'} || 0; 117 | my $mail_alone = $Config->{'MAIL.alone'} || 0; 118 | my $mail_stat = $Config->{'MAIL.stat'} || 0; 119 | my $mail_max_entries = $Config->{'MAIL.max_entries'} || 150; 120 | my $mail_check_report = $Config->{'MAIL.check_report'} || 0; 121 | my $mail_nofresh_report = $Config->{'MAIL.nofresh_report'} || 0; 122 | my $mail_subject = $Config->{'MAIL.subject'} || "zapret update!"; 123 | 124 | my $form_request = $Config->{'API.form_request'} || 0; 125 | 126 | my $our_blacklist = $Config->{'PATH.our_blacklist'} || ""; 127 | 128 | my $tmp_path = $Config->{'PATH.tmp_path'} || "/tmp"; 129 | 130 | my $ldd_iterations = 0; 131 | my $check_iterations = 0; 132 | 133 | my $max_check_iterations = $Config->{'API.max_check_iterations'} || 3; 134 | my $max_download_interval = $Config->{'API.max_download_interval'} || 60*60; 135 | my $max_result_iterations = $Config->{'API.max_result_iterations'} || 10; 136 | my $get_result_sleep_interval = $Config->{'API.get_result_sleep_interval'} || 60; 137 | 138 | ######## End config ##### 139 | 140 | my $start_work_time = time(); 141 | 142 | my $DBH; 143 | my ($lastDumpDateOld, $lastAction, $lastCode, $lastResult, $lastDocVersion); 144 | 145 | 146 | dbConnect(); 147 | getParams(); 148 | 149 | my $MAILTEXT = ''; 150 | my $MAIL_ADDED = ''; 151 | my $MAIL_ADDED_IPS = ''; 152 | my $MAIL_REMOVED = ''; 153 | my $MAIL_REMOVED_IPS = ''; 154 | my $MAIL_EXCLUDES = ''; 155 | my $MAIL_ALONE = ''; 156 | 157 | 158 | my $deleted_old_domains=0; 159 | my $deleted_old_urls=0; 160 | my $deleted_old_ips=0; 161 | my $deleted_old_only_ips=0; 162 | my $deleted_old_subnets=0; 163 | my $deleted_old_records=0; 164 | my $added_ipv4_ips=0; 165 | my $added_ipv6_ips=0; 166 | my $added_only_ipv4_ips=0; 167 | my $added_only_ipv6_ips=0; 168 | my $added_domains=0; 169 | my $added_urls=0; 170 | my $added_subnets=0; 171 | my $added_records=0; 172 | 173 | 174 | my @mail_add_urls; # if (entries > $mail_max_entries), then count $skiped_mail_add_urls 175 | my @mail_del_urls; 176 | 177 | my @mail_add_ips; 178 | my @mail_del_ips; 179 | my @mail_add_only_ips; 180 | my @mail_del_only_ips; 181 | my @mail_add_domains; 182 | my @mail_del_domains; 183 | my @mail_add_subnets; 184 | my @mail_del_subnets; 185 | my @mail_add_contents; 186 | my @mail_del_contents; 187 | 188 | my $mail_add_url_skipped = 0; 189 | my $mail_del_url_skipped = 0; 190 | 191 | my $mail_add_ip_skipped = 0; 192 | my $mail_del_ip_skipped = 0; 193 | my $mail_add_only_ip_skipped = 0; 194 | my $mail_del_only_ip_skipped = 0; 195 | my $mail_add_domain_skipped = 0; 196 | my $mail_del_domain_skipped = 0; 197 | my $mail_add_subnet_skipped = 0; 198 | my $mail_del_subnet_skipped = 0; 199 | my $mail_add_content_skipped = 0; 200 | my $mail_del_content_skipped = 0; 201 | 202 | my %all_records; # $all_records{$ips->{decision_id}} = $ips->{id}; 203 | 204 | $logger->debug("Last dump date:\t".$lastDumpDateOld); 205 | $logger->debug("Last action:\t".$lastAction); 206 | $logger->debug("Last code:\t".$lastCode); 207 | $logger->debug("Last result:\t".$lastResult); 208 | 209 | ############################################################# 210 | 211 | my $start_time = time(); 212 | my $register_processed = 0; 213 | 214 | $logger->info("Starting RKN at ".$start_time); 215 | 216 | eval 217 | { 218 | flock(DATA,LOCK_EX|LOCK_NB) or die "This script ($0) is already running!"; 219 | if(checkDumpDate()) 220 | { 221 | sendRequest(); 222 | my $files = getDumpFile(getResult($lastCode)); 223 | getAllContent(); 224 | parseFiles($files); 225 | parseOurBlacklist($our_blacklist) if($our_blacklist); 226 | analyzeOldContent(); 227 | $register_processed = 1; 228 | setParam('lastActionDate', time); 229 | } else { 230 | parseOurBlacklist($our_blacklist) if($our_blacklist); 231 | } 232 | }; 233 | if($@) 234 | { 235 | $MAILTEXT .= "Error occured while working with registry: ".$@; 236 | $logger->error("Error occured while working with registry: ".$@); 237 | $mail_subject = "Zapret error!"; 238 | processMail(); 239 | exit 1; 240 | } 241 | 242 | processMail(); 243 | 244 | # статистика 245 | $logger->info("Check iterations: ".$check_iterations); 246 | if($ldd_iterations) 247 | { 248 | $logger->info("Registry processing time: ".(parseDuration(time()-$start_time))." (wait time: ".(parseDuration($ldd_iterations*$get_result_sleep_interval)).")"); 249 | $logger->info("Load iterations: ".$ldd_iterations); 250 | $logger->info("Added: domains: ".$added_domains.", urls: ".$added_urls.", IPv4 ips: ".$added_ipv4_ips.", IPv6 ips: ".$added_ipv6_ips." IPv4 only IPs: ".$added_only_ipv4_ips.", IPv6 only IPs: ".$added_only_ipv6_ips.", subnets: ".$added_subnets.", records: ".$added_records); 251 | $logger->info("Deleted: old domains: ".$deleted_old_domains.", old urls: ".$deleted_old_urls.", old ips: ".$deleted_old_ips.", old only ips: ".$deleted_old_only_ips.", old subnets: ".$deleted_old_subnets.", old records: ".$deleted_old_records); 252 | } 253 | $logger->info("Stopping RKN at ".(localtime())); 254 | 255 | exit 0; 256 | 257 | 258 | sub dbConnect 259 | { 260 | $DBH = DBI->connect_cached("DBI:mysql:database=".$db_name.";host=".$db_host, $db_user, $db_pass,{mysql_enable_utf8 => 1, RaiseError => 1}) or die DBI->errstr; 261 | $DBH->do("set names utf8"); 262 | } 263 | 264 | sub isSomeDone 265 | { 266 | return 1 if($added_domains || $added_urls || $added_ipv4_ips || $added_ipv6_ips || $added_only_ipv4_ips || $added_only_ipv6_ips || $added_subnets || $added_records); 267 | return 1 if($deleted_old_domains || $deleted_old_urls || $deleted_old_ips || $deleted_old_only_ips || $deleted_old_subnets || $deleted_old_records); 268 | return 0; 269 | } 270 | 271 | sub processMail 272 | { 273 | if($mail_stat && $register_processed && isSomeDone()) 274 | { 275 | $MAILTEXT .= "\n\nRegistry processing time: ".(parseDuration(time()-$start_time))." (wait time: ".(parseDuration($ldd_iterations*$get_result_sleep_interval)).")\n"; 276 | } 277 | if($mail_check_report && $check_iterations) 278 | { 279 | $MAILTEXT .= "\n\n--- Registry check statistics ---\n"; 280 | $MAILTEXT .= "Check iterations: ".$check_iterations."\n"; 281 | } 282 | if($mail_stat && isSomeDone()) 283 | { 284 | $MAILTEXT .= "\n\n--- Registry processing statistics ---\n"; 285 | $MAILTEXT .= "Load iterations: ".$ldd_iterations."\n"; 286 | $MAILTEXT .= "Added: domains: ".$added_domains.", urls: ".$added_urls.", IPv4 ips: ".$added_ipv4_ips.", IPv6 ips: ".$added_ipv6_ips." IPv4 only IPs: ".$added_only_ipv4_ips.", IPv6 only IPs: ".$added_only_ipv6_ips.", subnets: ".$added_subnets.", records: ".$added_records."\n"; 287 | $MAILTEXT .= "Deleted: old domains: ".$deleted_old_domains.", old urls: ".$deleted_old_urls.", old ips: ".$deleted_old_ips.", old only ips: ".$deleted_old_only_ips.", old subnets: ".$deleted_old_subnets.", old records: ".$deleted_old_records."\n"; 288 | } 289 | if($mail_new) 290 | { 291 | if(@mail_add_contents) 292 | { 293 | $MAILTEXT .= "\n\n--- Added contents ---\n"; 294 | foreach my $cont (@mail_add_contents) 295 | { 296 | $MAILTEXT .= "Content: ".$cont->{value}." for id: ".$cont->{id}."\n"; 297 | } 298 | if($mail_add_content_skipped > 0) 299 | { 300 | $MAILTEXT .= "... and additionaly ".$mail_add_content_skipped." contents\n"; 301 | } 302 | } 303 | if(@mail_add_urls) 304 | { 305 | $MAILTEXT .= "\n\n--- Added URLs ---\n"; 306 | foreach my $url (@mail_add_urls) 307 | { 308 | $MAILTEXT .= "URL: ".$url->{value}." for id: ".$url->{id}."\n"; 309 | } 310 | if($mail_add_url_skipped > 0) 311 | { 312 | $MAILTEXT .= "... and additionaly ".$mail_add_url_skipped." urls\n"; 313 | } 314 | } 315 | if(@mail_add_domains) 316 | { 317 | $MAILTEXT .= "\n\n--- Added domains ---\n"; 318 | foreach my $domain (@mail_add_domains) 319 | { 320 | $MAILTEXT .= "Domain: ".$domain->{value}." for id: ".$domain->{id}."\n"; 321 | } 322 | if($mail_add_domain_skipped > 0) 323 | { 324 | $MAILTEXT .= "... and additionaly ".$mail_add_domain_skipped." domains\n"; 325 | } 326 | 327 | } 328 | if(@mail_add_ips) 329 | { 330 | $MAILTEXT .= "\n\n--- Added IPs ---\n"; 331 | foreach my $ip (@mail_add_ips) 332 | { 333 | $MAILTEXT .= "IP: ".$ip->{value}." for id: ".$ip->{id}."\n"; 334 | } 335 | if($mail_add_ip_skipped > 0) 336 | { 337 | $MAILTEXT .= "... and additionaly ".$mail_add_ip_skipped." IPs\n"; 338 | } 339 | 340 | } 341 | 342 | if(@mail_add_only_ips) 343 | { 344 | $MAILTEXT .= "\n\n--- Added only IPs ---\n"; 345 | foreach my $ip (@mail_add_only_ips) 346 | { 347 | $MAILTEXT .= "Only IP: ".$ip->{value}." for id: ".$ip->{id}."\n"; 348 | } 349 | if($mail_add_only_ip_skipped > 0) 350 | { 351 | $MAILTEXT .= "... and additionaly ".$mail_add_only_ip_skipped." only IPs\n"; 352 | } 353 | } 354 | 355 | if(@mail_add_subnets) 356 | { 357 | $MAILTEXT .= "\n\n--- Added subnets ---\n"; 358 | foreach my $subnet (@mail_add_subnets) 359 | { 360 | $MAILTEXT .= "Subnet: ".$subnet->{value}." for id: ".$subnet->{id}."\n"; 361 | } 362 | if($mail_add_subnet_skipped > 0) 363 | { 364 | $MAILTEXT .= "... and additionaly ".$mail_add_subnet_skipped." subnets\n"; 365 | } 366 | } 367 | 368 | } 369 | if($mail_removed) 370 | { 371 | if(@mail_del_contents) 372 | { 373 | $MAILTEXT .= "\n\n--- Removed contents ---\n"; 374 | foreach my $cont (@mail_del_contents) 375 | { 376 | $MAILTEXT .= "Content: ".$cont->{value}." for id: ".$cont->{id}."\n"; 377 | } 378 | if($mail_del_content_skipped > 0) 379 | { 380 | $MAILTEXT .= "... and additionaly ".$mail_del_content_skipped." records\n"; 381 | } 382 | } 383 | if(@mail_del_urls) 384 | { 385 | $MAILTEXT .= "\n\n--- Removed URLs ---\n"; 386 | foreach my $url (@mail_del_urls) 387 | { 388 | $MAILTEXT .= "URL: ".$url->{value}." for id: ".$url->{id}."\n"; 389 | } 390 | if($mail_del_url_skipped > 0) 391 | { 392 | $MAILTEXT .= "... and additionaly ".$mail_del_url_skipped." urls\n"; 393 | } 394 | } 395 | if(@mail_del_ips) 396 | { 397 | $MAILTEXT .= "\n\n--- Removed IPs ---\n"; 398 | foreach my $ip (@mail_del_ips) 399 | { 400 | $MAILTEXT .= "IP: ".$ip->{value}." for id: ".$ip->{id}."\n"; 401 | } 402 | if($mail_del_ip_skipped > 0) 403 | { 404 | $MAILTEXT .= "... and additionaly ".$mail_del_ip_skipped." ips\n"; 405 | } 406 | } 407 | if(@mail_del_only_ips) 408 | { 409 | $MAILTEXT .= "\n\n--- Removed only IPs ---\n"; 410 | foreach my $ip (@mail_del_only_ips) 411 | { 412 | $MAILTEXT .= "Only IP: ".$ip->{value}." for id: ".$ip->{id}."\n"; 413 | } 414 | if($mail_del_only_ip_skipped > 0) 415 | { 416 | $MAILTEXT .= "... and additionaly ".$mail_del_only_ip_skipped." only ips\n"; 417 | } 418 | } 419 | if(@mail_del_domains) 420 | { 421 | $MAILTEXT .= "\n\n--- Removed domains ---\n"; 422 | foreach my $domain (@mail_del_domains) 423 | { 424 | $MAILTEXT .= "Domain: ".$domain->{value}." for id: ".$domain->{id}."\n"; 425 | } 426 | if($mail_del_domain_skipped > 0) 427 | { 428 | $MAILTEXT .= "... and additionaly ".$mail_del_domain_skipped." domains\n"; 429 | } 430 | } 431 | 432 | if(@mail_del_subnets) 433 | { 434 | $MAILTEXT .= "\n\n--- Removed subnets ---\n"; 435 | foreach my $subnet (@mail_del_subnets) 436 | { 437 | $MAILTEXT .= "Subnet: ".$subnet->{value}." for id: ".$subnet->{id}."\n"; 438 | } 439 | if($mail_del_subnet_skipped > 0) 440 | { 441 | $MAILTEXT .= "... and additionaly ".$mail_del_subnet_skipped." subnets\n"; 442 | } 443 | } 444 | } 445 | Mail($MAILTEXT, $mail_subject) if($MAILTEXT); 446 | } 447 | 448 | sub Mail 449 | { 450 | my $text = shift; 451 | my $subj = shift || "zapret update!"; 452 | foreach (@mail_to) 453 | { 454 | eval { 455 | my $to = $_; 456 | my $smtp = Net::SMTP->new($smtp_host.':'.$smtp_port, Debug => 0) or do { $logger->error( "Can't connect to the SMTP server: $!"); return; }; 457 | 458 | eval { 459 | require MIME::Base64; 460 | require Authen::SASL; 461 | } or do { $logger->error( "Need MIME::Base64 and Authen::SASL to do smtp auth."); return; }; 462 | 463 | 464 | if( $smtp_auth eq '1' ) 465 | { 466 | if( $smtp_login eq '' || $smtp_password eq '' ) 467 | { 468 | $logger->debug("ERROR! SMTP Auth is enabled, but no login and password defined!"); 469 | return; 470 | } 471 | $smtp->auth($smtp_login, $smtp_password) or do {$logger->error( "Can't auth on smtp server: $!"); return; }; 472 | } 473 | $smtp->mail( $smtp_from ); 474 | $smtp->recipient( $to ); 475 | my $email = Email::MIME->create( 476 | header_str => [ From => $smtp_from, To => $to, Subject => $subj], 477 | attributes => { 478 | content_type => "text/plain", 479 | charset => "UTF-8", 480 | encoding => "quoted-printable" 481 | }, 482 | body_str => $text 483 | ); 484 | $smtp->data(); 485 | $smtp->datasend($email->as_string()); 486 | $smtp->dataend(); 487 | $smtp->quit; 488 | }; 489 | $logger->error("Email send error: $@") if $@; 490 | } 491 | } 492 | 493 | 494 | sub mail_add_url 495 | { 496 | my $url = shift; 497 | my $id = shift; 498 | if(scalar @mail_add_urls > $mail_max_entries) 499 | { 500 | $mail_add_url_skipped++; 501 | } else { 502 | push(@mail_add_urls, { value => $url, id => $id}); 503 | } 504 | } 505 | 506 | sub mail_add_ip 507 | { 508 | my $ip = shift; 509 | my $id = shift; 510 | if(scalar @mail_add_ips > $mail_max_entries) 511 | { 512 | $mail_add_ip_skipped++; 513 | } else { 514 | push(@mail_add_ips, { value => $ip, id => $id}); 515 | } 516 | } 517 | 518 | sub mail_add_only_ip 519 | { 520 | my $ip = shift; 521 | my $id = shift; 522 | if(scalar @mail_add_only_ips > $mail_max_entries) 523 | { 524 | $mail_add_only_ip_skipped++; 525 | } else { 526 | push(@mail_add_only_ips, { value => $ip, id => $id}); 527 | } 528 | } 529 | 530 | sub mail_add_domain 531 | { 532 | my $domain = shift; 533 | my $id = shift; 534 | if(scalar @mail_add_domains > $mail_max_entries) 535 | { 536 | $mail_add_domain_skipped++; 537 | } else { 538 | push(@mail_add_domains, { value => $domain, id => $id}); 539 | } 540 | } 541 | 542 | sub mail_add_subnet 543 | { 544 | my $subnet = shift; 545 | my $id = shift; 546 | if(scalar @mail_add_subnets > $mail_max_entries) 547 | { 548 | $mail_add_subnet_skipped++; 549 | } else { 550 | push(@mail_add_subnets, { value => $subnet, id => $id}); 551 | } 552 | } 553 | 554 | sub mail_add_content 555 | { 556 | my $content = shift; 557 | if(scalar @mail_add_contents > $mail_max_entries) 558 | { 559 | $mail_add_content_skipped++; 560 | } else { 561 | push(@mail_add_contents, { value => "includeTime: ".$content->{includeTime}.", blockType: ".$content->{blockType}, id => $content->{id}}); 562 | } 563 | } 564 | 565 | sub mail_del_ip 566 | { 567 | my $ip = shift; 568 | my $id = shift; 569 | if(scalar @mail_del_ips > $mail_max_entries) 570 | { 571 | $mail_del_ip_skipped++; 572 | } else { 573 | push(@mail_del_ips, { value => $ip, id => $id }); 574 | } 575 | } 576 | 577 | sub mail_del_only_ip 578 | { 579 | my $ip = shift; 580 | my $id = shift; 581 | if(scalar @mail_del_only_ips > $mail_max_entries) 582 | { 583 | $mail_del_only_ip_skipped++; 584 | } else { 585 | push(@mail_del_only_ips, { value => $ip, id => $id }); 586 | } 587 | } 588 | 589 | sub mail_del_domain 590 | { 591 | my $domain = shift; 592 | my $id = shift; 593 | if(scalar @mail_del_domains > $mail_max_entries) 594 | { 595 | $mail_del_domain_skipped++; 596 | } else { 597 | push(@mail_del_domains, { value => $domain, id => $id}); 598 | } 599 | } 600 | 601 | sub mail_del_subnet 602 | { 603 | my $subnet = shift; 604 | my $id = shift; 605 | if(scalar @mail_del_subnets > $mail_max_entries) 606 | { 607 | $mail_del_subnet_skipped++; 608 | } else { 609 | push(@mail_del_subnets, { value => $subnet, id => $id}); 610 | } 611 | } 612 | 613 | sub mail_del_url 614 | { 615 | my $url = shift; 616 | my $id = shift; 617 | if(scalar @mail_del_urls > $mail_max_entries) 618 | { 619 | $mail_del_url_skipped++; 620 | } else { 621 | push(@mail_del_urls, { value => $url, id => $id }); 622 | } 623 | } 624 | 625 | sub mail_del_content 626 | { 627 | my $content = shift; 628 | if(scalar @mail_del_contents > $mail_max_entries) 629 | { 630 | $mail_del_content_skipped++; 631 | } else { 632 | push(@mail_del_contents, { value => "record id: ".$content->{id}, id => $content->{decision_id}}); 633 | } 634 | } 635 | 636 | 637 | sub get_ip 638 | { 639 | my $ip_address=shift; 640 | my $d_size=length($ip_address); 641 | my $result; 642 | if($d_size == 4) 643 | { 644 | $result=ip_bintoip(unpack("B*",$ip_address),4); 645 | } else { 646 | $result=ip_bintoip(unpack("B*",$ip_address),6); 647 | } 648 | return $result; 649 | } 650 | 651 | 652 | sub getData 653 | { 654 | my $record_id = shift; 655 | my $table = shift; 656 | my $name = shift; 657 | my @values; 658 | my $sth = $DBH->prepare("SELECT * FROM $table WHERE record_id = $record_id"); 659 | $sth->execute or die DBI->errstr; 660 | while(my $ips = $sth->fetchrow_hashref()) 661 | { 662 | my $value = $ips->{$name} || ""; 663 | if($value) 664 | { 665 | if($name =~ /ip/) 666 | { 667 | push(@values, {value => get_ip($value), id => $ips->{id}}); 668 | } else { 669 | push(@values, {value => $value, id => $ips->{id}}); 670 | } 671 | } 672 | } 673 | $sth->finish(); 674 | return @values; 675 | } 676 | 677 | sub getContentByID 678 | { 679 | my $id = shift; 680 | my %content; 681 | my $sth = $DBH->prepare("SELECT * FROM zap2_records WHERE decision_id = $id"); 682 | $sth->execute(); 683 | while(my $ips = $sth->fetchrow_hashref()) 684 | { 685 | my $record_id = $ips->{id}; 686 | my @domains = getData($record_id, DOMAIN_TABLE, DOMAIN_COL_NAME); 687 | if(@domains) 688 | { 689 | $content{domain} = \@domains; 690 | } 691 | my @urls = getData($record_id, URL_TABLE, URL_COL_NAME); 692 | if(@urls) 693 | { 694 | $content{url} = \@urls; 695 | } 696 | my @subnets = getData($record_id, SUBNET_TABLE, SUBNET_COL_NAME); 697 | if(@subnets) 698 | { 699 | $content{ipSubnet} = \@subnets; 700 | } 701 | my @ips = getData($record_id, IP_TABLE, IP_COL_NAME); 702 | if(@ips) 703 | { 704 | $content{ip} = \@ips; 705 | } 706 | my @only_ips = getData($record_id, IP_ONLY_TABLE, IP_ONLY_COL_NAME); 707 | if(@only_ips) 708 | { 709 | $content{only_ip} = \@only_ips; 710 | } 711 | $content{hash} = $ips->{hash} || undef; 712 | $content{id} = $ips->{id}; 713 | $content{decision_id} = $id; 714 | } 715 | $sth->finish(); 716 | return %content; 717 | } 718 | 719 | sub getAllContent 720 | { 721 | my $sth = $DBH->prepare("SELECT id, decision_id, hash FROM zap2_records"); 722 | $sth->execute(); 723 | while(my $ips = $sth->fetchrow_hashref()) 724 | { 725 | $all_records{$ips->{decision_id}} = { id => $ips->{id}, hash => (defined $ips->{hash} ? $ips->{hash} : undef) }; 726 | } 727 | $sth->finish(); 728 | } 729 | 730 | sub checkData 731 | { 732 | my $c_array = shift; 733 | my $db_array = shift; 734 | my %c_array = map { $_ => 1 } @$c_array; 735 | my @add_entries; 736 | 737 | foreach my $entry (@$db_array) 738 | { 739 | if(exists $c_array{$entry->{value}}) 740 | { 741 | delete $entry->{id}; 742 | delete $c_array{$entry->{value}}; 743 | } 744 | } 745 | @add_entries = keys %c_array; 746 | return @add_entries; 747 | } 748 | 749 | =pod 750 | sub checkData 751 | { 752 | my $c_array = shift; 753 | my $db_array = shift; 754 | my @add_entries; 755 | foreach my $val (@{$c_array}) 756 | { 757 | my $found_db = 0; 758 | foreach my $entry (@{$db_array}) 759 | { 760 | if($val eq $entry->{value}) 761 | { 762 | $found_db = 1; 763 | delete $entry->{id}; 764 | last; 765 | } 766 | } 767 | if(!$found_db) 768 | { 769 | push(@add_entries, $val); 770 | } 771 | } 772 | return @add_entries; 773 | } 774 | =cut 775 | 776 | sub insertEntry 777 | { 778 | my $table = shift; 779 | my $col_name = shift; 780 | my $record_id = shift; 781 | my $value = shift; 782 | my $sth = $DBH->prepare("INSERT INTO $table (record_id, $col_name) VALUES(?,?)"); 783 | $sth->bind_param(1, $record_id); 784 | $sth->bind_param(2, $value); 785 | $sth->execute(); 786 | } 787 | 788 | sub removeEntry 789 | { 790 | my $table = shift; 791 | my $id = shift; 792 | my $sth = $DBH->prepare("DELETE FROM $table WHERE id=?"); 793 | $sth->bind_param(1, $id ); 794 | $sth->execute(); 795 | } 796 | 797 | sub insertContent 798 | { 799 | my $content = shift; 800 | my $sth = $DBH->prepare("INSERT INTO zap2_records (decision_id,decision_date,decision_num,decision_org,include_time,entry_type,hash) VALUES(?,?,?,?,?,?,?)"); 801 | $sth->bind_param(1, $content->{id}); 802 | $sth->bind_param(2, $content->{decision}{date}); 803 | $sth->bind_param(3, $content->{decision}{number}); 804 | $sth->bind_param(4, $content->{decision}{org}); 805 | $sth->bind_param(5, $content->{includeTime}); 806 | $sth->bind_param(6, $content->{entryType}); 807 | $sth->bind_param(7, $content->{hash}); 808 | $sth->execute(); 809 | return $sth->{mysql_insertid}; 810 | } 811 | 812 | sub updateHash 813 | { 814 | my $content = shift; 815 | my $hash = shift; 816 | my $sth = $DBH->prepare("UPDATE zap2_records SET hash = ? WHERE id = ?"); 817 | $sth->bind_param(1, $hash); 818 | $sth->bind_param(2, $content->{id}); 819 | $sth->execute(); 820 | } 821 | 822 | sub removeOldURL 823 | { 824 | my $db_content = shift; 825 | foreach my $del_url (@{$db_content->{url}}) 826 | { 827 | if(exists $del_url->{id}) 828 | { 829 | $logger->debug("Removing URL ".$del_url->{value}." (id ".$del_url->{id}.")"); 830 | mail_del_url($del_url->{value}, $db_content->{decision_id}); 831 | removeEntry(URL_TABLE, $del_url->{id}); 832 | $deleted_old_urls++; 833 | } 834 | } 835 | } 836 | 837 | sub processURL 838 | { 839 | my $content = shift; 840 | my $db_content = shift; 841 | my $record_id = $db_content->{id}; 842 | if(@{$content->{url}{value}}) 843 | { 844 | my @urls = @{$content->{url}{value}}; 845 | my @add_urls = checkData(\@urls, $db_content->{url}); 846 | if(@add_urls) 847 | { 848 | foreach my $url(@add_urls) 849 | { 850 | insertEntry(URL_TABLE, URL_COL_NAME, $record_id, $url); 851 | mail_add_url($url, $db_content->{decision_id}); 852 | # $MAIL_ADDED .= "Added new URL: ".$url." for $db_content{decision_id} \n"; 853 | $logger->debug("Added new URL: ".$url); 854 | $added_urls++; 855 | } 856 | } 857 | # print "in the url add array: ", Dumper(\@add_urls), "\n"; 858 | } 859 | } 860 | 861 | sub removeOldIP 862 | { 863 | my $db_content = shift; 864 | foreach my $del_ip (@{$db_content->{ip}}) 865 | { 866 | if(exists $del_ip->{id}) 867 | { 868 | $logger->debug("Removing IP ".$del_ip->{value}." (id ".$del_ip->{id}.")"); 869 | mail_del_ip($del_ip->{value}, $db_content->{decision_id}); 870 | removeEntry(IP_TABLE, $del_ip->{id}); 871 | $deleted_old_ips++; 872 | } 873 | } 874 | } 875 | 876 | sub removeOldSubnet 877 | { 878 | my $db_content = shift; 879 | foreach my $del_subnet (@{$db_content->{ipSubnet}}) 880 | { 881 | if(exists $del_subnet->{id}) 882 | { 883 | $logger->debug("Removing subnet ".$del_subnet->{value}." (id ".$del_subnet->{id}.")"); 884 | mail_del_subnet($del_subnet->{value}, $db_content->{decision_id}); 885 | removeEntry(SUBNET_TABLE, $del_subnet->{id}); 886 | $deleted_old_subnets++; 887 | } 888 | } 889 | } 890 | 891 | sub removeOldDomain 892 | { 893 | my $db_content = shift; 894 | foreach my $del_domain (@{$db_content->{domain}}) 895 | { 896 | if(exists $del_domain->{id}) 897 | { 898 | $logger->debug("Removing Domain ".$del_domain->{value}." (id ".$del_domain->{id}.")"); 899 | mail_del_domain($del_domain->{value}, $db_content->{decision_id}); 900 | removeEntry(DOMAIN_TABLE, $del_domain->{id}); 901 | $deleted_old_domains++; 902 | } 903 | } 904 | } 905 | 906 | sub removeOldOnlyIP 907 | { 908 | my $db_content = shift; 909 | foreach my $del_ip (@{$db_content->{only_ip}}) 910 | { 911 | if(exists $del_ip->{id}) 912 | { 913 | $logger->debug("Removing Only IP ".$del_ip->{value}." (id ".$del_ip->{id}.")"); 914 | mail_del_only_ip($del_ip->{value}, $db_content->{decision_id}); 915 | removeEntry(IP_ONLY_TABLE, $del_ip->{id}); 916 | $deleted_old_only_ips++; 917 | } 918 | } 919 | } 920 | 921 | sub removeContent 922 | { 923 | my $record_id = shift; 924 | my $sth = $DBH->prepare("DELETE FROM zap2_records WHERE id = ?"); 925 | $sth->bind_param(1, $record_id); 926 | $sth->execute(); 927 | my @tables = ( URL_TABLE, IP_TABLE, IP_ONLY_TABLE, DOMAIN_TABLE, SUBNET_TABLE ); 928 | foreach my $table (@tables) 929 | { 930 | $sth = $DBH->prepare("DELETE FROM $table WHERE record_id = ?"); 931 | $sth->bind_param(1, $record_id); 932 | $sth->execute(); 933 | } 934 | } 935 | 936 | sub processIP 937 | { 938 | my $content = shift; 939 | my $db_content = shift; 940 | my $record_id = $db_content->{id}; 941 | my @ips; 942 | push(@ips, @{$content->{ip}{value}}) if(defined $content->{ip}{value}); 943 | if(defined $content->{ipv6}{value}) 944 | { 945 | my @ipv6 = @{$content->{ipv6}{value}}; 946 | convertIPv6(\@ipv6); 947 | push(@ips, @ipv6); 948 | } 949 | if(@ips) 950 | { 951 | my @add_ips = checkData(\@ips, $db_content->{ip}); 952 | if(@add_ips) 953 | { 954 | foreach my $ip (@add_ips) 955 | { 956 | my $ipa = new Net::IP($ip); 957 | my $ip_packed=pack("B*",$ipa->binip()); 958 | insertEntry(IP_TABLE, IP_COL_NAME, $record_id, $ip_packed); 959 | mail_add_ip($ip, $db_content->{decision_id}); 960 | $logger->debug("Added new IP: ".$ip); 961 | if($ipa->version() == 4) 962 | { 963 | $added_ipv4_ips++; 964 | } else { 965 | $added_ipv6_ips++; 966 | } 967 | } 968 | } 969 | # print "in the ips add array: ", Dumper(\@add_ips), "\n"; 970 | } 971 | } 972 | 973 | sub processDomain 974 | { 975 | my $content = shift; 976 | my $db_content = shift; 977 | my $record_id = $db_content->{id}; 978 | if(@{$content->{domain}{value}}) 979 | { 980 | my @domains = @{$content->{domain}{value}}; 981 | my @add_domains = checkData(\@domains, $db_content->{domain}); 982 | if(@add_domains) 983 | { 984 | foreach my $domain (@add_domains) 985 | { 986 | insertEntry(DOMAIN_TABLE, DOMAIN_COL_NAME, $record_id, $domain); 987 | mail_add_domain($domain, $db_content->{decision_id}); 988 | $logger->debug("Added new Domain: ".$domain); 989 | $added_domains++; 990 | } 991 | } 992 | # print "in the domains add array: ", Dumper(\@add_domains), "\n"; 993 | } 994 | } 995 | 996 | sub processOnlyIP 997 | { 998 | my $content = shift; 999 | my $db_content = shift; 1000 | my $record_id = $db_content->{id}; 1001 | my @ips; 1002 | push(@ips, @{$content->{ip}{value}}) if(defined $content->{ip}{value}); 1003 | if(defined $content->{ipv6}{value}) 1004 | { 1005 | my @ipv6 = @{$content->{ipv6}{value}}; 1006 | convertIPv6(\@ipv6); 1007 | push(@ips, @ipv6); 1008 | } 1009 | if(@ips) 1010 | { 1011 | my @add_ips = checkData(\@ips, $db_content->{only_ip}); 1012 | if(@add_ips) 1013 | { 1014 | foreach my $ip (@add_ips) 1015 | { 1016 | my $ipa = new Net::IP($ip); 1017 | my $ip_packed=pack("B*",$ipa->binip()); 1018 | insertEntry(IP_ONLY_TABLE, IP_ONLY_COL_NAME, $record_id, $ip_packed); 1019 | mail_add_only_ip($ip, $db_content->{decision_id}); 1020 | $logger->debug("Added new only IP: ".$ip); 1021 | if($ipa->version() == 4) 1022 | { 1023 | $added_only_ipv4_ips++; 1024 | } else { 1025 | $added_only_ipv6_ips++; 1026 | } 1027 | } 1028 | } 1029 | # print "in the only ips add array: ", Dumper(\@add_ips), "\n"; 1030 | } 1031 | } 1032 | 1033 | sub processSubnet 1034 | { 1035 | my $content = shift; 1036 | my $db_content = shift; 1037 | my $record_id = $db_content->{id}; 1038 | my @subnets; 1039 | push(@subnets, @{$content->{ipSubnet}{value}}) if (defined $content->{ipSubnet}{value}); 1040 | push(@subnets, @{$content->{ipv6Subnet}{value}}) if (defined $content->{ipv6Subnet}{value}); 1041 | if(@subnets) 1042 | { 1043 | my @add_subnets = checkData(\@subnets, $db_content->{ipSubnet}); 1044 | if(@add_subnets) 1045 | { 1046 | foreach my $subnet (@add_subnets) 1047 | { 1048 | insertEntry(SUBNET_TABLE, SUBNET_COL_NAME, $record_id, $subnet); 1049 | mail_add_subnet($subnet, $db_content->{decision_id}); 1050 | $logger->debug("Added new subnet: ".$subnet); 1051 | $added_subnets++; 1052 | } 1053 | } 1054 | } 1055 | } 1056 | 1057 | sub processContent 1058 | { 1059 | my $content = shift; 1060 | my $db_content = shift; 1061 | if($content->{blockType} eq "default") 1062 | { 1063 | if(defined $content->{url}{value}) 1064 | { 1065 | processURL($content, $db_content); 1066 | } elsif(defined $content->{domain}{value}) 1067 | { 1068 | processDomain($content, $db_content); 1069 | } 1070 | if(!defined $content->{url}{value} && !defined $content->{domain}{value} && (defined $content->{ip}{value} || defined $content->{ipv6}{value})) 1071 | { 1072 | processOnlyIP($content, $db_content); 1073 | } else { 1074 | if(defined $content->{ip}{value} || defined $content->{ipv6}{value}) 1075 | { 1076 | processIP($content, $db_content); 1077 | } 1078 | } 1079 | if(defined $content->{ipSubnet}{value} || defined $content->{ipv6Subnet}{value}) 1080 | { 1081 | processSubnet($content, $db_content); 1082 | } 1083 | } elsif ($content->{blockType} eq "custom") 1084 | { 1085 | if(defined $content->{url}{value}) 1086 | { 1087 | processURL($content, $db_content); 1088 | } 1089 | if(defined $content->{domain}{value}) 1090 | { 1091 | processDomain($content, $db_content); 1092 | } 1093 | if(defined $content->{ip}{value} || defined $content->{ipv6}{value}) 1094 | { 1095 | processOnlyIP($content, $db_content); 1096 | } 1097 | if(defined $content->{ipSubnet}{value} || defined $content->{ipv6Subnet}{value}) 1098 | { 1099 | processSubnet($content, $db_content); 1100 | } 1101 | } else { 1102 | if($content->{blockType} eq "domain" || $content->{blockType} eq "domain-mask") 1103 | { 1104 | # block by domain 1105 | if(defined $content->{domain}{value}) 1106 | { 1107 | processDomain($content, $db_content); 1108 | } else { 1109 | $logger->error("Not found domain node for the entry $content->{id}, but blockType is domain"); 1110 | } 1111 | if(defined $content->{ip}{value} || defined $content->{ipv6}{value}) 1112 | { 1113 | processIP($content, $db_content); 1114 | } 1115 | } elsif ($content->{blockType} eq "ip") 1116 | { 1117 | # block by ip 1118 | if(defined $content->{ip}{value} || defined $content->{ipv6}{value}) 1119 | { 1120 | processOnlyIP($content, $db_content); 1121 | } 1122 | if(defined $content->{ipSubnet}{value} || $content->{ipv6Subnet}{value}) 1123 | { 1124 | processSubnet($content, $db_content); 1125 | } 1126 | if(!defined $content->{ip}{value} && !defined $content->{ipSubnet}{value} && !defined $content->{ipv6Subnet}{value} && !defined $content->{ipv6}{value}) 1127 | { 1128 | $logger->error("Not found ip node or subnet for the entry $content->{id}, but blockType is ip"); 1129 | } 1130 | } else { 1131 | $logger->error("Unknown blockType in content id ".$content->{id}); 1132 | } 1133 | } 1134 | if(defined $db_content->{url}) 1135 | { 1136 | removeOldURL($db_content); 1137 | } 1138 | if(defined $db_content->{domain}) 1139 | { 1140 | removeOldDomain($db_content); 1141 | } 1142 | if((!defined $content->{url}{value} && !defined $content->{domain}{value} && defined $db_content->{only_ip}) || ($content->{blockType} eq "ip" && defined $db_content->{only_ip})) 1143 | { 1144 | removeOldOnlyIP($db_content); 1145 | } elsif (defined $db_content->{ip}) 1146 | { 1147 | removeOldIP($db_content); 1148 | } 1149 | if(defined $db_content->{ipSubnet}) 1150 | { 1151 | removeOldSubnet($db_content); 1152 | } 1153 | } 1154 | 1155 | sub parseContent 1156 | { 1157 | my $content = shift; 1158 | $content->{blockType} = 'default' if(!exists $content->{blockType}); 1159 | if(exists $all_records{$content->{id}}) 1160 | { 1161 | if(!defined $all_records{$content->{id}}{hash} || $all_records{$content->{id}}{hash} ne $content->{hash}) 1162 | { 1163 | my %db_content = getContentByID($content->{id}); 1164 | processContent($content, \%db_content); 1165 | updateHash(\%db_content, $content->{hash}); 1166 | } 1167 | } else { 1168 | my %db_content; 1169 | $db_content{id} = insertContent($content); 1170 | $db_content{decision_id} = $content->{id}; 1171 | processContent($content, \%db_content); 1172 | mail_add_content($content); 1173 | $added_records++; 1174 | } 1175 | delete $all_records{$content->{id}} if(exists $all_records{$content->{id}}); 1176 | } 1177 | 1178 | sub analyzeOldContent 1179 | { 1180 | foreach my $record (keys %all_records) 1181 | { 1182 | my %content; 1183 | $content{decision_id} = $record; 1184 | $content{id} = $all_records{$record}{id}; 1185 | my %db_content = getContentByID($record); 1186 | if(defined $db_content{url}) 1187 | { 1188 | removeOldURL(\%db_content); 1189 | } 1190 | if(defined $db_content{domain}) 1191 | { 1192 | removeOldDomain(\%db_content); 1193 | } 1194 | if(defined $db_content{only_ip}) 1195 | { 1196 | removeOldOnlyIP(\%db_content); 1197 | } elsif (defined $db_content{ip}) 1198 | { 1199 | removeOldIP(\%db_content); 1200 | } 1201 | if(defined $db_content{ipSubnet}) 1202 | { 1203 | removeOldSubnet(\%db_content); 1204 | } 1205 | mail_del_content(\%content); 1206 | removeContent($all_records{$record}{id}); 1207 | $deleted_old_records++; 1208 | } 1209 | } 1210 | 1211 | sub parseDump 1212 | { 1213 | my $xml_file = shift; 1214 | my $reader = XML::LibXML::Reader->new(location => $xml_file) or die "Can't process xml file '$xml_file': ".$!."\n"; 1215 | my $do_content = 0; 1216 | my %register; 1217 | my $doc_pattern = XML::LibXML::Pattern->new('./content'); 1218 | my $z = 0; 1219 | while ($reader->read) 1220 | { 1221 | next unless $reader->nodeType() == XML_READER_TYPE_ELEMENT; 1222 | if($reader->name() eq 'reg:register') 1223 | { 1224 | my $reg = $reader->document()->documentElement(); 1225 | for my $attr ($reg->attributes) 1226 | { 1227 | $register{$attr->getName()} = $attr->getValue(); 1228 | } 1229 | } 1230 | if($z == 0) 1231 | { 1232 | if(!keys %register) 1233 | { 1234 | $reader->close(); 1235 | print "not found register!\n"; 1236 | return -2; 1237 | } 1238 | if(!defined $register{formatVersion}) 1239 | { 1240 | $reader->close(); 1241 | print "Not found attribute formatVersion!\n"; 1242 | return -3; 1243 | } 1244 | my $format_version = $register{formatVersion}; 1245 | $format_version += 0.0; 1246 | if($format_version > SUPPORTED_DUMP_VERSION) 1247 | { 1248 | $reader->close(); 1249 | print "Found unsupported format version $format_version\n"; 1250 | return -4; 1251 | } 1252 | } 1253 | next unless $reader->matchesPattern($doc_pattern); 1254 | my $xml = $reader->readOuterXml; 1255 | my $doc = XML::LibXML->load_xml(string => $xml); 1256 | my $element = $doc->documentElement(); 1257 | my %content; 1258 | for my $attr ($element->attributes) 1259 | { 1260 | $content{$attr->getName()} = $attr->getValue(); 1261 | } 1262 | foreach my $node ($element->childNodes()) 1263 | { 1264 | 1265 | my $node_name = $node->nodeName() || ""; 1266 | if($node_name ne "#text") 1267 | { 1268 | for my $attr ($node->attributes) 1269 | { 1270 | $content{$node_name}{$attr->getName()} = $attr->getValue(); 1271 | } 1272 | my $text = $node->textContent() || ""; 1273 | $text =~ s/^\s*//; 1274 | $text =~ s/\s*$//; 1275 | if($text) 1276 | { 1277 | push(@{$content{$node_name}{value}}, $text); 1278 | } 1279 | } 1280 | } 1281 | if(keys %content) 1282 | { 1283 | parseContent(\%content); 1284 | } 1285 | $reader->nextPatternMatch($doc_pattern); 1286 | $z++; 1287 | } 1288 | } 1289 | 1290 | sub getMD5Sum 1291 | { 1292 | my $file=shift; 1293 | open(my $MFILE, $file) or return ""; 1294 | binmode($MFILE); 1295 | my $hash=Digest::MD5->new->addfile(*$MFILE)->hexdigest; 1296 | close($MFILE); 1297 | return $hash; 1298 | } 1299 | 1300 | 1301 | sub rndStr{ join'', @_[ map{ rand @_ } 1 .. shift ] } 1302 | 1303 | sub parseOurBlacklist 1304 | { 1305 | my $filename = shift; 1306 | my %content; 1307 | $content{hash} = getMD5Sum($filename); 1308 | $content{id} = "0"; 1309 | $content{includeTime} = ''; 1310 | $content{blockType} = 'custom'; 1311 | if(!defined $all_records{$content{id}}) 1312 | { 1313 | my %db_content = getContentByID($content{id}); 1314 | if(keys %db_content) 1315 | { 1316 | return if($db_content{hash} eq $content{hash}); 1317 | $all_records{$content{id}}{hash} = $db_content{hash}; 1318 | } 1319 | } 1320 | $ldd_iterations++ if($ldd_iterations == 0); 1321 | open (my $fh, $filename) or die "Could not open file '$filename' $!"; 1322 | my $line = 1; 1323 | while (my $url = <$fh>) 1324 | { 1325 | chomp $url; 1326 | my $store_in_table; 1327 | if($url =~ /^http\:\/\// || $url =~ /^https\:\/\//) 1328 | { 1329 | push(@{$content{url}{value}}, $url); 1330 | } elsif ($url =~ /^ip\:\/\//) 1331 | { 1332 | my $ip = $url; 1333 | $ip =~ s/^ip\:\/\///; 1334 | push(@{$content{ip}{value}}, $ip); 1335 | } elsif ($url =~ /^ipv6\:\/\//) 1336 | { 1337 | my $ip = $url; 1338 | $ip =~ s/^ipv6\:\/\///; 1339 | push(@{$content{ipv6}{value}}, $ip); 1340 | } elsif ($url =~ /^subnet\:\/\//) 1341 | { 1342 | my $subnet = $url; 1343 | $subnet =~ s/^subnet\:\/\///; 1344 | push(@{$content{ipSubnet}{value}}, $subnet); 1345 | } else { 1346 | if($url =~ /^\*\./ || $url !~ /\//) 1347 | { 1348 | push (@{$content{domain}{value}}, $url); 1349 | } else { 1350 | push(@{$content{url}{value}}, "http://".$url); 1351 | } 1352 | } 1353 | $line++; 1354 | } 1355 | close $fh; 1356 | parseContent(\%content); 1357 | } 1358 | 1359 | sub getLastDumpDate 1360 | { 1361 | my $result; 1362 | while(!keys %{$result} && $check_iterations < $max_check_iterations) 1363 | { 1364 | eval { 1365 | $result = $soap->getLastDumpDateEx(); 1366 | }; 1367 | if($@) 1368 | { 1369 | $logger->error("Error while getLastDumpDateEx: ".$@); 1370 | $logger->info("Retrying..."); 1371 | } 1372 | $check_iterations++; 1373 | } 1374 | if(!keys %{$result}) 1375 | { 1376 | if($check_iterations == $max_check_iterations) 1377 | { 1378 | $logger->fatal("Exceeded number of check iterations"); 1379 | die "Exceeded number of check iterations"; 1380 | } 1381 | $logger->fatal("Empty result of getLastDumpDateEx()"); 1382 | die "Empty result of getLastDumpDateEx()"; 1383 | } 1384 | return $result; 1385 | } 1386 | 1387 | 1388 | sub checkDumpDate 1389 | { 1390 | $logger->debug("Checking dump date..."); 1391 | my $lastDumpDateEx = getLastDumpDate(); 1392 | my $dump_version = $lastDumpDateEx->{dumpFormatVersion}; 1393 | $dump_version += 0.0; 1394 | if($dump_version > SUPPORTED_DUMP_VERSION) 1395 | { 1396 | die "Dump version mismatch. Supported ".SUPPORTED_DUMP_VERSION." is lower than in the dump $lastDumpDateEx->{dumpFormatVersion}\n"; 1397 | } 1398 | if(defined $lastDocVersion && $lastDumpDateEx->{docVersion} ne $lastDocVersion) 1399 | { 1400 | $MAILTEXT .= "Warning! Documentation changed from version $lastDocVersion to $lastDumpDateEx->{docVersion}\n"; 1401 | $logger->warn("Documentation changed from version $lastDocVersion to $lastDumpDateEx->{docVersion}"); 1402 | } 1403 | my $lastDumpDateUrgently = $lastDumpDateEx->{lastDumpDateUrgently} / 1000; 1404 | my $lastDumpDate = $lastDumpDateEx->{lastDumpDate} / 1000; 1405 | $logger->debug("RKN last dump date: ".$lastDumpDateUrgently); 1406 | 1407 | if(!defined $lastDumpDateOld || $force_load || $lastDumpDateUrgently > $lastDumpDateOld || (time()-$lastDumpDateOld) > $max_download_interval) 1408 | { 1409 | $logger->debug("lastDumpDateUrgently > prev. dump date. Working now."); 1410 | return 1; 1411 | } 1412 | $logger->info("Registry has not changed since the last download at ".(scalar localtime $lastDumpDateOld).". lastDumpDateUrgently in the registry is ".(scalar localtime $lastDumpDateUrgently)); 1413 | 1414 | $MAILTEXT .= "Registry has not changed since the last download at ".(scalar localtime $lastDumpDateOld).". lastDumpDateUrgently in the registry is ".(scalar localtime $lastDumpDateUrgently)."\n" if($mail_nofresh_report); 1415 | 1416 | return 0; 1417 | } 1418 | 1419 | sub getParams 1420 | { 1421 | my $sth = $DBH->prepare("SELECT param, value FROM zap2_settings"); 1422 | $sth->execute or die DBI->errstr; 1423 | while(my $ips = $sth->fetchrow_hashref()) 1424 | { 1425 | my $param=$ips->{param}; 1426 | my $value=$ips->{value}; 1427 | if($param eq 'lastDumpDate') 1428 | { 1429 | $lastDumpDateOld = $value; 1430 | } 1431 | if($param eq 'lastAction') 1432 | { 1433 | $lastAction = $value; 1434 | } 1435 | if($param eq 'lastCode') 1436 | { 1437 | $lastCode = $value; 1438 | } 1439 | if($param eq 'lastResult' ) 1440 | { 1441 | $lastResult = $value; 1442 | } 1443 | if($param eq 'lastDocVersion') 1444 | { 1445 | $lastDocVersion = $value; 1446 | } 1447 | } 1448 | $sth->finish(); 1449 | } 1450 | 1451 | sub getParam 1452 | { 1453 | my $param = shift; 1454 | my $sth = $DBH->prepare("SELECT param, value FROM zap2_settings WHERE param = ?"); 1455 | $sth->bind_param(1, $param); 1456 | $sth->execute or die DBI->errstr; 1457 | my $res; 1458 | while(my $ips = $sth->fetchrow_hashref()) 1459 | { 1460 | $res = $ips->{value}; 1461 | } 1462 | $sth->finish(); 1463 | return $res; 1464 | } 1465 | 1466 | sub setParam 1467 | { 1468 | my $param = shift; 1469 | my $value = shift; 1470 | my $sth = $DBH->prepare("INSERT INTO zap2_settings (param, value) VALUES(?, ?) ON DUPLICATE KEY UPDATE value = ?"); 1471 | $sth->bind_param(1, $param); 1472 | $sth->bind_param(2, $value); 1473 | $sth->bind_param(3, $value); 1474 | $sth->execute() or die DBI->errstr; 1475 | } 1476 | 1477 | sub getResult 1478 | { 1479 | my $code = shift; 1480 | my $result = 0; 1481 | my $res; 1482 | while(!$result && $ldd_iterations < $max_result_iterations) 1483 | { 1484 | sleep($get_result_sleep_interval); 1485 | $res = $soap->getResult($code); 1486 | $result = int($res->{resultCode}); 1487 | $ldd_iterations++; 1488 | } 1489 | if($result != 1) 1490 | { 1491 | die "Too many getResult iterations ($ldd_iterations). Error: $res->{resultComment}\n" if($ldd_iterations >= $max_result_iterations); 1492 | die "getResult error: $res->{resultComment}\n"; 1493 | } 1494 | my $dump_version = $res->{dumpFormatVersion}; 1495 | $dump_version += 0.0; 1496 | if($dump_version > SUPPORTED_DUMP_VERSION) 1497 | { 1498 | die "Dump version $res->{dumpFormatVersion} in the dump is higher than supported ".SUPPORTED_DUMP_VERSION."\n"; 1499 | } 1500 | return decode_base64($res->{registerZipArchive}); 1501 | } 1502 | 1503 | sub formRequest 1504 | { 1505 | my $now = time(); 1506 | my $tz = strftime("%z", localtime($now)); 1507 | $tz =~ s/(\d{2})(\d{2})/$1:$2/; 1508 | my $dt = strftime("%Y-%m-%dT%H:%M:%S", localtime($now)) . $tz; 1509 | 1510 | my $buf = ''; 1511 | my $new = ''; 1512 | open TMPL, "<", $template_file or die "Can't open ".$template_file." for reading!\n"; 1513 | while( ) { 1514 | my $line = $_; 1515 | $line =~ s/\{\{TIME\}\}/$dt/g; 1516 | $new .= $line; 1517 | } 1518 | close TMPL; 1519 | 1520 | open REQ, ">", $req_file; 1521 | print REQ $new; 1522 | close REQ; 1523 | 1524 | `$openssl_bin_path/openssl smime -sign -in $req_file -out $sig_file -binary -signer $dir/cert.pem -outform DER`; 1525 | } 1526 | 1527 | sub sendRequest 1528 | { 1529 | $logger->debug( "Sending request..."); 1530 | 1531 | formRequest() if($form_request == 1 ); 1532 | 1533 | my $res = $soap->sendRequest($req_file, $sig_file); 1534 | 1535 | if($res->{result} ne 'true') 1536 | { 1537 | die "Can't get request result: ".$res->{resultComment}."\n"; 1538 | } 1539 | $lastCode = $res->{code}; 1540 | setParam('lastCode', $lastCode); 1541 | setParam('lastAction', 'sendRequest'); 1542 | setParam('lastResult', 'send'); 1543 | } 1544 | 1545 | sub getDumpFile 1546 | { 1547 | my $data = shift; 1548 | unlink $dir.'/dump.xml'; 1549 | unlink $dir.'/dump.xml.sig'; 1550 | my $file = "arch.zip"; 1551 | unless(mkdir($tmp_path)) 1552 | { 1553 | if ($! != 17) 1554 | { 1555 | die("Can't create a temp directory: ".$!); 1556 | } 1557 | } 1558 | my $tm=time(); 1559 | if($archive_path) 1560 | { 1561 | $file = strftime "arch-%Y-%m-%d-%H_%M_%S.zip", localtime($tm); 1562 | } 1563 | 1564 | open F, '>'.$tmp_path."/".$file || die "Can't open $dir/$file for writing: ".$!; 1565 | binmode F; 1566 | print F $data; 1567 | close F; 1568 | `unzip -o $tmp_path/$file -d $tmp_path/`; 1569 | if($archive_path) 1570 | { 1571 | my $apath = strftime "$archive_path/%Y/%Y-%m/%Y-%m-%d", localtime($tm); 1572 | make_path($apath); 1573 | copy($tmp_path."/".$file,$apath."/".$file); 1574 | unlink $tmp_path."/".$file; 1575 | } 1576 | $logger->debug("Got result..."); 1577 | setParam('lasltAction', 'getResult'); 1578 | setParam('lastResult', 'got'); 1579 | setParam('lastDumpDate', time()); 1580 | my @files = glob($tmp_path."/*.xml"); 1581 | return \@files; 1582 | } 1583 | 1584 | sub parseFiles 1585 | { 1586 | my $files = shift; 1587 | foreach my $file (@{$files}) 1588 | { 1589 | parseDump($file); 1590 | } 1591 | } 1592 | 1593 | sub parseDuration 1594 | { 1595 | my $duration = int(shift || 0); 1596 | return sprintf("%02d:%02d:%02d", $duration/3600, $duration/60%60, $duration%60); 1597 | } 1598 | 1599 | sub convertIPv6 1600 | { 1601 | my $arr = shift; 1602 | foreach my $ip (@{$arr}) 1603 | { 1604 | my @octs = split(/\:/, $ip); 1605 | $ip =~ s/\:\:$/\:0/g if(scalar(@octs) > 6); 1606 | my $ipa = new Net::IP($ip); 1607 | $ip = $ipa->ip(); 1608 | } 1609 | } 1610 | 1611 | __END__ 1612 | -------------------------------------------------------------------------------- /zapret.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.1.73, for redhat-linux-gnu (x86_64) 2 | -- 3 | -- Host: localhost Database: rkn1 4 | -- ------------------------------------------------------ 5 | -- Server version 5.5.5-10.2.8-MariaDB 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `zap2_domains` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `zap2_domains`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `zap2_domains` ( 26 | `id` int(6) unsigned NOT NULL AUTO_INCREMENT, 27 | `date_add` timestamp NOT NULL DEFAULT current_timestamp(), 28 | `record_id` int(6) unsigned NOT NULL, 29 | `domain` varchar(255) DEFAULT NULL, 30 | `domain_fixed` varchar(60) DEFAULT NULL, 31 | PRIMARY KEY (`id`), 32 | KEY `record_id_idx` (`record_id`) 33 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 34 | /*!40101 SET character_set_client = @saved_cs_client */; 35 | 36 | -- 37 | -- Table structure for table `zap2_ex_domains` 38 | -- 39 | 40 | DROP TABLE IF EXISTS `zap2_ex_domains`; 41 | /*!40101 SET @saved_cs_client = @@character_set_client */; 42 | /*!40101 SET character_set_client = utf8 */; 43 | CREATE TABLE `zap2_ex_domains` ( 44 | `id` int(6) unsigned NOT NULL AUTO_INCREMENT, 45 | `domain` varchar(50) NOT NULL, 46 | PRIMARY KEY (`id`) 47 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 48 | /*!40101 SET character_set_client = @saved_cs_client */; 49 | 50 | -- 51 | -- Table structure for table `zap2_ex_ips` 52 | -- 53 | 54 | DROP TABLE IF EXISTS `zap2_ex_ips`; 55 | /*!40101 SET @saved_cs_client = @@character_set_client */; 56 | /*!40101 SET character_set_client = utf8 */; 57 | CREATE TABLE `zap2_ex_ips` ( 58 | `id` int(6) unsigned NOT NULL AUTO_INCREMENT, 59 | `ip` int(12) unsigned NOT NULL, 60 | PRIMARY KEY (`id`), 61 | KEY `ip` (`ip`) 62 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 63 | /*!40101 SET character_set_client = @saved_cs_client */; 64 | 65 | -- 66 | -- Table structure for table `zap2_ex_nets` 67 | -- 68 | 69 | DROP TABLE IF EXISTS `zap2_ex_nets`; 70 | /*!40101 SET @saved_cs_client = @@character_set_client */; 71 | /*!40101 SET character_set_client = utf8 */; 72 | CREATE TABLE `zap2_ex_nets` ( 73 | `id` int(6) unsigned NOT NULL AUTO_INCREMENT, 74 | `subnet` varchar(30) NOT NULL, 75 | PRIMARY KEY (`id`), 76 | KEY `subnet` (`subnet`) 77 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 78 | /*!40101 SET character_set_client = @saved_cs_client */; 79 | 80 | -- 81 | -- Table structure for table `zap2_ips` 82 | -- 83 | 84 | DROP TABLE IF EXISTS `zap2_ips`; 85 | /*!40101 SET @saved_cs_client = @@character_set_client */; 86 | /*!40101 SET character_set_client = utf8 */; 87 | CREATE TABLE `zap2_ips` ( 88 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 89 | `record_id` int(6) unsigned NOT NULL, 90 | `date_add` timestamp NOT NULL DEFAULT current_timestamp(), 91 | `ip` varbinary(16) DEFAULT NULL, 92 | `resolved` int(1) NOT NULL DEFAULT 0, 93 | PRIMARY KEY (`id`), 94 | KEY `record_id` (`record_id`) 95 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 96 | /*!40101 SET character_set_client = @saved_cs_client */; 97 | 98 | -- 99 | -- Table structure for table `zap2_only_ips` 100 | -- 101 | 102 | DROP TABLE IF EXISTS `zap2_only_ips`; 103 | /*!40101 SET @saved_cs_client = @@character_set_client */; 104 | /*!40101 SET character_set_client = utf8 */; 105 | CREATE TABLE `zap2_only_ips` ( 106 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 107 | `record_id` int(6) unsigned NOT NULL, 108 | `date_add` timestamp NOT NULL DEFAULT current_timestamp(), 109 | `ip` varbinary(16) DEFAULT NULL, 110 | PRIMARY KEY (`id`), 111 | KEY `record_id_idx` (`record_id`) 112 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 113 | /*!40101 SET character_set_client = @saved_cs_client */; 114 | 115 | -- 116 | -- Table structure for table `zap2_records` 117 | -- 118 | 119 | DROP TABLE IF EXISTS `zap2_records`; 120 | /*!40101 SET @saved_cs_client = @@character_set_client */; 121 | /*!40101 SET character_set_client = utf8 */; 122 | CREATE TABLE `zap2_records` ( 123 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 124 | `date_add` timestamp NOT NULL DEFAULT current_timestamp(), 125 | `decision_id` int(10) unsigned NOT NULL, 126 | `decision_date` varchar(50) DEFAULT NULL, 127 | `decision_num` text DEFAULT NULL, 128 | `decision_org` varchar(50) DEFAULT NULL, 129 | `include_time` varchar(50) DEFAULT NULL, 130 | `entry_type` int(3) unsigned DEFAULT NULL, 131 | `hash` varchar(60) DEFAULT NULL, 132 | KEY `id` (`id`), 133 | KEY `decision_id` (`decision_id`) 134 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 135 | /*!40101 SET character_set_client = @saved_cs_client */; 136 | 137 | -- 138 | -- Table structure for table `zap2_settings` 139 | -- 140 | 141 | DROP TABLE IF EXISTS `zap2_settings`; 142 | /*!40101 SET @saved_cs_client = @@character_set_client */; 143 | /*!40101 SET character_set_client = utf8 */; 144 | CREATE TABLE `zap2_settings` ( 145 | `param` varchar(255) NOT NULL, 146 | `value` longtext NOT NULL, 147 | UNIQUE KEY `param` (`param`) 148 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 149 | /*!40101 SET character_set_client = @saved_cs_client */; 150 | 151 | INSERT INTO `zap2_settings` (`param`, `value`) VALUES 152 | ('lastDumpDate', '1406930960'), 153 | ('lastAction', 'getResult'), 154 | ('lastResult', 'got'), 155 | ('lastCode', '25ff77c0d152d7544ba2f72a95cbff50'), 156 | ('lastActionDate', '1406929097'), 157 | ('lastDump', '\r\n\r\n\r\n