├── t ├── perlcriticrc ├── 00_compile.t ├── release-fixme.t ├── release-pod-syntax.t ├── release-unused-vars.t ├── author-critic.t ├── author-dependencies.t ├── Util.pm ├── release-kwalitee.t ├── release-kwalitee-extra.t └── author-pod-spell.t ├── Build.PL ├── .travis.yml ├── .gitignore ├── dist.ini ├── Changes ├── cpanfile ├── lib └── App │ └── redis_traffic_stats.pm ├── META.json ├── README.md ├── LICENSE └── script └── redis-traffic-stats /t/perlcriticrc: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /t/00_compile.t: -------------------------------------------------------------------------------- 1 | use strict; 2 | use Test::More; 3 | 4 | BEGIN { use_ok 'App::redis_traffic_stats' } 5 | 6 | done_testing; 7 | -------------------------------------------------------------------------------- /Build.PL: -------------------------------------------------------------------------------- 1 | # This Build.PL for App-redis_traffic_stats was generated by Dist::Zilla::Plugin::ModuleBuildTiny 0.015. 2 | use strict; 3 | use warnings; 4 | 5 | use 5.008005; 6 | use Module::Build::Tiny 0.034; 7 | Build_PL(); 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: perl 2 | perl: 3 | - "5.16" 4 | - "5.14" 5 | before_install: 6 | - cpanm -n git://github.com/kan/coveralls-perl.git 7 | script: yes | (perl Build.PL && ./Build && cover -test) 8 | after_success: 9 | - cover -report coveralls 10 | notifications: 11 | email: 12 | - hirose31+travis@gmail.com 13 | 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /App-redis_traffic_stats-* 2 | /.build 3 | /_build* 4 | /Build 5 | MYMETA.* 6 | !META.json 7 | /.prove 8 | /blib 9 | 10 | /carton.lock 11 | /.carton/ 12 | /local/ 13 | 14 | nytprof.out 15 | nytprof/ 16 | 17 | cover_db/ 18 | 19 | *.bak 20 | *.old 21 | *~ 22 | *.swp 23 | *.o 24 | *.obj 25 | 26 | junk/* 27 | private/* 28 | TODO.todo 29 | MEMO 30 | xt/9*_* 31 | -------------------------------------------------------------------------------- /dist.ini: -------------------------------------------------------------------------------- 1 | name = App-redis_traffic_stats 2 | author = HIROSE Masaaki 3 | license = Perl_5 4 | copyright_holder = HIROSE Masaaki 5 | copyright_year = 2014 6 | 7 | [@Filter] 8 | -bundle = @Milla 9 | -remove = PodSyntaxTests 10 | -remove = ReadmeAnyFromPod 11 | -remove = NextRelease 12 | 13 | [NextRelease] 14 | time_zone = UTC 15 | 16 | [MetaProvides::Package] 17 | -------------------------------------------------------------------------------- /t/release-fixme.t: -------------------------------------------------------------------------------- 1 | #!perl 2 | 3 | BEGIN { 4 | unless ($ENV{RELEASE_TESTING}) { 5 | require Test::More; 6 | Test::More::plan(skip_all => 'these tests are for release candidate testing'); 7 | } 8 | } 9 | 10 | 11 | # This test is generated by Dist::Zilla::Plugin::Test::Fixme 12 | use strict; 13 | use warnings; 14 | use Test::More; 15 | 16 | eval { require Test::Fixme; Test::Fixme->import(); }; 17 | plan(skip_all => "Test::Fixme not installed: $@; skipping") if $@; 18 | 19 | run_tests(); 20 | -------------------------------------------------------------------------------- /t/release-pod-syntax.t: -------------------------------------------------------------------------------- 1 | #!perl 2 | 3 | BEGIN { 4 | unless ($ENV{RELEASE_TESTING}) { 5 | require Test::More; 6 | Test::More::plan(skip_all => 'these tests are for release candidate testing'); 7 | } 8 | } 9 | 10 | # This file was automatically generated by Dist::Zilla::Plugin::PodSyntaxTests. 11 | use Test::More; 12 | eval { require Test::Pod; Test::Pod->VERSION(1.41); Test::Pod->import(); }; 13 | plan(skip_all => "Test::Pod not installed: $@; skipping") if $@; 14 | 15 | all_pod_files_ok(); 16 | -------------------------------------------------------------------------------- /t/release-unused-vars.t: -------------------------------------------------------------------------------- 1 | #!perl 2 | 3 | BEGIN { 4 | unless ($ENV{RELEASE_TESTING}) { 5 | require Test::More; 6 | Test::More::plan(skip_all => 'these tests are for release candidate testing'); 7 | } 8 | } 9 | 10 | 11 | use Test::More 0.96 tests => 1; 12 | eval { require Test::Vars }; 13 | 14 | SKIP: { 15 | skip 1 => 'Test::Vars required for testing for unused vars' 16 | if $@; 17 | Test::Vars->import; 18 | 19 | subtest 'unused vars' => sub { 20 | all_vars_ok(); 21 | }; 22 | }; 23 | -------------------------------------------------------------------------------- /t/author-critic.t: -------------------------------------------------------------------------------- 1 | #!perl 2 | 3 | BEGIN { 4 | unless ($ENV{AUTHOR_TESTING}) { 5 | require Test::More; 6 | Test::More::plan(skip_all => 'these tests are for testing by the author'); 7 | } 8 | } 9 | 10 | 11 | use strict; 12 | use warnings; 13 | 14 | use Test::More; 15 | use English qw(-no_match_vars); 16 | 17 | eval "use Test::Perl::Critic"; 18 | plan skip_all => 'Test::Perl::Critic required to criticise code' if $@; 19 | Test::Perl::Critic->import( -profile => "perlcritic.rc" ) if -e "perlcritic.rc"; 20 | all_critic_ok(); 21 | -------------------------------------------------------------------------------- /t/author-dependencies.t: -------------------------------------------------------------------------------- 1 | 2 | BEGIN { 3 | unless ($ENV{AUTHOR_TESTING}) { 4 | require Test::More; 5 | Test::More::plan(skip_all => 'these tests are for testing by the author'); 6 | } 7 | } 8 | 9 | use strict; 10 | use warnings; 11 | use Test::More; 12 | eval "use App::scan_prereqs_cpanfile"; 13 | 14 | plan skip_all => "App::scan_prereqs_cpanfile required for testing module dependencies" 15 | if $@; 16 | 17 | my $diff = `scan-prereqs-cpanfile --ignore junk --diff cpanfile`; 18 | is($diff, "", "diff cpanfile"); 19 | 20 | done_testing; 21 | -------------------------------------------------------------------------------- /t/Util.pm: -------------------------------------------------------------------------------- 1 | package t::Util; 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use base qw(Exporter); 7 | our @EXPORT = qw(p); 8 | 9 | use Data::Dumper; 10 | 11 | sub p($) { 12 | local $Data::Dumper::Indent = 1; 13 | local $Data::Dumper::Deepcopy = 1; 14 | local $Data::Dumper::Sortkeys = 1; 15 | local $Data::Dumper::Terse = 1; 16 | local $Data::Dumper::Useqq = 1; 17 | local $Data::Dumper::Quotekeys = 0; 18 | my $d = Dumper($_[0]); 19 | $d =~ s/\\x{([0-9a-z]+)}/chr(hex($1))/ge; 20 | print STDERR $d; 21 | } 22 | 23 | 1; 24 | 25 | __END__ 26 | -------------------------------------------------------------------------------- /t/release-kwalitee.t: -------------------------------------------------------------------------------- 1 | 2 | BEGIN { 3 | unless ($ENV{RELEASE_TESTING}) { 4 | require Test::More; 5 | Test::More::plan(skip_all => 'these tests are for release candidate testing'); 6 | } 7 | } 8 | 9 | # this test was generated with Dist::Zilla::Plugin::Test::Kwalitee 2.11 10 | use strict; 11 | use warnings; 12 | use Test::More 0.88; 13 | eval { require Test::Kwalitee; Test::Kwalitee->VERSION(1.21); Test::Kwalitee->import(qw(kwalitee_ok)); }; 14 | plan(skip_all => "Test::Kwalitee not installed: $@; skipping") if $@; 15 | 16 | kwalitee_ok(qw( -has_readme )); 17 | 18 | done_testing; 19 | -------------------------------------------------------------------------------- /Changes: -------------------------------------------------------------------------------- 1 | Revision history for App-redis_traffic_stats 2 | 3 | {{$NEXT}} 4 | [IMPROVEMENTS] 5 | - Add --key-normalize-regexp option (#8 by @fujiwara) 6 | 7 | 1.004 2015-10-27 10:27:59+00:00 UTC 8 | [IMPROVEMENTS] 9 | - Reduce slow commands (#2 by @chenzhe07) 10 | 11 | 1.003 2014-10-03 12:58:27 JST 12 | [IMPROVEMENTS] 13 | - Collect stats for clients that were already connected when the capture starts (#1) 14 | 15 | 1.001 2014-02-26 13:03:56 JST 16 | [PACKAGING] 17 | - Refine cpanfile 18 | 19 | 1.000 2014-02-25 22:16:58 JST 20 | [NEW FEATURES] 21 | - Initial version 22 | 23 | -------------------------------------------------------------------------------- /t/release-kwalitee-extra.t: -------------------------------------------------------------------------------- 1 | 2 | BEGIN { 3 | unless ($ENV{RELEASE_TESTING}) { 4 | require Test::More; 5 | Test::More::plan(skip_all => 'these tests are for release candidate testing'); 6 | } 7 | } 8 | 9 | use strict; 10 | use warnings; 11 | use Test::More; 12 | eval { require Test::Kwalitee::Extra; Test::Kwalitee::Extra->import(qw(:experimental !prereq_matches_use !build_prereq_matches_use !has_readme)); }; 13 | #eval { require Test::Kwalitee::Extra; Test::Kwalitee::Extra->import(qw(:experimental !prereq_matches_use !build_prereq_matches_use)); }; 14 | plan(skip_all => "Test::Kwalitee::Extra not installed: $@; skipping") if $@; 15 | 16 | #done_testing; 17 | -------------------------------------------------------------------------------- /t/author-pod-spell.t: -------------------------------------------------------------------------------- 1 | 2 | BEGIN { 3 | unless ($ENV{AUTHOR_TESTING}) { 4 | require Test::More; 5 | Test::More::plan(skip_all => 'these tests are for testing by the author'); 6 | } 7 | } 8 | 9 | use strict; 10 | use warnings; 11 | use Test::More; 12 | 13 | eval { require Test::Spelling; Test::Spelling->VERSION(0.12); Test::Spelling->import(); }; 14 | plan(skip_all => "Test::Spelling not installed: $@; skipping") if $@; 15 | 16 | eval { require Pod::Wordlist; Pod::Wordlist->import(); }; 17 | plan(skip_all => "Pod::Wordlist not installed: $@; skipping") if $@; 18 | 19 | 20 | add_stopwords(); 21 | all_pod_files_spelling_ok( qw( bin lib ) ); 22 | __DATA__ 23 | HIROSE 24 | Masaaki 25 | hirose31 26 | lib 27 | Redis 28 | -------------------------------------------------------------------------------- /cpanfile: -------------------------------------------------------------------------------- 1 | # -*- mode: cperl -*- 2 | 3 | requires 'perl', '5.008005'; 4 | 5 | requires 'Net::Pcap'; 6 | requires 'NetPacket::Ethernet'; 7 | requires 'NetPacket::IP'; 8 | requires 'NetPacket::TCP'; 9 | requires 'Data::Validator'; 10 | requires 'File::Temp'; 11 | requires 'Getopt::Long'; 12 | requires 'List::Util'; 13 | requires 'Pod::Usage'; 14 | 15 | requires 'Test::Fixme'; 16 | requires 'Test::More'; 17 | 18 | on configure => sub { 19 | requires 'Module::Build::Tiny', '0.039'; 20 | }; 21 | 22 | on develop => sub { 23 | requires 'App::scan_prereqs_cpanfile', '0.09'; 24 | requires 'Pod::Wordlist'; 25 | requires 'Test::Fixme'; 26 | requires 'Test::Kwalitee'; 27 | requires 'Test::Kwalitee::Extra'; 28 | requires 'Test::Spelling', '0.12'; 29 | requires 'Test::More', '0.96'; 30 | requires 'Test::Pod'; 31 | requires 'Test::Vars'; 32 | }; 33 | 34 | on test => sub { 35 | requires 'Test::More', '0.96'; 36 | }; 37 | -------------------------------------------------------------------------------- /lib/App/redis_traffic_stats.pm: -------------------------------------------------------------------------------- 1 | package App::redis_traffic_stats; 2 | 3 | use strict; 4 | use warnings; 5 | use 5.008_005; 6 | 7 | our $VERSION = '1.005'; 8 | 9 | 1; 10 | 11 | __END__ 12 | 13 | =encoding utf-8 14 | 15 | =begin html 16 | 17 | Build Status 18 | Coverage Status 19 | 20 | =end html 21 | 22 | =head1 NAME 23 | 24 | App::redis_traffic_stats - Redis query analyzer for counting, traffic stats by command 25 | 26 | =begin readme 27 | 28 | =head1 INSTALLATION 29 | 30 | To install this module, run the following commands: 31 | 32 | perl Build.PL 33 | ./Build 34 | ./Build test 35 | ./Build install 36 | 37 | =end readme 38 | 39 | =head1 DESCRIPTION 40 | 41 | See L. 42 | 43 | =head1 AUTHOR 44 | 45 | HIROSE Masaaki Ehirose31@gmail.comE 46 | 47 | =head1 REPOSITORY 48 | 49 | L 50 | 51 | git clone https://github.com/hirose31/redis-traffic-stats.git 52 | 53 | patches and collaborators are welcome. 54 | 55 | =head1 COPYRIGHT 56 | 57 | Copyright HIROSE Masaaki 58 | 59 | =head1 LICENSE 60 | 61 | This library is free software; you can redistribute it and/or modify 62 | it under the same terms as Perl itself. 63 | 64 | =cut 65 | -------------------------------------------------------------------------------- /META.json: -------------------------------------------------------------------------------- 1 | { 2 | "abstract" : "Redis query analyzer for counting, traffic stats by command", 3 | "author" : [ 4 | "HIROSE Masaaki " 5 | ], 6 | "dynamic_config" : 0, 7 | "generated_by" : "Dist::Milla version v1.0.17, Dist::Zilla version 6.009, CPAN::Meta::Converter version 2.143240", 8 | "license" : [ 9 | "perl_5" 10 | ], 11 | "meta-spec" : { 12 | "url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec", 13 | "version" : "2" 14 | }, 15 | "name" : "App-redis_traffic_stats", 16 | "no_index" : { 17 | "directory" : [ 18 | "eg", 19 | "examples", 20 | "inc", 21 | "share", 22 | "t", 23 | "xt" 24 | ] 25 | }, 26 | "prereqs" : { 27 | "configure" : { 28 | "requires" : { 29 | "Module::Build::Tiny" : "0.039" 30 | } 31 | }, 32 | "develop" : { 33 | "requires" : { 34 | "App::scan_prereqs_cpanfile" : "0.09", 35 | "Dist::Milla" : "v1.0.17", 36 | "Pod::Wordlist" : "0", 37 | "Test::Fixme" : "0", 38 | "Test::Kwalitee" : "0", 39 | "Test::Kwalitee::Extra" : "0", 40 | "Test::More" : "0.96", 41 | "Test::Pod" : "0", 42 | "Test::Spelling" : "0.12", 43 | "Test::Vars" : "0" 44 | } 45 | }, 46 | "runtime" : { 47 | "requires" : { 48 | "Data::Validator" : "0", 49 | "File::Temp" : "0", 50 | "Getopt::Long" : "0", 51 | "List::Util" : "0", 52 | "Net::Pcap" : "0", 53 | "NetPacket::Ethernet" : "0", 54 | "NetPacket::IP" : "0", 55 | "NetPacket::TCP" : "0", 56 | "Pod::Usage" : "0", 57 | "Test::Fixme" : "0", 58 | "Test::More" : "0.96", 59 | "perl" : "5.008005" 60 | } 61 | }, 62 | "test" : { 63 | "requires" : { 64 | "Test::More" : "0.96" 65 | } 66 | } 67 | }, 68 | "provides" : { 69 | "App::redis_traffic_stats" : { 70 | "file" : "lib/App/redis_traffic_stats.pm", 71 | "version" : "1.005" 72 | } 73 | }, 74 | "release_status" : "stable", 75 | "resources" : { 76 | "bugtracker" : { 77 | "web" : "https://github.com/hirose31/redis-traffic-stats/issues" 78 | }, 79 | "homepage" : "https://github.com/hirose31/redis-traffic-stats", 80 | "repository" : { 81 | "type" : "git", 82 | "url" : "https://github.com/hirose31/redis-traffic-stats.git", 83 | "web" : "https://github.com/hirose31/redis-traffic-stats" 84 | } 85 | }, 86 | "version" : "1.005", 87 | "x_contributors" : [ 88 | "chenzhe07 ", 89 | "FUJIWARA Shunichiro ", 90 | "Nick Tiligadas " 91 | ], 92 | "x_serialization_backend" : "JSON::XS version 3.02" 93 | } 94 | 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-traffic-stats 2 | 3 | redis-traffic-stats is a query analyzer for Redis. 4 | 5 | redis-traffic-stats shows following statistics: 6 | 7 | * Total network traffic and average of byte/sec 8 | * Total number of requests and average and peak of req/sec 9 | * Top commands of count, percentage, req/sec by count 10 | * Top commands of network traffic and byte/sec by amount of traffic 11 | * Top keys for each command show bytes, byte/sec, count, percentage, req/sec 12 | * Worst slow requests 13 | 14 | redis-traffic-stats uses not MONITOR Redis command but pcap file (```tcpdump -w```) as data source, so: 15 | 16 | * No performance degradation (See: http://redis.io/commands/monitor) 17 | * Can analyze amount of traffic (size of Redis command request/response) 18 | 19 | ## SAMPLE USAGE 20 | 21 | Capture TCP packets by tcpdump: 22 | 23 | # tcpdump -s 65535 tcp port 6379 -w redis.pcap -i eth0 24 | (wait for a while and stop by ^C) 25 | 26 | and run redis-traffic-stats: 27 | 28 | $ redis-traffic-stats -r redis.pcap 29 | 30 | or redis-traffic-stats can capture packets internally (using tcpdump): 31 | 32 | (Capture 5000 packets) 33 | # redis-traffic-stats -i eth0 -c 5000 34 | 35 | (Capture for 10 sec) 36 | # redis-traffic-stats -i eth0 -t 10 37 | 38 | The output looks as follows: 39 | 40 | ``` 41 | # redis-traffic-stats 42 | 43 | ## Summary 44 | 45 | * Duration: 46 | * 2014-02-24 16:34:10 - 2014-02-24 16:34:13 (3s) 47 | * Total Traffic: 48 | * 43311772 bytes (14437257.33 bytes/sec) 49 | * Total Requests: 50 | * 110109 requests (Avg 36703.00 req/sec, Peak 47170.00 req/sec) 51 | 52 | ## Top Commands 53 | 54 | ### By count 55 | Command | Count | Pct | Req/sec 56 | -----------------|-------:|-------:|---------: 57 | LRANGE | 39423 | 35.80 | 13141.00 58 | PING | 18367 | 16.68 | 6122.33 59 | LPUSH | 12553 | 11.40 | 4184.33 60 | ... 61 | 62 | ### By traffic 63 | Command | Bytes | Byte/sec 64 | -----------------|----------:|-------------: 65 | LRANGE | 43089000 | 14363000.00 66 | PING | 73468 | 24489.33 67 | LPUSH | 62765 | 20921.67 68 | ... 69 | 70 | ## Command Detail 71 | 72 | ### LRANGE 73 | ... 74 | 75 | ### LPUSH 76 | ... 77 | 78 | ### INCR 79 | Key | Bytes | Byte/sec | Count | Pct | Req/sec 80 | --------------------------|----------:|-------------:|-------:|-------:|---------: 81 | counter:rand:000000000061 | 327 | 109.00 | 109 | 1.28 | 36.33 82 | counter:rand:000000000054 | 315 | 105.00 | 105 | 1.23 | 35.00 83 | counter:rand:000000000096 | 309 | 103.00 | 103 | 1.21 | 34.33 84 | ... 85 | 86 | ### GET 87 | Key | Bytes | Byte/sec | Count | Pct | Req/sec 88 | --------------------------|----------:|-------------:|-------:|-------:|---------: 89 | foo:rand:000000000084 | 321 | 107.00 | 107 | 1.34 | 35.67 90 | foo:rand:000000000004 | 303 | 101.00 | 101 | 1.26 | 33.67 91 | foo:rand:000000000061 | 303 | 101.00 | 101 | 1.26 | 33.67 92 | ... 93 | 94 | ## Slow Commands 95 | 96 | Time | Command 97 | ------:|------------------------------------------------------------------------ 98 | 0.759 | LRANGE mylist 0 599 99 | 0.759 | LRANGE mylist 0 599 100 | 0.759 | LRANGE mylist 0 599 101 | ... 102 | ``` 103 | 104 | Yes! Output of redis-traffic-stats is GitHub Flavored Markdown! We can get more pretty visual by copy and paste to Gist (sample output: https://gist.github.com/hirose31/9207096) or something supports GFM. 105 | 106 | ## SYNOPSIS 107 | 108 | * Analyze existing pcap file ( __RECOMMENDED WAY__ ) 109 | ```redis-traffic-stats -r pcapfile [-n top_n] [-p port]``` 110 | Example: 111 | ``` 112 | # tcpdump -s 65535 tcp port 6379 -w redis.pcap -i eth0 113 | (wait for a while and stop by ^C) 114 | $ redis-traffic-stats -r redis.pcap 115 | ``` 116 | __NOTICE__: You __MUST NOT__ use ```-i any``` because Net::Pcap cannot parse that pcap data 117 | 118 | * Capture packets on demand and analyze it 119 | ```redis-traffic-stats [-n top_n] [-p port] [-i device] [-c count_capture] [-t time_capture]``` 120 | Example: 121 | ``` 122 | (Capture 5000 packets) 123 | # redis-traffic-stats -i eth0 -c 5000 124 | 125 | (Capture for 10 sec) 126 | # redis-traffic-stats -i eth0 -t 10 127 | ``` 128 | 129 | * Show detailed usage 130 | ```redis-traffic-stats -h | --help | -?``` 131 | 132 | 133 | ## OPTIONS 134 | 135 | * ```-r pcapfile:Str, --pcapfile pcapfile:Str``` 136 | Read existing pcap file instead of on demand packet capturing. 137 | ``` 138 | # tcpdump -s 65535 tcp port 6379 -w redis.pcap -i eth0 139 | ``` 140 | 141 | * ```-p port:Int```, ```--port port:Int``` 142 | Target port of analyze. Default is 6379. 143 | 144 | * ```-n top_n:Int```, ```--top top_n:Int``` 145 | Show top N keys in "Command Detail" section. Default is 10. 146 | 147 | * ```-i device:Str```, ```--device device:Str``` 148 | Network interface name used by -i option of tcpdump. Default is no -i option (lowest numbered interface excluding loopback device). 149 | This option has a point only in on demand packet capture mode. 150 | 151 | * ```-c count:Int```, ```--count count:Int``` 152 | tcpdump will exit after captured number of this option packets. Default is 10000. 153 | 154 | * ```-t time:Int```, ```--time time:Int``` 155 | tcpdump will exit after number of this option seconds. 156 | 157 | * ```-d```, ```--debug``` 158 | increase debug level. ```-d -d``` more verbosely. 159 | 160 | * ```-k```, ```--key-normalize-regexp``` 161 | Replace parts of key to '*' with regexp. 162 | 163 | ## INSTALLATION 164 | 165 | ### QUICK TRIAL for CentOS 6 166 | 167 | If you are on CentOS 6, you can try redis-traffic-stats downloading single binary file. 168 | 169 | ``` 170 | curl -o redis-traffic-stats https://dl.dropboxusercontent.com/u/9824/redis-traffic-stats/centos6 171 | chmod +x redis-traffic-stats 172 | ``` 173 | 174 | This binary file is just for trial. I __highly recommend__ to install following installation if you will use redis-traffic-stats on a daily basis. 175 | 176 | 177 | ### RECOMMENDED INSTALLATION 178 | 179 | Install from CPAN using ```cpanm``` command: 180 | 181 | ``` 182 | cpanm App::redis_traffic_stats 183 | ``` 184 | If you don't have ```cpanm``` command: 185 | 186 | ``` 187 | curl -L http://cpanmin.us | perl - App::redis_traffic_stats 188 | ``` 189 | 190 | 191 | or clone this repository and install: 192 | 193 | ``` 194 | git clone https://github.com/hirose31/redis-traffic-stats.git 195 | cd redis-traffic-stats/ 196 | perl Build.PL 197 | ./Build 198 | ./Build test 199 | ./Build install 200 | ``` 201 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is copyright (c) 1 by HIROSE Masaaki . 2 | 3 | This is free software; you can redistribute it and/or modify it under 4 | the same terms as the Perl 5 programming language system itself. 5 | 6 | Terms of the Perl programming language system itself 7 | 8 | a) the GNU General Public License as published by the Free 9 | Software Foundation; either version 1, or (at your option) any 10 | later version, or 11 | b) the "Artistic License" 12 | 13 | --- The GNU General Public License, Version 1, February 1989 --- 14 | 15 | This software is Copyright (c) 1 by HIROSE Masaaki . 16 | 17 | This is free software, licensed under: 18 | 19 | The GNU General Public License, Version 1, February 1989 20 | 21 | GNU GENERAL PUBLIC LICENSE 22 | Version 1, February 1989 23 | 24 | Copyright (C) 1989 Free Software Foundation, Inc. 25 | 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 26 | 27 | Everyone is permitted to copy and distribute verbatim copies 28 | of this license document, but changing it is not allowed. 29 | 30 | Preamble 31 | 32 | The license agreements of most software companies try to keep users 33 | at the mercy of those companies. By contrast, our General Public 34 | License is intended to guarantee your freedom to share and change free 35 | software--to make sure the software is free for all its users. The 36 | General Public License applies to the Free Software Foundation's 37 | software and to any other program whose authors commit to using it. 38 | You can use it for your programs, too. 39 | 40 | When we speak of free software, we are referring to freedom, not 41 | price. Specifically, the General Public License is designed to make 42 | sure that you have the freedom to give away or sell copies of free 43 | software, that you receive source code or can get it if you want it, 44 | that you can change the software or use pieces of it in new free 45 | programs; and that you know you can do these things. 46 | 47 | To protect your rights, we need to make restrictions that forbid 48 | anyone to deny you these rights or to ask you to surrender the rights. 49 | These restrictions translate to certain responsibilities for you if you 50 | distribute copies of the software, or if you modify it. 51 | 52 | For example, if you distribute copies of a such a program, whether 53 | gratis or for a fee, you must give the recipients all the rights that 54 | you have. You must make sure that they, too, receive or can get the 55 | source code. And you must tell them their rights. 56 | 57 | We protect your rights with two steps: (1) copyright the software, and 58 | (2) offer you this license which gives you legal permission to copy, 59 | distribute and/or modify the software. 60 | 61 | Also, for each author's protection and ours, we want to make certain 62 | that everyone understands that there is no warranty for this free 63 | software. If the software is modified by someone else and passed on, we 64 | want its recipients to know that what they have is not the original, so 65 | that any problems introduced by others will not reflect on the original 66 | authors' reputations. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | GNU GENERAL PUBLIC LICENSE 72 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 73 | 74 | 0. This License Agreement applies to any program or other work which 75 | contains a notice placed by the copyright holder saying it may be 76 | distributed under the terms of this General Public License. The 77 | "Program", below, refers to any such program or work, and a "work based 78 | on the Program" means either the Program or any work containing the 79 | Program or a portion of it, either verbatim or with modifications. Each 80 | licensee is addressed as "you". 81 | 82 | 1. You may copy and distribute verbatim copies of the Program's source 83 | code as you receive it, in any medium, provided that you conspicuously and 84 | appropriately publish on each copy an appropriate copyright notice and 85 | disclaimer of warranty; keep intact all the notices that refer to this 86 | General Public License and to the absence of any warranty; and give any 87 | other recipients of the Program a copy of this General Public License 88 | along with the Program. You may charge a fee for the physical act of 89 | transferring a copy. 90 | 91 | 2. You may modify your copy or copies of the Program or any portion of 92 | it, and copy and distribute such modifications under the terms of Paragraph 93 | 1 above, provided that you also do the following: 94 | 95 | a) cause the modified files to carry prominent notices stating that 96 | you changed the files and the date of any change; and 97 | 98 | b) cause the whole of any work that you distribute or publish, that 99 | in whole or in part contains the Program or any part thereof, either 100 | with or without modifications, to be licensed at no charge to all 101 | third parties under the terms of this General Public License (except 102 | that you may choose to grant warranty protection to some or all 103 | third parties, at your option). 104 | 105 | c) If the modified program normally reads commands interactively when 106 | run, you must cause it, when started running for such interactive use 107 | in the simplest and most usual way, to print or display an 108 | announcement including an appropriate copyright notice and a notice 109 | that there is no warranty (or else, saying that you provide a 110 | warranty) and that users may redistribute the program under these 111 | conditions, and telling the user how to view a copy of this General 112 | Public License. 113 | 114 | d) You may charge a fee for the physical act of transferring a 115 | copy, and you may at your option offer warranty protection in 116 | exchange for a fee. 117 | 118 | Mere aggregation of another independent work with the Program (or its 119 | derivative) on a volume of a storage or distribution medium does not bring 120 | the other work under the scope of these terms. 121 | 122 | 3. You may copy and distribute the Program (or a portion or derivative of 123 | it, under Paragraph 2) in object code or executable form under the terms of 124 | Paragraphs 1 and 2 above provided that you also do one of the following: 125 | 126 | a) accompany it with the complete corresponding machine-readable 127 | source code, which must be distributed under the terms of 128 | Paragraphs 1 and 2 above; or, 129 | 130 | b) accompany it with a written offer, valid for at least three 131 | years, to give any third party free (except for a nominal charge 132 | for the cost of distribution) a complete machine-readable copy of the 133 | corresponding source code, to be distributed under the terms of 134 | Paragraphs 1 and 2 above; or, 135 | 136 | c) accompany it with the information you received as to where the 137 | corresponding source code may be obtained. (This alternative is 138 | allowed only for noncommercial distribution and only if you 139 | received the program in object code or executable form alone.) 140 | 141 | Source code for a work means the preferred form of the work for making 142 | modifications to it. For an executable file, complete source code means 143 | all the source code for all modules it contains; but, as a special 144 | exception, it need not include source code for modules which are standard 145 | libraries that accompany the operating system on which the executable 146 | file runs, or for standard header files or definitions files that 147 | accompany that operating system. 148 | 149 | 4. You may not copy, modify, sublicense, distribute or transfer the 150 | Program except as expressly provided under this General Public License. 151 | Any attempt otherwise to copy, modify, sublicense, distribute or transfer 152 | the Program is void, and will automatically terminate your rights to use 153 | the Program under this License. However, parties who have received 154 | copies, or rights to use copies, from you under this General Public 155 | License will not have their licenses terminated so long as such parties 156 | remain in full compliance. 157 | 158 | 5. By copying, distributing or modifying the Program (or any work based 159 | on the Program) you indicate your acceptance of this license to do so, 160 | and all its terms and conditions. 161 | 162 | 6. Each time you redistribute the Program (or any work based on the 163 | Program), the recipient automatically receives a license from the original 164 | licensor to copy, distribute or modify the Program subject to these 165 | terms and conditions. You may not impose any further restrictions on the 166 | recipients' exercise of the rights granted herein. 167 | 168 | 7. The Free Software Foundation may publish revised and/or new versions 169 | of the General Public License from time to time. Such new versions will 170 | be similar in spirit to the present version, but may differ in detail to 171 | address new problems or concerns. 172 | 173 | Each version is given a distinguishing version number. If the Program 174 | specifies a version number of the license which applies to it and "any 175 | later version", you have the option of following the terms and conditions 176 | either of that version or of any later version published by the Free 177 | Software Foundation. If the Program does not specify a version number of 178 | the license, you may choose any version ever published by the Free Software 179 | Foundation. 180 | 181 | 8. If you wish to incorporate parts of the Program into other free 182 | programs whose distribution conditions are different, write to the author 183 | to ask for permission. For software which is copyrighted by the Free 184 | Software Foundation, write to the Free Software Foundation; we sometimes 185 | make exceptions for this. Our decision will be guided by the two goals 186 | of preserving the free status of all derivatives of our free software and 187 | of promoting the sharing and reuse of software generally. 188 | 189 | NO WARRANTY 190 | 191 | 9. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 192 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 193 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 194 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 195 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 196 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 197 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 198 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 199 | REPAIR OR CORRECTION. 200 | 201 | 10. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 202 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 203 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 204 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 205 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 206 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 207 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 208 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 209 | POSSIBILITY OF SUCH DAMAGES. 210 | 211 | END OF TERMS AND CONDITIONS 212 | 213 | Appendix: How to Apply These Terms to Your New Programs 214 | 215 | If you develop a new program, and you want it to be of the greatest 216 | possible use to humanity, the best way to achieve this is to make it 217 | free software which everyone can redistribute and change under these 218 | terms. 219 | 220 | To do so, attach the following notices to the program. It is safest to 221 | attach them to the start of each source file to most effectively convey 222 | the exclusion of warranty; and each file should have at least the 223 | "copyright" line and a pointer to where the full notice is found. 224 | 225 | 226 | Copyright (C) 19yy 227 | 228 | This program is free software; you can redistribute it and/or modify 229 | it under the terms of the GNU General Public License as published by 230 | the Free Software Foundation; either version 1, or (at your option) 231 | any later version. 232 | 233 | This program is distributed in the hope that it will be useful, 234 | but WITHOUT ANY WARRANTY; without even the implied warranty of 235 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 236 | GNU General Public License for more details. 237 | 238 | You should have received a copy of the GNU General Public License 239 | along with this program; if not, write to the Free Software 240 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA 241 | 242 | 243 | Also add information on how to contact you by electronic and paper mail. 244 | 245 | If the program is interactive, make it output a short notice like this 246 | when it starts in an interactive mode: 247 | 248 | Gnomovision version 69, Copyright (C) 19xx name of author 249 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 250 | This is free software, and you are welcome to redistribute it 251 | under certain conditions; type `show c' for details. 252 | 253 | The hypothetical commands `show w' and `show c' should show the 254 | appropriate parts of the General Public License. Of course, the 255 | commands you use may be called something other than `show w' and `show 256 | c'; they could even be mouse-clicks or menu items--whatever suits your 257 | program. 258 | 259 | You should also get your employer (if you work as a programmer) or your 260 | school, if any, to sign a "copyright disclaimer" for the program, if 261 | necessary. Here a sample; alter the names: 262 | 263 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 264 | program `Gnomovision' (a program to direct compilers to make passes 265 | at assemblers) written by James Hacker. 266 | 267 | , 1 April 1989 268 | Ty Coon, President of Vice 269 | 270 | That's all there is to it! 271 | 272 | 273 | --- The Artistic License 1.0 --- 274 | 275 | This software is Copyright (c) 1 by HIROSE Masaaki . 276 | 277 | This is free software, licensed under: 278 | 279 | The Artistic License 1.0 280 | 281 | The Artistic License 282 | 283 | Preamble 284 | 285 | The intent of this document is to state the conditions under which a Package 286 | may be copied, such that the Copyright Holder maintains some semblance of 287 | artistic control over the development of the package, while giving the users of 288 | the package the right to use and distribute the Package in a more-or-less 289 | customary fashion, plus the right to make reasonable modifications. 290 | 291 | Definitions: 292 | 293 | - "Package" refers to the collection of files distributed by the Copyright 294 | Holder, and derivatives of that collection of files created through 295 | textual modification. 296 | - "Standard Version" refers to such a Package if it has not been modified, 297 | or has been modified in accordance with the wishes of the Copyright 298 | Holder. 299 | - "Copyright Holder" is whoever is named in the copyright or copyrights for 300 | the package. 301 | - "You" is you, if you're thinking about copying or distributing this Package. 302 | - "Reasonable copying fee" is whatever you can justify on the basis of media 303 | cost, duplication charges, time of people involved, and so on. (You will 304 | not be required to justify it to the Copyright Holder, but only to the 305 | computing community at large as a market that must bear the fee.) 306 | - "Freely Available" means that no fee is charged for the item itself, though 307 | there may be fees involved in handling the item. It also means that 308 | recipients of the item may redistribute it under the same conditions they 309 | received it. 310 | 311 | 1. You may make and give away verbatim copies of the source form of the 312 | Standard Version of this Package without restriction, provided that you 313 | duplicate all of the original copyright notices and associated disclaimers. 314 | 315 | 2. You may apply bug fixes, portability fixes and other modifications derived 316 | from the Public Domain or from the Copyright Holder. A Package modified in such 317 | a way shall still be considered the Standard Version. 318 | 319 | 3. You may otherwise modify your copy of this Package in any way, provided that 320 | you insert a prominent notice in each changed file stating how and when you 321 | changed that file, and provided that you do at least ONE of the following: 322 | 323 | a) place your modifications in the Public Domain or otherwise make them 324 | Freely Available, such as by posting said modifications to Usenet or an 325 | equivalent medium, or placing the modifications on a major archive site 326 | such as ftp.uu.net, or by allowing the Copyright Holder to include your 327 | modifications in the Standard Version of the Package. 328 | 329 | b) use the modified Package only within your corporation or organization. 330 | 331 | c) rename any non-standard executables so the names do not conflict with 332 | standard executables, which must also be provided, and provide a separate 333 | manual page for each non-standard executable that clearly documents how it 334 | differs from the Standard Version. 335 | 336 | d) make other distribution arrangements with the Copyright Holder. 337 | 338 | 4. You may distribute the programs of this Package in object code or executable 339 | form, provided that you do at least ONE of the following: 340 | 341 | a) distribute a Standard Version of the executables and library files, 342 | together with instructions (in the manual page or equivalent) on where to 343 | get the Standard Version. 344 | 345 | b) accompany the distribution with the machine-readable source of the Package 346 | with your modifications. 347 | 348 | c) accompany any non-standard executables with their corresponding Standard 349 | Version executables, giving the non-standard executables non-standard 350 | names, and clearly documenting the differences in manual pages (or 351 | equivalent), together with instructions on where to get the Standard 352 | Version. 353 | 354 | d) make other distribution arrangements with the Copyright Holder. 355 | 356 | 5. You may charge a reasonable copying fee for any distribution of this 357 | Package. You may charge any fee you choose for support of this Package. You 358 | may not charge a fee for this Package itself. However, you may distribute this 359 | Package in aggregate with other (possibly commercial) programs as part of a 360 | larger (possibly commercial) software distribution provided that you do not 361 | advertise this Package as a product of your own. 362 | 363 | 6. The scripts and library files supplied as input to or produced as output 364 | from the programs of this Package do not automatically fall under the copyright 365 | of this Package, but belong to whomever generated them, and may be sold 366 | commercially, and may be aggregated with this Package. 367 | 368 | 7. C or perl subroutines supplied by you and linked into this Package shall not 369 | be considered part of this Package. 370 | 371 | 8. The name of the Copyright Holder may not be used to endorse or promote 372 | products derived from this software without specific prior written permission. 373 | 374 | 9. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED 375 | WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF 376 | MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. 377 | 378 | The End 379 | 380 | -------------------------------------------------------------------------------- /script/redis-traffic-stats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | use 5.008_005; 6 | 7 | our $VERSION = '1.000'; 8 | 9 | use Getopt::Long qw(:config posix_default no_ignore_case no_ignore_case_always); 10 | use Pod::Usage; 11 | use IO::File; 12 | STDOUT->autoflush(1); 13 | STDERR->autoflush(1); 14 | use Data::Dumper; 15 | $Data::Dumper::Indent = 1; 16 | $Data::Dumper::Deepcopy = 1; 17 | $Data::Dumper::Sortkeys = 1; 18 | $Data::Dumper::Terse = 1; 19 | $Data::Dumper::Useqq = 1; 20 | $Data::Dumper::Quotekeys = 0; 21 | BEGIN { 22 | sub p($) { ## no critic 23 | my $d = Dumper($_[0]); 24 | # $d =~ s/\\x{([0-9a-z]+)}/chr(hex($1))/ge; 25 | print STDERR $d; 26 | } 27 | } 28 | 29 | use Net::Pcap; 30 | use NetPacket::Ethernet; 31 | use NetPacket::IP; 32 | use NetPacket::TCP; 33 | use List::Util qw(max); 34 | use POSIX qw(strftime); 35 | use File::Temp qw(tempfile); 36 | 37 | my $Debug = 0; 38 | 39 | sub debug { 40 | return unless $Debug; 41 | my @m = @_; 42 | chomp @m; 43 | warn '[DEBUG] '.join(" ", @m); 44 | } 45 | 46 | MAIN: { 47 | my $opt = {}; 48 | GetOptions( 49 | $opt, 50 | 'port|p=i', 51 | 'pcapfile|r=s', 52 | 'top|n=i', 53 | 'device|i=s', 54 | 'count|c=i', 55 | 'time|t=i', 56 | 'key-normalize-regexp|k=s', 57 | 'debug|d+' => \$Debug, 58 | 'help|h|?' => sub { pod2usage(-verbose=>1) }, 59 | ) or pod2usage(); 60 | 61 | $opt->{port} ||= 6379; 62 | $opt->{top} ||= 10; 63 | $opt->{count} ||= 10000; 64 | 65 | debug("opt: ".Dumper($opt)); 66 | debug("port: $opt->{port}"); 67 | 68 | my $pcapfile; 69 | if (exists $opt->{pcapfile}) { 70 | $pcapfile = $opt->{pcapfile}; 71 | } else { 72 | progressln("capture packet by tcpdump"); 73 | (undef, $pcapfile) = tempfile(UNLINK => 1); 74 | # if $pid == 0, do child process; if $pid is undefined, fork failed. the do block in child process. 75 | my $pid = fork || do { 76 | my @cmd = ('tcpdump', '-s', 65535, 'tcp', 'port', $opt->{port}, '-w', $pcapfile,); 77 | if (exists $opt->{device}) { 78 | push @cmd, '-i', $opt->{device}; 79 | } 80 | if (exists $opt->{time}) { 81 | progressln(" Capture packets until after $opt->{time} sec or interrupted by ^C"); 82 | } else { 83 | push @cmd, '-c', $opt->{count}; 84 | progressln(" Capture packets until received $opt->{count} packets or interrupted by ^C"); 85 | } 86 | progressln(" ".join(" ", @cmd)); 87 | 88 | exec {$cmd[0]} @cmd; 89 | }; 90 | local $SIG{INT} = sub { 91 | # child pid which > 0 92 | kill 'TERM', $pid, 93 | }; 94 | if (exists $opt->{time}) { 95 | sleep $opt->{time}; 96 | kill 'TERM', $pid; 97 | } 98 | my $r = waitpid $pid, 0; 99 | if ($r < 0 || $? != 0) { 100 | die "tcpdump exit with ".($?>>8); 101 | } 102 | } 103 | progressln("pcapfile: $pcapfile"); 104 | my $err; 105 | my $pcap = Net::Pcap::open_offline($pcapfile, \$err) 106 | or die "open_offline: $err"; 107 | 108 | my $filter; 109 | pcap_compile($pcap, \$filter, "tcp port $opt->{port}", 1, 0); 110 | pcap_setfilter($pcap, $filter); 111 | 112 | progress("read pcap data and rebuild TCP stream "); 113 | my $stream = {}; 114 | my $count = 0; 115 | Net::Pcap::loop($pcap, -1, \&process_packet, { 116 | port => $opt->{port}, 117 | stream => $stream, 118 | count => \$count, 119 | }); 120 | progressln(" done ($count pkt)"); 121 | 122 | progress("process Redis protocol "); 123 | my $stats = process_redis($stream, $opt->{"key-normalize-regexp"}); 124 | progressln(" done"); 125 | debug("stats: ".Dumper($stats)); 126 | 127 | format_stats($opt->{top}, $stream, $stats); 128 | 129 | exit 0; 130 | } 131 | 132 | sub progress { 133 | my @m = @_; 134 | chomp @m; 135 | print STDERR @m; 136 | } 137 | 138 | sub progressln { 139 | progress(@_); 140 | print STDERR "\n"; 141 | } 142 | 143 | sub process_packet { 144 | my($opt, $header, $packet) = @_; 145 | my $port = $opt->{port}; 146 | my $stream = $opt->{stream}; 147 | ${$opt->{count}}++; 148 | 149 | progress(".") if ${$opt->{count}}%1000 == 0; 150 | 151 | # ETH_TYPE_IP => 0x0800 152 | my $eth = NetPacket::Ethernet->decode($packet); 153 | if ($eth->{type} != NetPacket::Ethernet::ETH_TYPE_IP) { 154 | warn("Not a IP ethernet frame: $eth->{type}"); 155 | return; 156 | } 157 | 158 | # IP_PROTO_TCP => 6 159 | my $ip = NetPacket::IP->decode($eth->{data}); 160 | if ($ip->{proto} != NetPacket::IP::IP_PROTO_TCP) { 161 | warn("Not a TCP packet: $ip->{proto}"); 162 | return; 163 | } 164 | 165 | my $tcp = NetPacket::TCP->decode($ip->{data}); 166 | 167 | # FIN => 0x01; SYN => 0x02; ACK => 0x10; NetPacket::TCP 168 | # tcp 3 handshake 169 | if ($tcp->{flags} & SYN && !($tcp->{flags} & ACK)) { 170 | my $next_seqnum = $tcp->{seqnum} + 1; 171 | $stream->{ $next_seqnum } = { 172 | start => [ $header->{tv_sec}, $header->{tv_usec} ], 173 | end => [ 0, 0 ], 174 | state => 'syn', 175 | data => { req => [], res => [] }, 176 | }; 177 | } elsif ($tcp->{flags} & SYN && $tcp->{flags} & ACK) { 178 | if (exists $stream->{ $tcp->{acknum} }) { 179 | $stream->{ $tcp->{acknum} }{state} = 'syn+ack'; 180 | } else { 181 | debug("Got syn+ack but no syn packet"); 182 | } 183 | } elsif ($tcp->{flags} & ACK && length($tcp->{data}) > 0) { 184 | if (exists $stream->{ $tcp->{seqnum} }) { 185 | # client -> server 186 | my $next_seqnum = $tcp->{seqnum} + length($tcp->{data}); 187 | $stream->{ $next_seqnum } = delete $stream->{ $tcp->{seqnum} }; 188 | $stream->{ $next_seqnum }{state} = 'ack'; 189 | push @{ $stream->{ $next_seqnum }{data}{req} }, $tcp->{data}; 190 | } elsif (exists $stream->{ $tcp->{acknum} }) { 191 | # server -> client 192 | my $next_seqnum = $tcp->{acknum}; 193 | $stream->{ $next_seqnum } = delete $stream->{ $tcp->{acknum} }; 194 | $stream->{ $next_seqnum }{state} = 'ack'; 195 | push @{ $stream->{ $next_seqnum }{data}{res} }, $tcp->{data}; 196 | } else { 197 | # client -> server already existing connection 198 | my $next_seqnum = $tcp->{seqnum} + length($tcp->{data}); 199 | $stream->{ $next_seqnum } = { 200 | start => [ $header->{tv_sec}, $header->{tv_usec} ], 201 | end => [ 0, 0 ], 202 | state => 'ack', 203 | data => { req => [$tcp->{data}], res => [] }, 204 | }; 205 | } 206 | } elsif ($tcp->{flags} & ACK && $tcp->{flags} & FIN) { 207 | if (exists $stream->{ $tcp->{seqnum} }) { 208 | my $next_seqnum = $tcp->{seqnum} + 1; 209 | $stream->{ $next_seqnum } = delete $stream->{ $tcp->{seqnum} }; 210 | $stream->{ $next_seqnum }{state} = 'fin1'; 211 | } elsif (exists $stream->{ $tcp->{acknum} }) { 212 | my $next_seqnum = $tcp->{acknum}; 213 | $stream->{ $next_seqnum }{state} = 'fin2'; 214 | } else { 215 | debug("Got unknown FIN packet"); 216 | } 217 | } elsif ($tcp->{flags} & ACK && length($tcp->{data}) == 0) { 218 | if (exists $stream->{ $tcp->{seqnum} }) { 219 | my $next_seqnum = $tcp->{seqnum}; 220 | $stream->{ $next_seqnum }{state} = 'closed'; 221 | $stream->{ $next_seqnum }{end} = [ $header->{tv_sec}, $header->{tv_usec} ]; 222 | } else { 223 | ; # no problem multiple ack 224 | } 225 | } 226 | } 227 | 228 | sub process_redis { 229 | my($stream, $normalize) = @_; 230 | 231 | my %stats = ( 232 | start => 1<<31, 233 | end => 0, 234 | total_req => 0, 235 | ); 236 | my @slow = ({ time => 0, cmd => "dummy" }); 237 | 238 | for my $tcp (values %$stream) { 239 | my @req = split(/\r\n/, join('', @{ $tcp->{data}{req} })); 240 | my @res = split(/\r\n/, join('', @{ $tcp->{data}{res} })); 241 | if ($Debug >= 2) { 242 | debug("tcp: ".Dumper($tcp)); 243 | debug("req: ".Dumper(\@req)); 244 | debug("res: ".Dumper(\@res)); 245 | } 246 | 247 | $stats{start} = $tcp->{start}[0] if $tcp->{start}[0] < $stats{start}; 248 | $stats{end} = $tcp->{end}[0] if $tcp->{end}[0] > $stats{end}; 249 | 250 | while (@req) { 251 | my($line, $type, $data); 252 | my %cmd = ( req => [], res => "" ); 253 | 254 | ### parse request 255 | my @req_data; 256 | if ($req[0] =~ /^[a-zA-Z]/) { 257 | # inline command 258 | @req_data = split /\s+/, shift @req; 259 | } else { 260 | @req_data = _parse_RESP(\@req) 261 | } 262 | if (@req_data) { 263 | # uppercase command name 264 | $req_data[0] = uc $req_data[0]; 265 | push @{ $cmd{req} }, @req_data; 266 | } else { 267 | warn("Cannot parse request"); 268 | last; 269 | } 270 | 271 | ### parse response 272 | next unless $res[0]; # maybe broken packet 273 | if (my @res_data = _parse_RESP(\@res)) { 274 | $cmd{res} .= join '', @res_data; 275 | } 276 | 277 | ### statistics 278 | $stats{total_req}++; 279 | $stats{reqsec}{ $tcp->{start}[0] }++; 280 | my $cmd = $cmd{req}[0]; 281 | $stats{cmd}{$cmd}{count}++; 282 | $stats{cmd}{$cmd}{bytes} += length( $cmd{res} ); 283 | if ($cmd{req}[1]) { 284 | my $key = $cmd{req}[1]; 285 | $key =~ s{$normalize}{*}og if defined $normalize; 286 | $stats{cmd}{$cmd}{key}{$key}{count}++; 287 | $stats{cmd}{$cmd}{key}{$key}{bytes} += length( $cmd{res} ); 288 | } 289 | 290 | my $worst_n_slow = 20; 291 | my $elapsed = ($tcp->{end}[0]+$tcp->{end}[1]/1000000) 292 | - ($tcp->{start}[0]+$tcp->{start}[1]/1000000); 293 | if ($elapsed > $slow[-1]{time}) { 294 | my @s = sort { $b->{time} <=> $a->{time} } 295 | @slow, 296 | { 297 | time => $elapsed, 298 | cmd => join(" ", @{ $cmd{req} }), 299 | }; 300 | @slow = splice(@s, 0, $worst_n_slow); 301 | } 302 | 303 | progress(".") if $stats{total_req}%500 == 0; 304 | } 305 | if (@res) { 306 | warn("remain responce: ".join("\n", @res)); 307 | } 308 | } 309 | 310 | pop @slow if $slow[-1]{cmd} eq 'dummy'; 311 | $stats{slow} = \@slow; 312 | 313 | return \%stats; 314 | } 315 | 316 | sub _read_line { 317 | return (substr($_[0],0,1), substr($_[0],1)); 318 | } 319 | 320 | # http://redis.io/topics/protocol 321 | sub _parse_RESP { 322 | my($lines) = @_; 323 | my @data; 324 | 325 | my $line = shift @$lines; 326 | my ($type, $data) = _read_line($line); 327 | 328 | if ($type eq '$') { 329 | if ($data > 0) { 330 | push @data, shift @$lines; 331 | } else { 332 | ; # $0 (empty string) or $-1 (no data) 333 | } 334 | } elsif ($type eq '+') { # simple string 335 | push @data, $data; 336 | } elsif ($type eq '-') { # error 337 | push @data, $data; 338 | } elsif ($type eq ':') { # integer 339 | push @data, $data; 340 | } elsif ($type eq '*') { # array 341 | my $nelm = $data; 342 | for (my $i=0; $i<$nelm; $i++) { 343 | if (my @data2 = _parse_RESP($lines)) { 344 | push @data, @data2; 345 | } 346 | } 347 | 348 | } else { 349 | warn("Cannot parse: $line"); 350 | } 351 | 352 | # such as: ("SELECT", "2") 353 | return @data; 354 | } 355 | 356 | sub format_stats { 357 | my($top_n, $stream, $stats) = @_; 358 | 359 | my $count = 0; 360 | my @width; 361 | 362 | my $duration = $stats->{end} - $stats->{start}; 363 | $duration = 1 if $duration <= 0; 364 | 365 | my $total_traffic = 0; 366 | for my $s (values %{ $stats->{cmd} }) { 367 | $total_traffic += $s->{bytes} if $s->{bytes}; 368 | } 369 | 370 | my $fmt; 371 | 372 | $fmt = <<'EOR'; 373 | 374 | # redis-traffic-stats 375 | 376 | ## Summary 377 | 378 | * Duration: 379 | * %s - %s (%ds) 380 | * Total Traffic: 381 | * %d bytes (%.2f bytes/sec) 382 | * Total Requests: 383 | * %d requests (Avg %.2f req/sec, Peak %.2f req/sec) 384 | 385 | EOR 386 | printf($fmt, 387 | strftime("%Y-%m-%d %H:%M:%S", localtime( $stats->{start} )), 388 | strftime("%Y-%m-%d %H:%M:%S", localtime( $stats->{end} )), 389 | $duration, 390 | 391 | $total_traffic, 392 | $total_traffic/$duration, 393 | 394 | $stats->{total_req}, 395 | $stats->{total_req}/$duration, 396 | (max values %{ $stats->{reqsec} }), 397 | ); 398 | 399 | print "## Top Commands\n\n"; 400 | 401 | print "### By count\n"; 402 | @width = (16, 6, 6, 8); 403 | printf("%-*s | %-*s | %-*s | %-*s\n", 404 | $width[0], "Command", 405 | $width[1], "Count", 406 | $width[2], "Pct", 407 | $width[3], "Req/sec", 408 | ); 409 | print table_separator(\@width, [qw(L R R R)]); 410 | 411 | $count = 0; 412 | for my $cmd (sort { $stats->{cmd}{$b}{count} <=> $stats->{cmd}{$a}{count} } 413 | keys %{ $stats->{cmd} }) { 414 | my $s = $stats->{cmd}{$cmd}; 415 | printf("%-*s | %*d | %*.2f | %*.2f\n", 416 | $width[0], $cmd, 417 | $width[1], $s->{count}, 418 | $width[2], $s->{count}/$stats->{total_req}*100, 419 | $width[3], $s->{count}/$duration, 420 | ); 421 | last if ++$count >= $top_n; 422 | } 423 | print "\n"; 424 | 425 | print "### By traffic\n"; 426 | @width = (16, 9, 12); 427 | printf("%-*s | %-*s | %-*s\n", 428 | $width[0], "Command", 429 | $width[1], "Bytes", 430 | $width[2], "Byte/sec", 431 | ); 432 | print table_separator(\@width, [qw(L R R)]); 433 | 434 | $count = 0; 435 | for my $cmd (sort { $stats->{cmd}{$b}{bytes} <=> $stats->{cmd}{$a}{bytes} } 436 | keys %{ $stats->{cmd} }) { 437 | my $s = $stats->{cmd}{$cmd}; 438 | printf("%-*s | %*d | %*.2f\n", 439 | $width[0], $cmd, 440 | $width[1], $s->{bytes}, 441 | $width[2], $s->{bytes}/$duration, 442 | ); 443 | last if ++$count >= $top_n; 444 | } 445 | print "\n"; 446 | 447 | print "## Command Detail\n\n"; 448 | 449 | my $keylen = 0; 450 | for my $cmd (values %{ $stats->{cmd} }) { 451 | for my $key (keys %{ $cmd->{key} }) { 452 | my $l = length $key; 453 | $keylen = $l if $l > $keylen; 454 | } 455 | } 456 | 457 | for my $cmd (sort { $stats->{cmd}{$b}{bytes} <=> $stats->{cmd}{$a}{bytes} } 458 | keys %{ $stats->{cmd} }) { 459 | print "### $cmd\n"; 460 | @width = ($keylen, 9, 12, 6, 6, 8); 461 | printf("%-*s | %-*s | %-*s | %-*s | %-*s | %-*s\n", 462 | $width[0], "Key", 463 | $width[1], "Bytes", 464 | $width[2], "Byte/sec", 465 | $width[3], "Count", 466 | $width[4], "Pct", 467 | $width[5], "Req/sec", 468 | ); 469 | print table_separator(\@width, [qw(L R R R R R)]); 470 | 471 | $count = 0; 472 | for my $key (sort { $stats->{cmd}{$cmd}{key}{$b}{bytes} <=> $stats->{cmd}{$cmd}{key}{$a}{bytes} } 473 | keys %{ $stats->{cmd}{$cmd}{key} }) { 474 | my $s = $stats->{cmd}{$cmd}{key}{$key}; 475 | printf("%-*s | %*d | %*.2f | %*d | %*.2f | %*.2f\n", 476 | $width[0], $key, 477 | $width[1], $s->{bytes}, 478 | $width[2], $s->{bytes}/$duration, 479 | $width[3], $s->{count}, 480 | $width[4], $s->{count}/$stats->{cmd}{$cmd}{count}*100, 481 | $width[5], $s->{count}/$duration, 482 | ); 483 | last if ++$count >= $top_n; 484 | } 485 | print "\n"; 486 | } 487 | 488 | print "## Slow Commands\n\n"; 489 | 490 | @width = (6, 70); 491 | printf("%-*s | %-*s\n", 492 | $width[0], "Time", 493 | $width[1], "Command", 494 | ); 495 | print table_separator(\@width, [qw(R L)]); 496 | 497 | for my $s ( remove_dup_array(@{ $stats->{slow} }) ) { 498 | printf("%*.3f | %-*s\n", 499 | $width[0], $s->{time}, 500 | $width[1], trim(escape_nonprintable($s->{cmd})), 501 | ); 502 | } 503 | 504 | print "\n"; 505 | } 506 | 507 | # remove the duplicate value in array list. 508 | sub remove_dup_array { 509 | my %count; 510 | return grep { ++$count{join(',', %$_)} < 2 } @_; 511 | } 512 | 513 | sub table_separator { 514 | my($width, $align) = @_; 515 | my $separator = ""; 516 | 517 | my $ncol = scalar(@$width); 518 | for (my $i=0; $i<$ncol; $i++) { 519 | if ($i == 0) { 520 | $separator .= '-'x($width->[$i]) . ($align->[$i] eq 'R' ? ':' : '-'); 521 | } else { 522 | $separator .= '|' . '-'x($width->[$i]+1) . ($align->[$i] eq 'R' ? ':' : '-'); 523 | } 524 | } 525 | 526 | return $separator."\n"; 527 | } 528 | 529 | sub escape_nonprintable { 530 | my $s = shift; 531 | $s =~ s/([^[:print:]])/sprintf('\x{%02X}', ord($1))/eg; 532 | return $s; 533 | } 534 | 535 | sub trim { 536 | my $s = shift; 537 | my $len = 70; 538 | if (length $s > $len) { 539 | return substr($s,0,$len).'...'; 540 | } else { 541 | return $s; 542 | } 543 | } 544 | 545 | __END__ 546 | 547 | =head1 NAME 548 | 549 | B - Redis query analyzer for counting, traffic stats by command 550 | 551 | =head1 SYNOPSIS 552 | 553 | B -r pcapfile [-n top_n] [-p port] 554 | 555 | A) Analyze existing pcap file (RECOMMENDED WAY) 556 | 557 | # tcpdump -s 65535 tcp port 6379 -w redis.pcap -i eth0 558 | (wait for a while and stop by ^C) 559 | $ redis-traffic-stats -r redis.pcap 560 | 561 | B [-n top_n] [-p port] [-i device] [-c count_capture] [-t time_capture] 562 | 563 | B) Capture packets on demand and analyze it 564 | 565 | Capture 5000 packets 566 | # redis-traffic-stats -i eth0 -c 5000 567 | 568 | Capture for 10 sec 569 | # redis-traffic-stats -i eth0 -t 10 570 | 571 | B B<-h> | B<--help> | B<-?> 572 | 573 | Show detailed usage 574 | 575 | =head1 DESCRIPTION 576 | 577 | redis-traffic-stats is a query analyzer for Redis. 578 | 579 | The output looks as follows: 580 | 581 | =over 4 582 | 583 | =item Total network traffic and average of bytes/sec 584 | 585 | =item Total number of requests and average and peak of req/sec 586 | 587 | =item Top commands of count, percentage, req/sec by count 588 | 589 | =item Top commands of network traffic and byte/sec by amount of traffic 590 | 591 | =item Top keys for each command show bytes, byte/sec, count, percentage, req/sec 592 | 593 | =item Worst slow requests 594 | 595 | =back 596 | 597 | =head1 OPTIONS 598 | 599 | =over 4 600 | 601 | =item B<-r> pcapfile:Str, B<--pcapfile> pcapfile:Str 602 | 603 | Read existing pcap file instead of on demand packet capturing. 604 | 605 | # tcpdump -s 65535 tcp port 6379 -w redis.pcap -i eth0 606 | 607 | =item B<-p> port:Int, B<--port> port:Int 608 | 609 | Target port of analyze. Default is 6379. 610 | 611 | =item B<-n> top_n:Int, B<--top> top_n:Int 612 | 613 | Show top N keys in "Command Detail" section. Default is 10. 614 | 615 | =item B<-i> device:Str, B<--device> device:Str 616 | 617 | Network interface name used by -i option of tcpdump. Default is no -i option (lowest numbered interface excluding loopback device). 618 | 619 | This option has a point only in on demand packet capture mode. 620 | 621 | =item B<-c> count:Int, B<--count> count:Int 622 | 623 | tcpdump will exit after captured number of this option packets. Default is 10000. 624 | 625 | =item B<-t> time:Int, B<--time> time:Int 626 | 627 | tcpdump will exit after number of this option seconds. 628 | 629 | =item B<-d>, B<--debug> 630 | 631 | increase debug level. 632 | -d -d more verbosely. 633 | 634 | =back 635 | 636 | =head1 SEE ALSO 637 | 638 | L 639 | 640 | =head1 AUTHOR 641 | 642 | HIROSE Masaaki Ehirose31@gmail.comE 643 | 644 | =cut 645 | 646 | # for Emacsen 647 | # Local Variables: 648 | # mode: cperl 649 | # cperl-indent-level: 4 650 | # cperl-close-paren-offset: -4 651 | # cperl-indent-parens-as-block: t 652 | # indent-tabs-mode: nil 653 | # coding: utf-8 654 | # End: 655 | 656 | # vi: set ts=4 sw=4 sts=0 et ft=perl fenc=utf-8 : 657 | --------------------------------------------------------------------------------