├── .gitignore ├── README-CN.markdown ├── README.markdown ├── accessed-files ├── check-debug-info ├── fix-lua-bt ├── line-prof-by-func ├── merge-incomplete-bts ├── ngx-active-reqs ├── ngx-backtrace ├── ngx-body-filters ├── ngx-chain-bufs ├── ngx-cycle-pool ├── ngx-filter-lua-vm-bt ├── ngx-filter-zlib-bt ├── ngx-header-filters ├── ngx-leaked-pools ├── ngx-lua-bt ├── ngx-lua-conn-pools ├── ngx-lua-shdict ├── ngx-pcre-stats ├── ngx-pcrejit ├── ngx-phase-handlers ├── ngx-req-distr ├── ngx-sample-lua-bt ├── ngx-shm ├── resolve-inlines ├── resolve-src-lines ├── reverse-bts ├── sample-bt ├── sample-bt-off-cpu ├── sample-bt-vfs ├── tcp-accept-queue └── tcp-recv-queue /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.swp 3 | *.swo 4 | *.o 5 | *.so 6 | reindex 7 | *.html 8 | tags 9 | genmobi.sh 10 | *.mobi 11 | *.stp 12 | go 13 | ctags 14 | nginx 15 | *.svg 16 | *.bt 17 | *.cbt 18 | -------------------------------------------------------------------------------- /accessed-files: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Long qw( GetOptions ); 10 | 11 | GetOptions("a=s", \(my $stap_args), 12 | "d", \(my $dump_src), 13 | "h", \(my $help), 14 | "l=i", \(my $limit), 15 | "p=i", \(my $pid), 16 | "r", \(my $check_reads), 17 | "w", \(my $check_writes), 18 | "t=i", \(my $time)) 19 | or die usage(); 20 | 21 | if ($help) { 22 | print usage(); 23 | exit; 24 | } 25 | 26 | if (!$pid) { 27 | die "No user process pid specified by the -p option\n"; 28 | } 29 | 30 | if (!$limit) { 31 | $limit = 10; 32 | } 33 | 34 | if (!defined $stap_args) { 35 | $stap_args = ''; 36 | } 37 | 38 | if (!defined $check_reads && !defined $check_writes) { 39 | die "Neither -r nor -w options are specified.\n"; 40 | } 41 | 42 | my ($probes, $operation); 43 | 44 | if (defined $check_reads) { 45 | if (defined $check_writes) { 46 | $probes = "vfs.read.return, vfs.write.return"; 47 | $operation = "reads/writes"; 48 | 49 | } else { 50 | $probes = "vfs.read.return"; 51 | $operation = "reads"; 52 | } 53 | 54 | } else { 55 | $probes = "vfs.write.return"; 56 | $operation = "writes"; 57 | } 58 | 59 | if ($^O ne 'linux') { 60 | die "Only linux is supported but I am on $^O.\n"; 61 | } 62 | 63 | my $exec_file = "/proc/$pid/exe"; 64 | if (!-f $exec_file) { 65 | die "Nginx process $pid is not running or ", 66 | "you do not have enough permissions.\n"; 67 | } 68 | 69 | my $exec_path = readlink $exec_file; 70 | 71 | my $postamble = <<_EOC_; 72 | probe end { 73 | if (!found) { 74 | println("No file $operation observed so far.") 75 | 76 | } else { 77 | printf("\\n=== Top $limit file $operation ===\\n") 78 | 79 | i = 0 80 | foreach (path in stats- limit $limit) { 81 | printf("#%d: %d times, %d bytes $operation in file %s.\\n", 82 | ++i, \@count(stats[path]), \@sum(stats[path]), path) 83 | } 84 | } 85 | } 86 | _EOC_ 87 | 88 | my $guide; 89 | if (defined $time) { 90 | $guide = "Please wait for $time seconds."; 91 | $postamble .= <<_EOC_; 92 | probe timer.s($time) { 93 | exit() 94 | } 95 | _EOC_ 96 | 97 | } else { 98 | $guide = "Hit Ctrl-C to end."; 99 | } 100 | 101 | my $preamble = <<_EOC_; 102 | global stats 103 | global found 104 | 105 | probe begin 106 | { 107 | printf("Tracing %d ($exec_path)...\\n$guide\\n", target()) 108 | } 109 | _EOC_ 110 | chop $preamble; 111 | 112 | my $stap_src = <<_EOC_; 113 | $preamble 114 | 115 | probe $probes { 116 | if (pid() == target() && \$return > 0) { 117 | path = __file_filename(file) 118 | //println(path) 119 | stats[path] <<< \$return 120 | found++; 121 | } 122 | } 123 | 124 | $postamble 125 | _EOC_ 126 | 127 | if ($dump_src) { 128 | print $stap_src; 129 | exit; 130 | } 131 | 132 | open my $in, "|stap --skip-badvars $stap_args -x $pid -" 133 | or die "Cannot run stap: $!\n"; 134 | 135 | print $in $stap_src; 136 | 137 | close $in; 138 | 139 | sub usage { 140 | return <<'_EOC_'; 141 | Usage: 142 | accessed-files [optoins] 143 | 144 | Options: 145 | -a Pass extra arguments to the stap utility. 146 | -d Dump out the systemtap script source. 147 | -h Print this usage. 148 | -l Limiting the number of different file names printed. 149 | (Default to 10.) 150 | -p Specify the user worker process pid. 151 | -r Trace file reads. 152 | -w Trace file writes. 153 | -t Specify the time period in seconds for sampling 154 | 155 | Examples: 156 | accessed-files -p 12345 -t 5 -r 157 | accessed-files -p 12345 -w -l 20 158 | accessed-files -p 12345 -w -r -a '-DMAXACTION=100000' 159 | _EOC_ 160 | } 161 | -------------------------------------------------------------------------------- /check-debug-info: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use Getopt::Std qw( getopts ); 7 | 8 | my %opts; 9 | 10 | getopts("hp:", \%opts) 11 | or die usage(); 12 | 13 | my $pid = $opts{p} 14 | or die "No -p option specified.\n"; 15 | 16 | my $exec_file = "/proc/$pid/exe"; 17 | if (!-f $exec_file) { 18 | die "User process $pid is not running or ", 19 | "you do not have enough permissions.\n"; 20 | } 21 | 22 | my $path = readlink $exec_file; 23 | 24 | my %files = ($path => 1); 25 | 26 | { 27 | my $maps_file = "/proc/$pid/maps"; 28 | open my $in, $maps_file 29 | or die "Cannot open $maps_file for reading: $!\n"; 30 | 31 | while (<$in>) { 32 | if (/\s+(\/\S+\.so)$/) { 33 | if (!exists $files{$1}) { 34 | $files{$1} = 1; 35 | #warn $1, "\n"; 36 | } 37 | } 38 | } 39 | 40 | close $in; 41 | } 42 | 43 | for my $file (sort keys %files) { 44 | my $lines = `objdump --dwarf=str $file|wc -l`; 45 | if ($? != 0) { 46 | warn "Failed to check file $file: $?\n"; 47 | next; 48 | } 49 | 50 | if ($lines <= 3) { 51 | warn "File $file has no debug info embedded.\n"; 52 | 53 | } else { 54 | #warn "File $file checked.\n"; 55 | } 56 | } 57 | 58 | sub usage { 59 | print <<_EOC_ 60 | Usage: 61 | check-debug-info -p 62 | _EOC_ 63 | } 64 | -------------------------------------------------------------------------------- /fix-lua-bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use 5.006001; 4 | use strict; 5 | use warnings; 6 | 7 | my $infile = shift or 8 | die "No input file specified.\n"; 9 | 10 | my %funcs; 11 | 12 | open my $in, $infile 13 | or die "Cannot open $infile for reading: $!\n"; 14 | 15 | while (<$in>) { 16 | if (/^((?:\w+:)?)\@(.*):(\d+)/) { 17 | my ($prefix, $path, $ln) = ($1, $2, $3); 18 | my $key = "$path:$ln"; 19 | my $func = $funcs{$key}; 20 | if (defined $func) { 21 | #warn "Found $key\n"; 22 | print "$func\n"; 23 | next; 24 | } 25 | 26 | #print "looking for $path:$ln\n"; 27 | 28 | my $file; 29 | if ($path =~ m{lib((?:\/[-\w]+)+)\.lua}) { 30 | $file = $1; 31 | $file =~ s/^\///g; 32 | $file =~ s/\//\./g; 33 | $file =~ s/-/_/g; 34 | 35 | } elsif ($path =~ m{[-\w]+\.lua}) { 36 | $file = $&; 37 | 38 | } else { 39 | $file = $path; 40 | } 41 | 42 | if ($ln > 0) { 43 | open my $in, $path 44 | or die "failed to open $path for reading: $!\n"; 45 | while (<$in>) { 46 | if ($. == $ln) { 47 | if (/function\s+((?:\w+[.:])*\w+)\s*\(/) { 48 | $func = $1; 49 | 50 | } elsif (/(\S+)\s*=\s*function\s*\(/) { 51 | $func = $1; 52 | 53 | } else { 54 | $func = $ln; 55 | } 56 | 57 | last; 58 | } 59 | } 60 | 61 | close $in; 62 | 63 | if (!defined $func) { 64 | die "$path:$ln not found.\n"; 65 | } 66 | 67 | } else { 68 | $func = $ln; 69 | } 70 | 71 | $funcs{$key} = "$file:$func"; 72 | 73 | print "$prefix$file:$func\n"; 74 | 75 | } else { 76 | print $_; 77 | } 78 | } 79 | 80 | close $in; 81 | -------------------------------------------------------------------------------- /line-prof-by-func: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | #use warnings; 5 | 6 | use Getopt::Std qw( getopts ); 7 | 8 | my %opts; 9 | getopts("l:f:p:h", \%opts) or usage(); 10 | 11 | if ($opts{h}) { 12 | usage(); 13 | } 14 | 15 | my %addr2line; 16 | my %hits; 17 | my $addr; 18 | 19 | my $func = $opts{f} or die "No -f specified.\n"; 20 | if ($func !~ /^[_a-zA-Z]\w+$/) { 21 | die "Bad function name: $func\n"; 22 | } 23 | 24 | my $lib = $opts{l} or die "No -l specified.\n"; 25 | 26 | my $pid = $opts{p} or die "No -p specified.\n"; 27 | 28 | my $mapsfile = "/proc/$pid/maps"; 29 | open my $maps, "<$mapsfile" 30 | or die "Cannot open $mapsfile for reading: $!\n"; 31 | 32 | my $libfile; 33 | my @maps; 34 | while (<$maps>) { 35 | if (/^([a-f0-9]+)-([a-f0-9]+) [-r][-w]xp .*? (\/\S*?\Q$lib\E\S*)$/) { 36 | push @maps, [hex($1), hex($2)]; 37 | if (!defined $libfile) { 38 | $libfile = $3; 39 | print "Found library $libfile for $lib.\n"; 40 | } 41 | } 42 | } 43 | 44 | close $maps; 45 | 46 | if (!@maps) { 47 | die "No $lib maps found in process $pid.\n"; 48 | } 49 | 50 | my $infile = shift or die "No input file specified.\n"; 51 | 52 | open my $in, "<$infile" 53 | or die "Cannot open $infile for reading: $!\n"; 54 | 55 | while (<$in>) { 56 | if (/^ (0x[a-f0-9]+) : $func\b/) { 57 | $addr = $1; 58 | 59 | } elsif (/^\t(\d+)$/) { 60 | my $cnt = $1; 61 | if ($addr) { 62 | $addr = hex($addr); 63 | my $line = $addr2line{$addr}; 64 | if (!$line) { 65 | my $startaddr; 66 | for my $map (@maps) { 67 | if ($addr >= $map->[0] && $addr <= $map->[1]) { 68 | $startaddr = $map->[0]; 69 | } 70 | } 71 | if (!$startaddr) { 72 | warn "Addr $addr not found in memory maps.\n"; 73 | undef $addr; 74 | next; 75 | } 76 | 77 | my $a = sprintf("%#x", $addr - $startaddr); 78 | #warn "addr: $a\n"; 79 | $line = `addr2line -s -i -e $libfile $a`; 80 | chomp $line; 81 | $line =~ s/\n+/ < /g; 82 | #warn "line: $line\n"; 83 | $addr2line{$addr} = $line; 84 | } 85 | 86 | $hits{$line} += $cnt; 87 | undef $addr; 88 | } 89 | } 90 | } 91 | 92 | close $in; 93 | 94 | my (@entries, $total); 95 | $total = 0; 96 | while (my ($k, $v) = each %hits) { 97 | push @entries, [$k, $v]; 98 | $total += $v; 99 | } 100 | 101 | @entries = sort { $b->[1] <=> $a->[1] } @entries; 102 | for (@entries) { 103 | my $cnt = $_->[1]; 104 | my $ratio = $cnt * 100 / $total; 105 | printf "%.02f%% (%d) %s\n", $ratio, $cnt, $_->[0]; 106 | } 107 | 108 | sub usage { 109 | die "Usage: $0 -f -l -p \n"; 110 | } 111 | -------------------------------------------------------------------------------- /merge-incomplete-bts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | while (<>) { 7 | s/^ (0x[0-9a-f]+)\s+:\s+\1/ $1 : /; 8 | print; 9 | } 10 | -------------------------------------------------------------------------------- /ngx-active-reqs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("c:ua:dhp:mk", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $child_pids = get_child_processes($pid); 29 | if (!@$child_pids) { 30 | push @$child_pids, $pid; 31 | } 32 | 33 | my $condition = gen_pid_test_condition($child_pids); 34 | 35 | my $check_pools = $opts{m}; 36 | 37 | my $check_upstreams = $opts{u}; 38 | 39 | my $keep_waiting = $opts{k} || 0; 40 | 41 | my $stap_args = $opts{a} || ''; 42 | 43 | my $conn_fd = $opts{c} || 0; 44 | 45 | if ($^O ne 'linux') { 46 | die "Only linux is supported but I am on $^O.\n"; 47 | } 48 | 49 | my $exec_file = "/proc/$pid/exe"; 50 | if (!-f $exec_file) { 51 | die "Nginx process $pid is not running or ", 52 | "you do not have enough permissions.\n"; 53 | } 54 | 55 | my $nginx_path = readlink $exec_file; 56 | 57 | my $ver = `stap --version 2>&1`; 58 | if (!defined $ver) { 59 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 60 | } 61 | 62 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 63 | my $v = $1; 64 | if ($v < 2.1) { 65 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 66 | } 67 | 68 | } else { 69 | die "ERROR: unknown version of systemtap:\n$ver\n"; 70 | } 71 | 72 | my $stap_src; 73 | 74 | my $preamble; 75 | if (@$child_pids == 1) { 76 | $preamble = <<_EOC_; 77 | probe begin { 78 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 79 | } 80 | _EOC_ 81 | 82 | } else { 83 | $preamble = <<_EOC_; 84 | probe begin { 85 | printf("Tracing @$child_pids ($nginx_path)...\\n\\n") 86 | } 87 | _EOC_ 88 | } 89 | 90 | #chop $preamble; 91 | 92 | my $c = '@cast(c, "ngx_connection_t")'; 93 | my $r = '@cast(r, "ngx_http_request_t")'; 94 | my $u = '@cast(u, "ngx_http_upstream_t")'; 95 | my $p = '@cast(p, "ngx_event_pipe_t")'; 96 | 97 | my $need_blank_line; 98 | 99 | my ($check_upstream_code, $check_pool_init_code, $check_pool_code); 100 | 101 | my $fin_code = ''; 102 | 103 | if ($check_upstreams) { 104 | $need_blank_line = 1; 105 | 106 | $preamble .= <<'_EOC_'; 107 | global funcs 108 | _EOC_ 109 | 110 | $check_upstream_code = <<_EOC_; 111 | u = $r->upstream 112 | if (u) { 113 | c = $u->peer->connection 114 | if (c) { 115 | printf(" upstream conn: ssl=%d, fd=%d, buffered=%x, err=%d, action=%s\\n", 116 | \@defined($c->ssl) ? $c->ssl : 0, $c->fd, 117 | $c->buffered, 118 | $c->error, user_string($c->log->action)) 119 | 120 | printf(" upstream read: ready=%d, active=%d, timer_set=%d, timedout=%d, eof=%d, error=%d\\n", 121 | $c->read->ready, $c->read->active, $c->read->timer_set, 122 | $c->read->timedout, $c->read->eof, $c->read->error) 123 | 124 | printf(" upstream write: ready=%d, active=%d, timer_set=%d, timedout=%d, eof=%d, error=%d\\n", 125 | $c->write->ready, $c->write->active, $c->write->timer_set, 126 | $c->write->timedout, $c->write->eof, $c->write->error) 127 | 128 | } else { 129 | printf(" no upstream conn found\\n") 130 | } 131 | 132 | rev_handler = $u->read_event_handler 133 | if (rev_handler) { 134 | rev_handler_name = usymname(rev_handler) 135 | funcs[rev_handler] = rev_handler_name 136 | } 137 | 138 | wev_handler = $u->write_event_handler 139 | if (wev_handler) { 140 | wev_handler_name = usymname(wev_handler) 141 | funcs[wev_handler] = wev_handler_name 142 | } 143 | 144 | printf(" upstream read/write ev handlers: %s %s\\n", rev_handler_name, wev_handler_name) 145 | 146 | p = $u->pipe 147 | if (p) { 148 | printf(" upstream pipe: done=%d, eof=%d, err=%d\\n", 149 | $p->upstream_done, $p->upstream_eof, $p->upstream_error) 150 | } 151 | 152 | printf(" upstream headers in: status=%d, content_length=%d, conn_close=%d, chunked=%d\\n", 153 | $u->headers_in->status_n, 154 | $u->headers_in->content_length_n, 155 | $u->headers_in->connection_close, 156 | $u->headers_in->chunked) 157 | 158 | pos = $u->buffer->pos 159 | last = $u->buffer->last 160 | end = $u->buffer->end 161 | size = last - pos 162 | if (size && pos) { 163 | if (size > 40) { 164 | data = text_str(user_string_n(pos, 20) . "..." . user_string_n(last - 20 , 20)) 165 | 166 | } else { 167 | data = text_str(user_string_n(pos, size)) 168 | } 169 | 170 | } else { 171 | data = "" 172 | } 173 | 174 | printf(" upstream buffer: len=%d, data=\\"%s\\", free=%d, size=%d\\n", 175 | size, data, end - last, end - $u->buffer->start) 176 | 177 | if ($u->state) { 178 | printf(" upstream state: resp_len=%d\\n", $u->state->response_length) 179 | } 180 | 181 | printf(" upstream: length=%d, busy_bufs=%p, out_bufs=%p\\n", $u->length, 182 | $u->busy_bufs, $u->out_bufs) 183 | 184 | } else { 185 | printf(" no upstream found\\n") 186 | } 187 | _EOC_ 188 | 189 | $fin_code .= " delete funcs\n"; 190 | 191 | } else { 192 | $check_upstream_code = ''; 193 | } 194 | 195 | if ($check_pools) { 196 | $need_blank_line = 1; 197 | 198 | $check_pool_init_code = <<_EOC_; 199 | ngx_pool_t_size = &\@cast(0, "ngx_pool_t")[1] 200 | size_t_size = &\@cast(0, "size_t")[1] 201 | 202 | total_mem_for_reqs = 0 203 | _EOC_ 204 | chop $check_pool_init_code; 205 | 206 | $check_pool_code = <<_EOC_; 207 | pool = \@cast(r, "ngx_http_request_t")->pool 208 | 209 | if (!pool) { 210 | printf(" pool is null\\n") 211 | continue 212 | } 213 | 214 | end = \@cast(pool, "ngx_pool_t")->d->end 215 | 216 | printf(" pool chunk size: %d\\n", end - pool) 217 | 218 | lim = \@cast(pool, "ngx_pool_t")->max + 1 219 | 220 | /* analyze small blocks */ 221 | 222 | used = 0 223 | unused = 0 224 | 225 | p = pool 226 | n = \@cast(pool, "ngx_pool_t")->d->next 227 | 228 | for ( ;; ) { 229 | 230 | last = \@cast(p, "ngx_pool_t")->d->last 231 | end = \@cast(p, "ngx_pool_t")->d->end 232 | 233 | used += last - p - ngx_pool_t_size 234 | unused += end - last 235 | 236 | /* printf("used: %d, unused %d\\n", last - p - ngx_pool_t_size, end - last) */ 237 | 238 | if (n == 0) { 239 | break 240 | } 241 | 242 | p = n 243 | n = \@cast(n, "ngx_pool_t")->d->next 244 | } 245 | 246 | printf(" small blocks (< %d): %d bytes used, %d bytes unused\\n", 247 | lim, used, unused) 248 | 249 | /* analyze large blocks */ 250 | 251 | total = 0 252 | blocks = 0 253 | 254 | for (l = \@cast(pool, "ngx_pool_t")->large; 255 | l; 256 | l = \@cast(l, "ngx_pool_large_t")->next) 257 | { 258 | ptr = \@cast(l, "ngx_pool_large_t")->alloc 259 | if (ptr) { 260 | blocks++ 261 | 262 | /* XXX specific to the glibc malloc implementation */ 263 | ptr -= size_t_size 264 | block_size = \@cast(ptr, "size_t")[0] & ~(size_t_size - 1) 265 | /* printf("large block size: %d %d\\n", 266 | \@cast(ptr, "size_t")[0], block_size) */ 267 | 268 | total += block_size 269 | } 270 | } 271 | 272 | printf(" large blocks (>= %d): %d blocks, %d bytes (used)\\n", 273 | lim, blocks, total) 274 | 275 | /* total summary */ 276 | 277 | printf(" total used: %d bytes\\n", total + used) 278 | 279 | total_mem_for_reqs += total + used 280 | _EOC_ 281 | chop $check_pool_code; 282 | 283 | $fin_code .= <<_EOC_; 284 | printf("total memory used for all %d active requests: %d bytes\\n", 285 | nreqs, total_mem_for_reqs) 286 | _EOC_ 287 | chop $fin_code; 288 | 289 | } else { 290 | $check_pool_init_code = ''; 291 | $check_pool_code = ''; 292 | $fin_code .= <<_EOC_; 293 | printf("\\nfound %d active requests.\\n", nreqs) 294 | _EOC_ 295 | chop $fin_code; 296 | } 297 | 298 | my $formatting_code = ''; 299 | if ($need_blank_line) { 300 | $formatting_code = qq{ printf("\\n")\n}; 301 | } 302 | 303 | $stap_src = <<_EOC_; 304 | $preamble 305 | 306 | probe process("$nginx_path").function("ngx_process_events_and_timers"), 307 | process("$nginx_path").function("ngx_http_handler") 308 | { 309 | my_pid = pid() 310 | if ($condition) { 311 | 312 | begin = local_clock_us() 313 | 314 | $check_pool_init_code 315 | 316 | cached_sec = \@var("ngx_cached_time")->sec 317 | cached_msec = \@var("ngx_cached_time")->msec 318 | 319 | conns = \@var("ngx_cycle\@ngx_cycle.c")->connections 320 | conn_n = \@var("ngx_cycle\@ngx_cycle.c")->connection_n 321 | 322 | nreqs = 0 323 | for (i = 0; i < conn_n; i++) { 324 | c = &\@cast(conns, "ngx_connection_t")[i] 325 | 326 | fd = $c->fd 327 | 328 | read = $c->read 329 | reqs = $c->requests 330 | destroyed = $c->destroyed 331 | 332 | if (reqs && fd != -1 && read && !destroyed) { 333 | r = $c->data 334 | 335 | if ($conn_fd && $conn_fd != fd) { 336 | if (!$r->upstream) { 337 | continue 338 | } 339 | 340 | c = $r->upstream->peer->connection 341 | if (!c) { 342 | continue 343 | } 344 | 345 | if ($c->fd != $conn_fd) { 346 | continue 347 | } 348 | } 349 | 350 | nreqs++ 351 | 352 | /* get uri */ 353 | 354 | uri = &$r->uri 355 | uri_data = \@cast(uri, "ngx_str_t")->data 356 | uri_len = \@cast(uri, "ngx_str_t")->len 357 | 358 | /* get uri args */ 359 | 360 | args = &$r->args 361 | args_data = \@cast(args, "ngx_str_t")->data 362 | args_len = \@cast(args, "ngx_str_t")->len 363 | 364 | if (args_len == 0 || args_data == 0) { 365 | args_str = "" 366 | 367 | } else { 368 | args_str = user_string_n(args_data, args_len) 369 | } 370 | 371 | /* get request time */ 372 | 373 | start_sec = $r->start_sec 374 | start_msec = $r->start_msec 375 | 376 | ms = (cached_sec - start_sec) * 1000 + (cached_msec - start_msec) 377 | if (ms < 0) { 378 | ms = 0 379 | } 380 | 381 | /* get host */ 382 | 383 | host = $r->headers_in->host 384 | if (host && $r->headers_in->host->value->data) { 385 | host_name = user_string_n($r->headers_in->host->value->data, $r->headers_in->host->value->len) 386 | 387 | } else { 388 | host_name = "" 389 | } 390 | 391 | /* get method name */ 392 | 393 | method = &$r->method_name 394 | method_data = \@cast(method, "ngx_str_t")->data 395 | method_len = \@cast(method, "ngx_str_t")->len 396 | 397 | /* get main request */ 398 | 399 | main = $r->main 400 | 401 | printf("%s \\"%s %s\?%s\\", r=%p, count=%d, keepalive=%d, spdy=%d, host=%s, status=%d, time=%d.%03ds, buffered=%x, conn: ssl=%d, from=%s, reqs=%d, err=%d, fd=%d, buffered=%x, %s\\n", 402 | main == r ? "req" : "subreq", 403 | method_data ? user_string_n(method_data, method_len) : "", 404 | uri_data ? user_string_n(uri_data, uri_len) : "", args_str, 405 | r, $r->main->count, 406 | $r->keepalive, 407 | \@defined($r->spdy_stream) ? $r->spdy_stream != 0 : 0, 408 | host_name, $r->headers_out->status, 409 | ms / 1000, ms % 1000, 410 | $r->buffered, 411 | \@defined($c->ssl) ? $c->ssl : 0, 412 | $c->addr_text->data ? user_string_n($c->addr_text->data, $c->addr_text->len) : "", 413 | reqs, $c->error, fd, 414 | $c->buffered, 415 | $c->log->action ? user_string($c->log->action) : "") 416 | 417 | $check_pool_code 418 | $check_upstream_code 419 | $formatting_code 420 | 421 | if ($conn_fd) { 422 | break 423 | } 424 | } 425 | } 426 | 427 | $fin_code 428 | 429 | elapsed = local_clock_us() - begin 430 | printf("%d microseconds elapsed in the probe handler.\\n", elapsed) 431 | if (nreqs || !$keep_waiting) { 432 | exit() 433 | } 434 | } /* $condition */ 435 | } 436 | _EOC_ 437 | 438 | if ($opts{d}) { 439 | print $stap_src; 440 | exit; 441 | } 442 | 443 | my @opts; 444 | if (@$child_pids == 1) { 445 | push @opts, '-x', $child_pids->[0]; 446 | } 447 | 448 | open my $in, "|stap --skip-badvars @opts $stap_args -" 449 | or die "Cannot run stap: $!\n"; 450 | 451 | print $in $stap_src; 452 | 453 | close $in; 454 | 455 | sub usage { 456 | return <<'_EOC_'; 457 | Usage: 458 | ngx-active-reqs [options] 459 | 460 | Options: 461 | -a Pass extra arguments to the stap utility. 462 | -c Specify the file descriptor number of a downstream 463 | or upstream connection to filter the requests. 464 | -d Dump out the systemtap script source. 465 | -h Print this usage. 466 | -k When no active rquests are found in the current 467 | event cycle, this option will keep probing at 468 | subsequent cycles, instead of quitting immediately. 469 | -m Gather the information about request memory pools. 470 | -p Specify the nginx worker or master process pid. 471 | -u Enable upstream information dump. 472 | 473 | Examples: 474 | ngx-active-reqs -p 12345 475 | ngx-active-reqs -p 12345 -k 476 | ngx-active-reqs -p 12345 -m 477 | ngx-active-reqs -p 12345 -m -a '-DMAXACTION=100000' 478 | ngx-active-reqs -p 12345 -u 479 | _EOC_ 480 | } 481 | 482 | sub get_child_processes { 483 | my $pid = shift; 484 | my @files = glob "/proc/[0-9]*/stat"; 485 | my @children; 486 | for my $file (@files) { 487 | #print "file: $file\n"; 488 | if ($file =~ m{^/proc/$pid/}) { 489 | next; 490 | } 491 | 492 | open my $in, $file or next; 493 | my $line = <$in>; 494 | close $in; 495 | if ($line =~ /^(\d+) \S+ \S+ (\d+)/) { 496 | my ($child, $parent) = ($1, $2); 497 | if ($parent eq $pid) { 498 | push @children, $child; 499 | } 500 | } 501 | } 502 | 503 | @children = sort { $a <=> $b } @children; 504 | return \@children; 505 | } 506 | 507 | sub gen_pid_test_condition { 508 | my $pids = shift; 509 | if (@$pids == 1) { 510 | return '(my_pid == target())'; 511 | } 512 | my @c; 513 | for my $pid (@$pids) { 514 | push @c, "my_pid == $pid"; 515 | } 516 | return '(' . join(" || ", @c) . ')'; 517 | } 518 | -------------------------------------------------------------------------------- /ngx-backtrace: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("hp:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx master process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | if ($^O ne 'linux') { 29 | die "Only linux is supported but I am on $^O.\n"; 30 | } 31 | 32 | my $exec_file = "/proc/$pid/exe"; 33 | if (!-f $exec_file) { 34 | die "Nginx process $pid is not running or ", 35 | "you do not have enough permissions.\n"; 36 | } 37 | 38 | my $nginx_path = readlink $exec_file; 39 | 40 | exec 'addr2line', '-e', $exec_file, '-f', @ARGV; 41 | 42 | sub usage { 43 | return <<'_EOC_'; 44 | Usage: 45 | ngx-backtrace [optoins] 46 | 47 | Options: 48 | -h Print this usage. 49 | -p Specify the nginx (worker) process pid. 50 | 51 | Examples: 52 | ngx-backtrace -p 12345 53 | _EOC_ 54 | } 55 | 56 | -------------------------------------------------------------------------------- /ngx-body-filters: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:n:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | if ($^O ne 'linux') { 31 | die "Only linux is supported but I am on $^O.\n"; 32 | } 33 | 34 | my $exec_file = "/proc/$pid/exe"; 35 | if (!-f $exec_file) { 36 | die "Nginx process $pid is not running or ", 37 | "you do not have enough permissions.\n"; 38 | } 39 | 40 | my $nginx_path = readlink $exec_file; 41 | 42 | my $ver = `stap --version 2>&1`; 43 | if (!defined $ver) { 44 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 45 | } 46 | 47 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 48 | my $v = $1; 49 | if ($v < 2.1) { 50 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 51 | } 52 | 53 | } else { 54 | die "ERROR: unknown version of systemtap:\n$ver\n"; 55 | } 56 | 57 | my $stap_src; 58 | 59 | my $zone = $opts{n}; 60 | 61 | my $preamble = <<_EOC_; 62 | probe begin { 63 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 64 | } 65 | _EOC_ 66 | chop $preamble; 67 | 68 | $stap_src = <<_EOC_; 69 | $preamble 70 | 71 | probe process("$nginx_path").function("ngx_http_write_filter") 72 | { 73 | if (pid() == target()) { 74 | #println("=============================") 75 | 76 | begin = local_clock_us() 77 | res = "" 78 | top_filter = usymname(\@var("ngx_http_top_body_filter")) 79 | #printf("top filter: %s\\n", top_filter) 80 | 81 | b = ubacktrace(); 82 | token = tokenize(b, " "); 83 | if (token != "") { 84 | found = 0 85 | for ( ;; ) { 86 | token = tokenize("", " "); 87 | if (token == "") { 88 | break; 89 | } 90 | 91 | addr = strtol(token, 16) 92 | sym = usymname(addr) 93 | 94 | #printf("found sym: %s (%p)\\n", sym, addr) 95 | 96 | if (sym == "ngx_http_header_filter") { 97 | res = "" 98 | break 99 | } 100 | 101 | res = sym . "\\n" . res 102 | 103 | if (sym == top_filter) { 104 | #println("found top filter!") 105 | found = 1 106 | break 107 | } 108 | } 109 | 110 | if (!found) { 111 | res = "" 112 | } 113 | } 114 | 115 | if (res != "") { 116 | print(res) 117 | println("ngx_http_write_filter") 118 | 119 | elapsed = local_clock_us() - begin 120 | printf("\\n%d microseconds elapsed in the probe handler.\\n", elapsed) 121 | exit() 122 | } 123 | } 124 | } 125 | _EOC_ 126 | 127 | if ($opts{d}) { 128 | print $stap_src; 129 | exit; 130 | } 131 | 132 | open my $in, "|stap --skip-badvars $stap_args -x $pid -" 133 | or die "Cannot run stap: $!\n"; 134 | 135 | print $in $stap_src; 136 | 137 | close $in; 138 | 139 | sub usage { 140 | return <<'_EOC_'; 141 | Usage: 142 | ngx-body-filters [optoins] 143 | 144 | Options: 145 | -a Pass extra arguments to the stap utility. 146 | -d Dump out the systemtap script source. 147 | -h Print this usage. 148 | -p Specify the nginx worker process pid. 149 | 150 | Examples: 151 | ngx-body-filters -p 12345 152 | _EOC_ 153 | } 154 | 155 | -------------------------------------------------------------------------------- /ngx-chain-bufs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | my $chain_ptr = shift 31 | or die "No chain pointer specified.\n"; 32 | 33 | if ($^O ne 'linux') { 34 | die "Only linux is supported but I am on $^O.\n"; 35 | } 36 | 37 | my $exec_file = "/proc/$pid/exe"; 38 | if (!-f $exec_file) { 39 | die "Nginx process $pid is not running or ", 40 | "you do not have enough permissions.\n"; 41 | } 42 | 43 | my $nginx_path = readlink $exec_file; 44 | 45 | my $ver = `stap --version 2>&1`; 46 | if (!defined $ver) { 47 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 48 | } 49 | 50 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 51 | my $v = $1; 52 | if ($v < 2.1) { 53 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 54 | } 55 | 56 | } else { 57 | die "ERROR: unknown version of systemtap:\n$ver\n"; 58 | } 59 | 60 | my $preamble = <<_EOC_; 61 | probe begin { 62 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 63 | } 64 | _EOC_ 65 | chop $preamble; 66 | 67 | my $c = '@cast(c, "ngx_connection_t")'; 68 | my $r = '@cast(r, "ngx_http_request_t")'; 69 | my $u = '@cast(u, "ngx_http_upstream_t")'; 70 | my $p = '@cast(p, "ngx_event_pipe_t")'; 71 | 72 | my $need_blank_line; 73 | 74 | my ($check_upstream_code, $check_pool_init_code, $check_pool_code); 75 | 76 | my $fin_code = ''; 77 | 78 | my $cl = '@cast(cl, "ngx_chain_t")'; 79 | my $buf = '@cast(buf, "ngx_buf_t")'; 80 | 81 | my $stap_src = <<_EOC_; 82 | $preamble 83 | 84 | probe process("$nginx_path").function("ngx_process_events_and_timers"), 85 | process("$nginx_path").function("ngx_http_handler") 86 | { 87 | if (pid() == target()) { 88 | 89 | begin = local_clock_us() 90 | 91 | if (!$chain_ptr) { 92 | printf("NULL\\n") 93 | exit() 94 | } 95 | 96 | total_size = 0 97 | nlinks = 0 98 | cl = $chain_ptr 99 | while (cl) { 100 | nlinks++; 101 | 102 | buf = $cl->buf 103 | 104 | if ($buf->temporary || $buf->memory || $buf->mmap) { 105 | size = $buf->last - $buf->pos 106 | printf("[%s]", text_str(user_string_n($buf->pos, size))) 107 | total_size += size 108 | 109 | } else { 110 | printf("\\"\\"") 111 | } 112 | 113 | if ($buf->last_buf) { 114 | printf("") 115 | } 116 | 117 | if ($buf->last_in_chain) { 118 | printf("") 119 | } 120 | 121 | if ($buf->sync) { 122 | printf("") 123 | } 124 | 125 | if ($buf->flush) { 126 | printf("") 127 | } 128 | 129 | if ($buf->in_file) { 130 | printf("") 131 | } 132 | 133 | cl = $cl->next 134 | if (cl) { 135 | printf(" ") 136 | } 137 | } 138 | 139 | printf("\\n\\nFor total %d chain links and %d bytes data found.\\n\\n", 140 | nlinks, total_size) 141 | 142 | elapsed = local_clock_us() - begin 143 | printf("%d microseconds elapsed in the probe handler.\\n", elapsed) 144 | exit() 145 | } /* pid() == target() */ 146 | } 147 | _EOC_ 148 | 149 | if ($opts{d}) { 150 | print $stap_src; 151 | exit; 152 | } 153 | 154 | open my $in, "|stap --skip-badvars -x $pid $stap_args -" 155 | or die "Cannot run stap: $!\n"; 156 | 157 | print $in $stap_src; 158 | 159 | close $in; 160 | 161 | sub usage { 162 | return <<'_EOC_'; 163 | Usage: 164 | ngx-chain-bufs [optoins] 165 | 166 | Options: 167 | -a Pass extra arguments to the stap utility. 168 | -d Dump out the systemtap script source. 169 | -h Print this usage. 170 | -p Specify the nginx (worker) process pid. 171 | 172 | Examples: 173 | ngx-chain-bufs -p 12345 0x6789a 174 | ngx-chain-bufs -p 12345 -a '-DMAXACTION=100000' 0x6789a 175 | _EOC_ 176 | } 177 | 178 | -------------------------------------------------------------------------------- /ngx-cycle-pool: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | if ($^O ne 'linux') { 31 | die "Only linux is supported but I am on $^O.\n"; 32 | } 33 | 34 | my $exec_file = "/proc/$pid/exe"; 35 | if (!-f $exec_file) { 36 | die "Nginx process $pid is not running or ", 37 | "you do not have enough permissions.\n"; 38 | } 39 | 40 | my $nginx_path = readlink $exec_file; 41 | 42 | my $ver = `stap --version 2>&1`; 43 | if (!defined $ver) { 44 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 45 | } 46 | 47 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 48 | my $v = $1; 49 | if ($v < 2.1) { 50 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 51 | } 52 | 53 | } else { 54 | die "ERROR: unknown version of systemtap:\n$ver\n"; 55 | } 56 | 57 | my $stap_src; 58 | 59 | my $preamble = <<_EOC_; 60 | probe begin { 61 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 62 | } 63 | _EOC_ 64 | chop $preamble; 65 | 66 | $stap_src = <<_EOC_; 67 | $preamble 68 | 69 | probe process("$nginx_path").function("ngx_process_events_and_timers"), 70 | process("$nginx_path").function("ngx_http_handler") 71 | { 72 | if (pid() == target()) { 73 | begin = local_clock_us() 74 | 75 | ngx_pool_t_size = &\@cast(0, "ngx_pool_t")[1] 76 | /* printf("pool t size: %d\\n", ngx_pool_t_size) */ 77 | 78 | size_t_size = &\@cast(0, "size_t")[1] 79 | /* printf("size_t size: %d\\n", size_t_size) */ 80 | 81 | pool = \@var("ngx_cycle\@ngx_cycle.c")->pool 82 | 83 | if (!pool) { 84 | error("The Nginx cycle pool is null.") 85 | exit() 86 | } 87 | 88 | end = \@cast(pool, "ngx_pool_t")->d->end 89 | 90 | printf("pool chunk size: %d\\n", end - pool) 91 | 92 | lim = \@cast(pool, "ngx_pool_t")->max + 1 93 | 94 | /* analyze small blocks */ 95 | 96 | used = 0 97 | unused = 0 98 | 99 | p = pool 100 | n = \@cast(pool, "ngx_pool_t")->d->next 101 | 102 | for ( ;; ) { 103 | 104 | last = \@cast(p, "ngx_pool_t")->d->last 105 | end = \@cast(p, "ngx_pool_t")->d->end 106 | 107 | used += last - p - ngx_pool_t_size 108 | unused += end - last 109 | 110 | /* printf("used: %d, unused %d\\n", 111 | last - p - ngx_pool_t_size, end - last) */ 112 | 113 | if (n == 0) { 114 | break 115 | } 116 | 117 | p = n 118 | n = \@cast(n, "ngx_pool_t")->d->next 119 | } 120 | 121 | printf("small blocks (< %d): %d bytes used, %d bytes unused\\n", 122 | lim, used, unused) 123 | 124 | /* analyze large blocks */ 125 | 126 | total = 0 127 | blocks = 0 128 | 129 | for (l = \@cast(pool, "ngx_pool_t")->large; 130 | l; 131 | l = \@cast(l, "ngx_pool_large_t")->next) 132 | { 133 | ptr = \@cast(l, "ngx_pool_large_t")->alloc 134 | if (ptr) { 135 | blocks++ 136 | 137 | /* XXX specific to the glibc malloc implementation */ 138 | ptr -= size_t_size 139 | block_size = \@cast(ptr, "size_t")[0] & ~(size_t_size - 1) 140 | /* printf("large block size: %d %d\\n", 141 | \@cast(ptr, "size_t")[0], block_size) */ 142 | 143 | total += block_size 144 | } 145 | } 146 | 147 | printf("large blocks (>= %d): %d blocks, %d bytes (used)\\n", 148 | lim, blocks, total) 149 | 150 | /* total summary */ 151 | 152 | printf("total used: %d bytes\\n", total + used) 153 | 154 | elapsed = local_clock_us() - begin 155 | printf("\\n%d microseconds elapsed in the probe handler.\\n", elapsed) 156 | exit() 157 | } /* pid() == target() */ 158 | } 159 | _EOC_ 160 | 161 | if ($opts{d}) { 162 | print $stap_src; 163 | exit; 164 | } 165 | 166 | open my $in, "|stap -x $pid $stap_args -" 167 | or die "Cannot run stap: $!\n"; 168 | 169 | print $in $stap_src; 170 | 171 | close $in; 172 | 173 | sub usage { 174 | return <<'_EOC_'; 175 | Usage: 176 | ngx-cycle-pool [optoins] 177 | 178 | Options: 179 | -a Pass extra arguments to the stap utility. 180 | -d Dump out the systemtap script source. 181 | -h Print this usage. 182 | -p Specify the nginx (worker) process pid. 183 | 184 | Examples: 185 | ngx-cycle-pool -p 12345 186 | ngx-cycle-pool -p 12345 -a '-DMAXACTION=100000' 187 | _EOC_ 188 | } 189 | 190 | -------------------------------------------------------------------------------- /ngx-filter-lua-vm-bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | my @bt; 7 | my $bt; 8 | my $found; 9 | while (<>) { 10 | if (/^\t\d+$/) { 11 | if ($found) { 12 | print $_; 13 | } 14 | undef $bt; 15 | undef $found; 16 | next; 17 | } 18 | 19 | if (!$found) { 20 | if (/\b(?:ngx_http_lua_run_thread|ngx_http_lua_log_by_chunk)/) { 21 | $bt .= $_; 22 | print $bt; 23 | undef $bt; 24 | $found = 1; 25 | next; 26 | } 27 | 28 | if (// && $bt && $bt =~ /\blj_\w+/s) { 29 | $bt .= $_; 30 | print $bt; 31 | undef $bt; 32 | $found = 1; 33 | next; 34 | } 35 | 36 | $bt .= $_; 37 | 38 | } else { 39 | # ignore 40 | next; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /ngx-filter-zlib-bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | my @bt; 7 | my $bt; 8 | my $found; 9 | while (<>) { 10 | if (/^\t\d+$/) { 11 | if ($found) { 12 | print $_; 13 | } 14 | undef $bt; 15 | undef $found; 16 | next; 17 | } 18 | 19 | if (!$found) { 20 | if (/\b(?:ngx_http_gzip_body_filter)/) { 21 | $bt .= $_; 22 | print $bt; 23 | undef $bt; 24 | $found = 1; 25 | next; 26 | } 27 | 28 | $bt .= $_; 29 | 30 | } else { 31 | # ignore 32 | next; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ngx-header-filters: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:n:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | if ($^O ne 'linux') { 31 | die "Only linux is supported but I am on $^O.\n"; 32 | } 33 | 34 | my $exec_file = "/proc/$pid/exe"; 35 | if (!-f $exec_file) { 36 | die "Nginx process $pid is not running or ", 37 | "you do not have enough permissions.\n"; 38 | } 39 | 40 | my $nginx_path = readlink $exec_file; 41 | 42 | my $ver = `stap --version 2>&1`; 43 | if (!defined $ver) { 44 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 45 | } 46 | 47 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 48 | my $v = $1; 49 | if ($v < 2.1) { 50 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 51 | } 52 | 53 | } else { 54 | die "ERROR: unknown version of systemtap:\n$ver\n"; 55 | } 56 | 57 | my $stap_src; 58 | 59 | my $zone = $opts{n}; 60 | 61 | my $preamble = <<_EOC_; 62 | probe begin { 63 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 64 | } 65 | _EOC_ 66 | chop $preamble; 67 | 68 | $stap_src = <<_EOC_; 69 | $preamble 70 | 71 | probe process("$nginx_path").function("ngx_http_header_filter") 72 | { 73 | if (pid() == target()) { 74 | #println("=============================") 75 | 76 | begin = local_clock_us() 77 | res = "" 78 | top_filter = usymname(\@var("ngx_http_top_header_filter")) 79 | #printf("top filter: %s\\n", top_filter) 80 | 81 | b = ubacktrace(); 82 | token = tokenize(b, " "); 83 | if (token != "") { 84 | found = 0 85 | for ( ;; ) { 86 | token = tokenize("", " "); 87 | if (token == "") { 88 | break; 89 | } 90 | 91 | addr = strtol(token, 16) 92 | sym = usymname(addr) 93 | 94 | #printf("found sym: %s (%p)\\n", sym, addr) 95 | 96 | res = sym . "\\n" . res 97 | 98 | if (sym == top_filter) { 99 | #println("found top filter!") 100 | found = 1 101 | break 102 | } 103 | } 104 | 105 | if (!found) { 106 | res = "" 107 | } 108 | } 109 | 110 | if (res != "") { 111 | print(res) 112 | println("ngx_http_header_filter") 113 | 114 | elapsed = local_clock_us() - begin 115 | printf("\\n%d microseconds elapsed in the probe handler.\\n", elapsed) 116 | exit() 117 | } 118 | } 119 | } 120 | _EOC_ 121 | 122 | if ($opts{d}) { 123 | print $stap_src; 124 | exit; 125 | } 126 | 127 | open my $in, "|stap --skip-badvars $stap_args -x $pid -" 128 | or die "Cannot run stap: $!\n"; 129 | 130 | print $in $stap_src; 131 | 132 | close $in; 133 | 134 | sub usage { 135 | return <<'_EOC_'; 136 | Usage: 137 | ngx-header-filters [optoins] 138 | 139 | Options: 140 | -a Pass extra arguments to the stap utility. 141 | -d Dump out the systemtap script source. 142 | -h Print this usage. 143 | -p Specify the nginx worker process pid. 144 | 145 | Examples: 146 | ngx-header-filters -p 12345 147 | _EOC_ 148 | } 149 | 150 | -------------------------------------------------------------------------------- /ngx-leaked-pools: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx master process pid specified by the -p option.\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid.\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | if ($^O ne 'linux') { 31 | die "Only linux is supported but I am on $^O.\n"; 32 | } 33 | 34 | my $exec_file = "/proc/$pid/exe"; 35 | if (!-f $exec_file) { 36 | die "Nginx process $pid is not running or ", 37 | "you do not have enough permissions.\n"; 38 | } 39 | 40 | my $nginx_path = readlink $exec_file; 41 | 42 | my $ver = `stap --version 2>&1`; 43 | if (!defined $ver) { 44 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 45 | } 46 | 47 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 48 | my $v = $1; 49 | if ($v < 2.1) { 50 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 51 | } 52 | 53 | } else { 54 | die "ERROR: unknown version of systemtap:\n$ver\n"; 55 | } 56 | 57 | #warn "Nginx worker processes: @$child_pids\n"; 58 | 59 | my $stap_src; 60 | 61 | my $preamble = <<_EOC_; 62 | probe begin { 63 | printf("Tracing %d ($nginx_path)...\\nHit Ctrl-C to end.\\n", target()) 64 | } 65 | _EOC_ 66 | 67 | $stap_src = <<_EOC_; 68 | $preamble 69 | global pools 70 | global btcount 71 | global total_count 72 | 73 | probe process("$nginx_path").provider("nginx").mark("create-pool-done") 74 | { 75 | if (pid() == target()) { 76 | bt = ubacktrace() 77 | 78 | pools[\$arg1] = bt 79 | btcount[bt]++ 80 | 81 | #printf("bt: %s, key: %s\\n", bt, key) 82 | #key = sprintf("%x", \$arg1) 83 | #printf("create pool: %s (bt: %s)\\n", key, ubacktrace()) 84 | 85 | total_count++ 86 | } 87 | } 88 | 89 | probe process("$nginx_path").function("ngx_destroy_pool") 90 | { 91 | if (pid() == target()) { 92 | bt = pools[\$pool] 93 | if (bt != "") { 94 | #printf("destroy pool: %s\\n", key) 95 | if (--btcount[bt] == 0) { 96 | delete btcount[bt] 97 | } 98 | delete pools[\$pool] 99 | 100 | } else { 101 | #printf("destroy pool: %s (bt: %s)\\n", key, bt) 102 | } 103 | } 104 | } 105 | 106 | probe end { 107 | printf("\\n") 108 | 109 | hits = 0 110 | foreach (bt in btcount- limit 10) { 111 | printf("%d pools leaked at backtrace %s\\n", btcount[bt], bt) 112 | hits++ 113 | } 114 | 115 | if (hits) { 116 | printf("\\nRun the command \\"./ngx-backtrace -p %d \\" to get details.\\n", 117 | target()) 118 | } else { 119 | println("\\nNo leaked pools found.") 120 | } 121 | 122 | printf("For total %d pools allocated.\\n", total_count) 123 | } 124 | _EOC_ 125 | 126 | if ($opts{d}) { 127 | print $stap_src; 128 | exit; 129 | } 130 | 131 | open my $in, "|stap --skip-badvars -x $pid --ldd $stap_args -" 132 | or die "Cannot run stap: $!\n"; 133 | 134 | print $in $stap_src; 135 | 136 | close $in; 137 | 138 | sub usage { 139 | return <<'_EOC_'; 140 | Usage: 141 | ngx-leaked-pools [optoins] 142 | 143 | Options: 144 | -a Pass extra arguments to the stap utility. 145 | -d Dump out the systemtap script source. 146 | -h Print this usage. 147 | -p Specify the nginx (worker) process pid. 148 | 149 | Examples: 150 | ngx-leaked-pools -p 12345 151 | ngx-leaked-pools -p 12345 -a '-DMAXACTION=100000' 152 | _EOC_ 153 | } 154 | 155 | -------------------------------------------------------------------------------- /ngx-lua-bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Long qw( GetOptions ); 10 | 11 | GetOptions("a=s", \(my $stap_args), 12 | "d", \(my $dump_src), 13 | "h", \(my $help), 14 | "p=i", \(my $pid), 15 | "lua51", \(my $lua51), 16 | "luajit20", \(my $luajit20)) 17 | or die usage(); 18 | 19 | if ($help) { 20 | print usage(); 21 | exit; 22 | } 23 | 24 | if (!$luajit20 && !$lua51) { 25 | die "You have to specify either the --lua51 or --luajit20 options.\n"; 26 | } 27 | 28 | if ($luajit20 && $lua51) { 29 | die "You cannot specify --lua51 and --luajit20 at the same time.\n"; 30 | } 31 | 32 | if (!defined $pid) { 33 | die "No nginx master process pid specified by the -p option.\n"; 34 | } 35 | 36 | if (!defined $stap_args) { 37 | $stap_args = ''; 38 | } 39 | 40 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 41 | $stap_args .= " -DMAXACTION=100000" 42 | } 43 | 44 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 45 | $stap_args .= " -DMAXMAPENTRIES=5000" 46 | } 47 | 48 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 49 | $stap_args .= " -DMAXBACKTRACE=200" 50 | } 51 | 52 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 53 | $stap_args .= " -DMAXSTRINGLEN=2048" 54 | } 55 | 56 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSKIPPED=1024/) { 57 | $stap_args .= " -DMAXSKIPPED=1024" 58 | } 59 | 60 | if ($^O ne 'linux') { 61 | die "Only linux is supported but I am on $^O.\n"; 62 | } 63 | 64 | my $exec_file = "/proc/$pid/exe"; 65 | if (!-f $exec_file) { 66 | die "Nginx process $pid is not running or ", 67 | "you do not have enough permissions.\n"; 68 | } 69 | 70 | my $nginx_path = readlink $exec_file; 71 | 72 | my $condition = "target() == pid()"; 73 | 74 | my $ver = `stap --version 2>&1`; 75 | if (!defined $ver) { 76 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 77 | } 78 | 79 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 80 | my $v = $1; 81 | if ($v < 2.1) { 82 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 83 | } 84 | 85 | } else { 86 | die "ERROR: unknown version of systemtap:\n$ver\n"; 87 | } 88 | 89 | my $context; 90 | if ($lua51) { 91 | $context = "standard Lua 5.1"; 92 | 93 | } elsif ($luajit20) { 94 | $context = "LuaJIT 2.0"; 95 | } 96 | 97 | my $preamble = <<_EOC_; 98 | probe begin { 99 | warn("Tracing $pid ($nginx_path) for $context...\\n") 100 | } 101 | _EOC_ 102 | 103 | my $stap_src; 104 | 105 | my $lua_path; 106 | 107 | { 108 | my $maps_file = "/proc/$pid/maps"; 109 | open my $in, $maps_file 110 | or die "Cannot open $maps_file for reading: $!\n"; 111 | 112 | while (<$in>) { 113 | if (m{\S+\bliblua-(\d+\.\d+)\.so(?:\.\d+)*$}) { 114 | my ($path, $ver) = ($&, $1); 115 | 116 | if ($luajit20) { 117 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 118 | } 119 | 120 | if ($ver ne '5.1') { 121 | die "Nginx server $pid uses a Lua $ver library ", 122 | "but only Lua 5.1 is supported.\n"; 123 | } 124 | 125 | $lua_path = $path; 126 | last; 127 | 128 | } elsif (m{\S+\bliblua\.so(?:\.\d+)*$}) { 129 | my $path = $&; 130 | 131 | if ($luajit20) { 132 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 133 | } 134 | 135 | $lua_path = $path; 136 | last; 137 | 138 | } elsif (m{\S+\blibluajit-(\d+\.\d+)\.so(?:\.\d+)*$}) { 139 | my ($path, $ver) = ($&, $1); 140 | 141 | if ($lua51) { 142 | die "The --lua51 option is specified but seen the LuaJIT library: $path\n"; 143 | } 144 | 145 | if ($ver ne '5.1') { 146 | die "Nginx server $pid uses a Lua $ver compatible LuaJIT library ", 147 | "but only Lua 5.1 is supported.\n"; 148 | } 149 | 150 | $lua_path = $path; 151 | last; 152 | } 153 | } 154 | 155 | close $in; 156 | 157 | if (!defined $lua_path) { 158 | #warn "FALL BACK TO NGINX PATH"; 159 | $lua_path = $nginx_path; 160 | } 161 | } 162 | 163 | my $probes = <<_EOC_; 164 | process("$lua_path").function("luaL_*"), 165 | process("$lua_path").function("lua_*"), 166 | process("$nginx_path").function("ngx_http_lua_ngx_*") 167 | _EOC_ 168 | 169 | if ($lua51) { 170 | my $cl = qq{\@cast(cl, "Closure", "$lua_path")}; 171 | my $ci = qq{\@cast(ci, "CallInfo", "$lua_path")}; 172 | my $p = qq{\@cast(p, "Proto", "$lua_path")}; 173 | my $L = qq{\@cast(L, "lua_State", "$lua_path")}; 174 | my $sizeof_Instruction = qq{&\@cast(0, "Instruction", "$lua_path")[1]}; 175 | 176 | $stap_src = <<_EOC_; 177 | $preamble 178 | 179 | global ci_offset = 0 180 | global cfuncs 181 | 182 | 183 | function clvalue(o) { 184 | return &\@cast(o, "TValue", "$lua_path")->value->gc->cl; 185 | } 186 | 187 | 188 | function ci_func(ci) { 189 | return clvalue(\@cast(ci, "CallInfo", "$lua_path")->func) 190 | } 191 | 192 | 193 | function f_isLua(ci) { 194 | f = ci_func(ci) 195 | if (f == 0) { 196 | return 0; 197 | } 198 | 199 | //printf("f_isLua: ci=%x, f=%x, c=%p\\n", ci, f, &\@cast(f, "Closure", "$lua_path")->c) 200 | //print_ubacktrace() 201 | return !\@cast(f, "Closure", "$lua_path")->c->isC 202 | } 203 | 204 | 205 | function getstr(ts) { 206 | return user_string(&\@cast(ts, "TString", "$lua_path")[1]) 207 | } 208 | 209 | 210 | function funcinfo(cl) { 211 | //printf("funcinfo: cl: %x\\n", cl) 212 | if ($cl->c->isC) { 213 | cfunc = $cl->c->f 214 | sym = cfuncs[cfunc] 215 | if (sym != "") { 216 | info = sym 217 | 218 | } else { 219 | sym = "C:" . usymname(cfunc) 220 | cfuncs[cfunc] = sym 221 | info = sym 222 | } 223 | 224 | } else { 225 | src = $cl->l->p->source 226 | return getstr(src) 227 | } 228 | 229 | return info 230 | } 231 | 232 | 233 | function pcRel(pc, p) { 234 | //printf("sizeof ptr: %d\\n", $sizeof_Instruction) 235 | return (pc - $p->code) / $sizeof_Instruction - 1 236 | } 237 | 238 | 239 | function currentpc(L, ci) { 240 | cl = ci_func(ci) 241 | if ($cl->c->isC) { 242 | return -1 243 | } 244 | 245 | if (ci == $L->ci) { 246 | savedpc = $L->savedpc 247 | 248 | } else { 249 | savedpc = $ci->savedpc 250 | } 251 | 252 | cl = ci_func(ci) 253 | return pcRel(savedpc, $cl->l->p) 254 | } 255 | 256 | 257 | function getline(p, pc) { 258 | //printf("p: %p, lineinfo: %p\\n", p, $p->lineinfo) 259 | return $p->lineinfo ? $p->lineinfo[pc] : 0 260 | } 261 | 262 | 263 | function currentline(L, ci) { 264 | pc = currentpc(L, ci) 265 | if (pc < 0) { 266 | return -1 267 | } 268 | //printf("pc = %d\\n", pc) 269 | cl = ci_func(ci) 270 | return getline($cl->l->p, pc) 271 | } 272 | 273 | 274 | function lua_getinfo(L, i_ci) { 275 | ci = 0 /* CallInfo *ci */ 276 | f = 0 /* Closure *f */ 277 | 278 | if (i_ci != 0) { 279 | base_ci = $L->base_ci 280 | ci = base_ci + i_ci; 281 | f = ci_func(ci) 282 | //printf("lua_getinfo: ci=%x, f=%x, isLua=%d\\n", ci, f, f_isLua(ci)); 283 | } 284 | 285 | if (f == 0) { 286 | /* info_tailcall() */ 287 | return "[tail]" 288 | } 289 | 290 | /* f != 0 */ 291 | finfo = funcinfo(f) 292 | 293 | lineno = currentline(L, ci) 294 | 295 | //fname = getfuncname(L, ci) 296 | 297 | if (lineno == -1) { 298 | return finfo 299 | } 300 | 301 | return sprintf("%s:%d", finfo, lineno) 302 | } 303 | 304 | 305 | function lua_getstack(L, level) { 306 | ci = $L->ci 307 | base_ci = $L->base_ci 308 | 309 | //printf("L=%x, ci=%x, base_ci=%x\\n", L, ci, base_ci) 310 | if (ci_offset == 0) { 311 | ci_offset = &\@cast(0, "CallInfo", "$lua_path")[1] 312 | } 313 | 314 | //printf("ci offset: %d\\n", ci_offset) 315 | 316 | for (; level > 0 && ci > base_ci; ci -= ci_offset) { 317 | level--; 318 | 319 | //tt = \@cast(ci, "CallInfo", "$lua_path")->func->tt 320 | //printf("ci tt: %d\\n", tt) 321 | 322 | if (f_isLua(ci)) { /* Lua function? */ 323 | tailcalls = \@cast(ci, "CallInfo", "$lua_path")->tailcalls 324 | //printf("it is a lua func! tailcalls=%d\\n", tailcalls) 325 | level -= tailcalls; /* skip lost tail calls */ 326 | } 327 | } 328 | 329 | if (level == 0 && ci > base_ci) { /* level found? */ 330 | //printf("lua_getstack: ci=%x\\n", ci); 331 | 332 | //tt = \@cast(ci, "CallInfo", "$lua_path")->func->tt 333 | //printf("ci tt: %d\\n", tt) 334 | 335 | //ff = &\@cast(ci, "CallInfo", "$lua_path")->func->value->gc->cl 336 | 337 | //isC = \@cast(ci, "CallInfo", "$lua_path")->func->value->gc->cl->c->isC 338 | //printf("isC: %d, %d ff=%x\\n", isC, \@cast(ff, "Closure", "$lua_path")->c->isC, ff) 339 | 340 | //f = ci_func(ci) 341 | //printf("lua_getstack: ci=%x, f=%x, isLua=%d\\n", ci, f, f_isLua(ci)); 342 | 343 | return ci - base_ci; 344 | } 345 | 346 | if (level < 0) { /* level is of a lost tail call? */ 347 | return 0; 348 | } 349 | 350 | return -1; 351 | } 352 | 353 | 354 | probe 355 | process("$lua_path").function("str_find"), 356 | $probes 357 | { 358 | if (\@defined(\$L) && $condition) { 359 | L = \$L 360 | //printf("HERE: %d\\n", L) 361 | if (L) { 362 | //println("===============") 363 | stack = "" 364 | level = 0 365 | prev_is_tail = 0 366 | while (1) { 367 | //printf("--- begin: l=%d, u=%d\\n", level, user_mode()) 368 | i_ci = lua_getstack(L, level++) 369 | 370 | //printf("lua_getstack returned: %d\\n", i_ci) 371 | 372 | if (i_ci < 0 || level > 200) { 373 | break 374 | } 375 | 376 | //printf("%d: i_ci: %s\\n", level, lua_getinfo(L, i_ci)) 377 | frame = lua_getinfo(L, i_ci) 378 | if (frame == "[tail]") { 379 | if (prev_is_tail) { 380 | continue 381 | } 382 | 383 | prev_is_tail = 1 384 | 385 | } else { 386 | prev_is_tail = 0 387 | } 388 | 389 | stack .= frame . "\\n" 390 | } 391 | 392 | if (stack != "") { 393 | print(stack) 394 | exit() 395 | } 396 | } 397 | } 398 | } 399 | _EOC_ 400 | 401 | } else { 402 | # LuaJIT 2.0 403 | my $L = qq{\@cast(L, "lua_State", "$lua_path")}; 404 | my $mref = qq{\@cast(mref, "MRef", "$lua_path")}; 405 | my $tvalue = qq{\@cast(tvalue, "TValue", "$lua_path")}; 406 | my $gcr = qq{\@cast(gcr, "GCRef", "$lua_path")}; 407 | #my $pc = qq{\@cast(pc, "uint32_t", "$lua_path")}; # BCins is uint32_t 408 | my $sizeof_TValue = qq{\&\@cast(0, "TValue", "$lua_path")[1]}; 409 | my $fn = qq{\@cast(fn, "GCfunc", "$lua_path")}; 410 | my $pt = qq{\@cast(pt, "GCproto", "$lua_path")}; 411 | my $gcobj = qq{\@cast(gcobj, "GCobj", "$lua_path")}; 412 | my $sizeof_GCproto = qq{\&\@cast(0, "GCproto", "$lua_path")[1]}; 413 | my $sizeof_GCstr = qq{\&\@cast(0, "GCstr", "$lua_path")[1]}; 414 | my $sizeof_ptr = qq{\&\@cast(0, "global_State", "$lua_path")->strmask}; 415 | my $sizeof_BCIns = qq{\&\@cast(0, "BCIns", "$lua_path")[1]}; 416 | my $NO_BCPOS = "~0"; 417 | my $FRAME_LUA = '0'; 418 | my $FRAME_C = '1'; 419 | my $FRAME_CONT = '2'; 420 | my $CFRAME_RESUME = 1; 421 | my $CFRAME_UNWIND_FF = 2; 422 | my $CFRAME_OFS_PREV = 400; 423 | my $CFRAME_OFS_PC = 412; 424 | 425 | $stap_src = <<_EOC_; 426 | $preamble 427 | 428 | global cfuncs 429 | 430 | 431 | /* 432 | function dd(s) { 433 | //printf("--- %s\\n", s) 434 | } 435 | */ 436 | 437 | 438 | function tvref(mref) { 439 | return $mref->ptr32 440 | } 441 | 442 | 443 | function gcref(gcr) { 444 | return $gcr->gcptr32 445 | } 446 | 447 | 448 | function frame_gc(tvalue) { 449 | return gcref(\&$tvalue->fr->func) 450 | } 451 | 452 | 453 | function frame_ftsz(tvalue) { 454 | return $tvalue->fr->tp->ftsz 455 | } 456 | 457 | 458 | function frame_type(f) { 459 | /* FRAME_TYPE == 3 */ 460 | return frame_ftsz(f) & 3 461 | } 462 | 463 | 464 | function frame_typep(f) { 465 | /* FRAME_TYPEP == 7 */ 466 | return frame_ftsz(f) & 7 467 | } 468 | 469 | 470 | function frame_islua(f) { 471 | return frame_type(f) == $FRAME_LUA 472 | } 473 | 474 | 475 | function frame_pc(tvalue) { 476 | return $tvalue->fr->tp->pcr->ptr32 477 | } 478 | 479 | 480 | function bc_a(i) { 481 | //dd(sprintf("instruction %d", i)) 482 | return (i >> 8) & 0xff 483 | } 484 | 485 | 486 | function frame_prevl(f) { 487 | pc = frame_pc(f) 488 | return f - (1 + bc_a(user_uint32(pc - 4))) * $sizeof_TValue 489 | } 490 | 491 | 492 | function frame_isvarg(f) { 493 | /* FRAME_VARG == 3 */ 494 | return frame_typep(f) == 3 495 | } 496 | 497 | 498 | function frame_sized(f) { 499 | /* 500 | * FRAME_TYPE == 3 501 | * FRAME_P == 4 502 | * FRAME_TYPEP == (FRAME_TYPE | FRAME_P) 503 | */ 504 | 505 | return frame_ftsz(f) & ~(3|4) 506 | } 507 | 508 | 509 | function frame_prevd(f) { 510 | return f - frame_sized(f) 511 | } 512 | 513 | 514 | function lua_getstack(L, level) { 515 | /* code from function lj_debug_frame in LuaJIT 2.0 */ 516 | 517 | /* TValue *frame, *nextframe, *bot; */ 518 | bot = tvref(\&$L->stack) // TValue * 519 | found_frame = 0 520 | 521 | for (nextframe = frame = $L->base - $sizeof_TValue; frame > bot; ) { 522 | //dd(sprintf("checking frame: %d, level: %d", frame, level)) 523 | 524 | /* Traverse frames backwards */ 525 | if (frame_gc(frame) == L) { 526 | //dd("Skip dummy frames. See lj_meta_call") 527 | level++ 528 | } 529 | 530 | if (level-- == 0) { 531 | //dd(sprintf("Level found, frame=%p, nextframe=%p", frame, nextframe)) 532 | size = (nextframe - frame) / $sizeof_TValue 533 | found_frame = 1 534 | break 535 | } 536 | 537 | nextframe = frame 538 | if (frame_islua(frame)) { 539 | frame = frame_prevl(frame) 540 | 541 | } else { 542 | if (frame_isvarg(frame)) { 543 | //dd("Skip vararg pseudo-frame") 544 | level++ 545 | } 546 | 547 | frame = frame_prevd(frame) 548 | } 549 | } 550 | 551 | if (!found_frame) { 552 | //dd("Level not found") 553 | size = level 554 | frame = 0 555 | } 556 | 557 | /* code from function lua_getstatck in LuaJIT 2.0 */ 558 | 559 | if (frame) { 560 | i_ci = (size << 16) + (frame - bot) / $sizeof_TValue 561 | return i_ci 562 | } 563 | 564 | return -1 565 | } 566 | 567 | 568 | function frame_func(f) { 569 | gcobj = frame_gc(f) 570 | return \&$gcobj->fn 571 | } 572 | 573 | 574 | function isluafunc(fn) { 575 | /* FF_LUA == 0 */ 576 | return $fn->c->ffid == 0 577 | } 578 | 579 | 580 | function funcproto(fn) { 581 | return $fn->l->pc->ptr32 - $sizeof_GCproto 582 | } 583 | 584 | 585 | function strref(r) { 586 | gcobj = gcref(r) 587 | return \&$gcobj->str 588 | } 589 | 590 | 591 | function proto_chunkname(pt) { 592 | return strref(\&$pt->chunkname) 593 | } 594 | 595 | 596 | function strdata(s) { 597 | return s + $sizeof_GCstr 598 | } 599 | 600 | //proto_bc(pt) ((BCIns *)((char *)(pt) + sizeof(GCproto))) 601 | function proto_bc(pt) { 602 | return pt + $sizeof_GCproto; 603 | } 604 | 605 | 606 | // proto_bcpos(pt, pc) ((BCPos)((pc) - proto_bc(pt))) 607 | function proto_bcpos(pt, pc) { 608 | //printf("sizeof BCIns: %d\\n", $sizeof_BCIns) 609 | //printf("proto_bc(pt): %p\\n", proto_bc(pt)) 610 | return (pc - proto_bc(pt)) / $sizeof_BCIns 611 | } 612 | 613 | 614 | function frame_iscont(f) { 615 | return frame_typep(f) == $FRAME_CONT 616 | } 617 | 618 | 619 | function frame_contpc(f) { 620 | return frame_pc(f) - 1 * $sizeof_BCIns 621 | } 622 | 623 | 624 | function frame_isc(f) { 625 | return frame_type(f) == $FRAME_C 626 | } 627 | 628 | 629 | function cframe_raw(cf) { 630 | return cf & ~(1|2) 631 | } 632 | 633 | 634 | // #define cframe_prev(cf) (*(void **)(((char *)(cf))+CFRAME_OFS_PREV)) 635 | function cframe_prev(cf) { 636 | return user_long(cf + $CFRAME_OFS_PREV) 637 | } 638 | 639 | 640 | // (mref(*(MRef *)(((char *)(cf))+CFRAME_OFS_PC), const BCIns)) 641 | function cframe_pc(cf) { 642 | return cf + $CFRAME_OFS_PC 643 | } 644 | 645 | 646 | function debug_framepc(L, fn, nextframe) { 647 | //printf("debug framepc: L=%p, fn=%p, nextframe = %p\\n", L, fn, nextframe) 648 | if (nextframe == 0) { 649 | return $NO_BCPOS 650 | } 651 | 652 | if (frame_islua(nextframe)) { 653 | ins = frame_pc(nextframe); 654 | //printf("frame is lua, ins = %p\\n", ins) 655 | 656 | } else if (frame_iscont(nextframe)) { 657 | //println("frame is cont") 658 | ins = frame_contpc(nextframe) 659 | 660 | } else { 661 | //println("frame is raw cframe") 662 | cf = cframe_raw($L->cframe) 663 | if (cf == 0) { 664 | return $NO_BCPOS 665 | } 666 | f = $L->base - 1 * $sizeof_TValue /* TValue* */ 667 | while (f > nextframe) { 668 | if (frame_islua(f)) { 669 | f = frame_prevl(f); 670 | 671 | } else { 672 | if (frame_isc(f)) { 673 | cf = cframe_raw(cframe_prev(cf)); 674 | } 675 | f = frame_prevd(f); 676 | } 677 | } 678 | if (cframe_prev(cf)) { 679 | cf = cframe_raw(cframe_prev(cf)); 680 | } 681 | ins = cframe_pc(cf); 682 | } 683 | 684 | //printf("debug framepc: ins = %p\\n", ins) 685 | pt = funcproto(fn); 686 | return proto_bcpos(pt, ins) - 1; 687 | } 688 | 689 | 690 | function proto_lineinfo(pt) { 691 | return $pt->lineinfo->ptr32 692 | } 693 | 694 | 695 | function lj_debug_line(pt, pc) { 696 | lineinfo = proto_lineinfo(pt) 697 | //printf("lj_debug_line: lineinfo = %p, %x <= %x\\n", lineinfo, 698 | //pc, $pt->sizebc) 699 | if (pc <= $pt->sizebc && lineinfo) { 700 | first = $pt->firstline 701 | if (pc == $pt->sizebc) { 702 | return first + $pt->numline 703 | } 704 | if (pc-- == 0) { 705 | return first 706 | } 707 | if ($pt->numline < 256) { 708 | return first + \@cast(lineinfo, "uint8_t", "$lua_path")[pc] 709 | } 710 | if ($pt->numline < 65536) { 711 | return first + \@cast(lineinfo, "uint16_t", "$lua_path")[pc] 712 | } 713 | return first + \@cast(lineinfo, "uint32_t", "$lua_path")[pc] 714 | } 715 | return 0 716 | } 717 | 718 | 719 | function debug_frameline(L, fn, nextframe) { 720 | pc = debug_framepc(L, fn, nextframe) 721 | //printf("debug frameline: pc = %p\\n", pc) 722 | if (pc != $NO_BCPOS) { 723 | pt = funcproto(fn) 724 | //printf("pc <= pt->sizebc: %x %x\\n", pt, pc) 725 | return lj_debug_line(pt, pc) 726 | } 727 | return -1 728 | } 729 | 730 | 731 | function lua_getinfo(L, i_ci) { 732 | /* code from function lj_debug_getinfo in LuaJIT 2.0 */ 733 | 734 | offset = (i_ci & 0xffff) 735 | if (offset == 0) { 736 | //dd(sprintf("assertion failed: offset == 0: i_ci=%x", i_ci)) 737 | return "" 738 | } 739 | 740 | frame = tvref(\&$L->stack) + offset * $sizeof_TValue 741 | 742 | size = (i_ci >> 16) 743 | if (size) { 744 | nextframe = frame + size * $sizeof_TValue 745 | 746 | } else { 747 | nextframe = 0 748 | } 749 | 750 | //dd(sprintf("getinfo: frame=%p nextframe=%p", frame, nextframe)) 751 | 752 | maxstack = tvref(\&$L->maxstack) 753 | if (!(frame <= maxstack && (!nextframe || nextframe <= maxstack))) { 754 | //dd("assertion failed: frame <= maxstack && (!nextframe || nextframe <= maxstack)") 755 | return "" 756 | } 757 | 758 | fn = frame_func(frame) 759 | 760 | /* LJ_TFUNC == ~8u */ 761 | if (!($fn->c->gct == 8)) { 762 | //dd(sprintf("assertion failed: fn->c.gct == ~LJ_TFUNC: %d", $fn->c->gct)) 763 | return "" 764 | } 765 | 766 | if (isluafunc(fn)) { 767 | currentline = frame ? debug_frameline(L, fn, nextframe) : -1 768 | pt = funcproto(fn) 769 | name = proto_chunkname(pt) /* GCstr *name */ 770 | src = strdata(name) 771 | if (currentline == -1) { 772 | return user_string(src) 773 | } 774 | return sprintf("%s:%d", user_string(src), currentline) 775 | } 776 | 777 | /* being a C function */ 778 | 779 | cfunc = $fn->c->f 780 | sym = cfuncs[cfunc] 781 | if (sym != "") { 782 | return sym 783 | } 784 | 785 | sym = "C:" . usymname(cfunc) 786 | cfuncs[cfunc] = sym 787 | return sym 788 | } 789 | 790 | 791 | probe 792 | process("$lua_path").function("lj_cf_string_find"), 793 | $probes 794 | { 795 | if (\@defined(\$L) && $condition) { 796 | L = \$L 797 | //printf("HERE: %d\\n", L) 798 | if (L) { 799 | //println("===============") 800 | stack = "" 801 | level = 0 802 | prev_is_tail = 0 803 | while (1) { 804 | //dd(sprintf("begin: l=%d, u=%d", level, user_mode())) 805 | i_ci = lua_getstack(L, level++) 806 | 807 | //printf("lua_getstack returned: %d\\n", i_ci) 808 | 809 | if (i_ci < 0 || level > 100) { 810 | break 811 | } 812 | 813 | //printf("%d: i_ci: %s\\n", level, lua_getinfo(L, i_ci)) 814 | frame = lua_getinfo(L, i_ci) 815 | if (frame == "") { 816 | stack = "" 817 | break 818 | } 819 | 820 | //dd(sprintf("got frame: %s", frame)) 821 | 822 | if (frame == "[tail]") { 823 | if (prev_is_tail) { 824 | continue 825 | } 826 | 827 | prev_is_tail = 1 828 | 829 | } else { 830 | prev_is_tail = 0 831 | } 832 | 833 | stack .= frame . "\\n" 834 | } 835 | 836 | if (stack != "") { 837 | print(stack) 838 | exit() 839 | } 840 | } 841 | } 842 | } 843 | _EOC_ 844 | } 845 | 846 | if ($dump_src) { 847 | print $stap_src; 848 | exit; 849 | } 850 | 851 | open my $in, "|stap --skip-badvars --all-modules -d '$nginx_path' -x $pid --ldd $stap_args -" 852 | or die "Cannot run stap: $!\n"; 853 | 854 | print $in $stap_src; 855 | 856 | close $in; 857 | 858 | sub usage { 859 | return <<'_EOC_'; 860 | Usage: 861 | ngx-lua-bt [optoins] 862 | 863 | Options: 864 | -a Pass extra arguments to the stap utility. 865 | -d Dump out the systemtap script source. 866 | -h Print this usage. 867 | --lua51 Specify that the Nginx is using the standard Lua 5.1. 868 | interpreter. 869 | --luajit20 Specify that the Nginx is using LuaJIT 2.0. 870 | -p Specify the user process pid. 871 | 872 | Examples: 873 | ngx-lua-bt --lua51 -p 12345 874 | ngx-lua-bt --luajit20 -p 12345 -a '-DMAXACTION=100000' 875 | _EOC_ 876 | } 877 | -------------------------------------------------------------------------------- /ngx-lua-conn-pools: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | # Copyright (C) Guanlan Dai 5 | 6 | use 5.006001; 7 | use strict; 8 | use warnings; 9 | use Config; 10 | 11 | use Getopt::Long qw( GetOptions ); 12 | 13 | GetOptions("a=s", \(my $stap_args), 14 | "d", \(my $dump_src), 15 | "h", \(my $help), 16 | "p=i", \(my $pid), 17 | "distr", \(my $distr), 18 | "lua51", \(my $lua51), 19 | "luajit20", \(my $luajit20)) 20 | or die usage(); 21 | 22 | if ($help) { 23 | print usage(); 24 | exit; 25 | } 26 | 27 | if (!$luajit20 && !$lua51) { 28 | die "You have to specify either the --lua51 or --luajit20 options.\n"; 29 | } 30 | 31 | if ($luajit20 && $lua51) { 32 | die "You cannot specify --lua51 and --luajit20 at the same time.\n"; 33 | } 34 | 35 | if (!$pid) { 36 | die "No nginx process pid specified by the -p option\n"; 37 | } 38 | 39 | if (!defined $stap_args) { 40 | $stap_args = ''; 41 | } 42 | 43 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 44 | $stap_args .= " -DMAXACTION=100000" 45 | } 46 | 47 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 48 | $stap_args .= " -DMAXMAPENTRIES=5000" 49 | } 50 | 51 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 52 | $stap_args .= " -DMAXBACKTRACE=200" 53 | } 54 | 55 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 56 | $stap_args .= " -DMAXSTRINGLEN=2048" 57 | } 58 | 59 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSKIPPED=1024/) { 60 | $stap_args .= " -DMAXSKIPPED=1024" 61 | } 62 | my ($probes, $operation); 63 | 64 | if ($^O ne 'linux') { 65 | die "Only linux is supported but I am on $^O.\n"; 66 | } 67 | 68 | my $arch = $Config{archname}; 69 | if ($arch !~ m/x86_64/) { 70 | die "Only x86_64 is supported, current arch is $arch .\n"; 71 | } 72 | 73 | my $nginx_file = "/proc/$pid/exe"; 74 | if (!-f $nginx_file) { 75 | die "Nginx process $pid is not running or ", 76 | "you do not have enough permissions.\n"; 77 | } 78 | 79 | my $ver = `stap -V 2>&1`; 80 | if (!defined $ver) { 81 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 82 | } 83 | 84 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 85 | my $v = $1; 86 | if ($v < 2.3) { 87 | die "ERROR: at least systemtap 2.3 is required but found $v\n"; 88 | } 89 | 90 | } else { 91 | die "ERROR: unknown version of systemtap:\n$ver\n"; 92 | } 93 | 94 | my $context; 95 | if ($lua51) { 96 | $context = "standard Lua 5.1"; 97 | 98 | } elsif ($luajit20) { 99 | $context = "LuaJIT 2.0"; 100 | } 101 | 102 | my $nginx_path = readlink $nginx_file; 103 | 104 | my $lua_path; 105 | 106 | { 107 | my $maps_file = "/proc/$pid/maps"; 108 | open my $in, $maps_file 109 | or die "Cannot open $maps_file for reading: $!\n"; 110 | 111 | while (<$in>) { 112 | if (m{\S+\bliblua-(\d+\.\d+)\.so(?:\.\d+)*$}) { 113 | my ($path, $ver) = ($&, $1); 114 | 115 | if ($luajit20) { 116 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 117 | } 118 | 119 | if ($ver ne '5.1') { 120 | die "Nginx server $pid uses a Lua $ver library ", 121 | "but only Lua 5.1 is supported.\n"; 122 | } 123 | 124 | $lua_path = $path; 125 | last; 126 | 127 | } elsif (m{\S+\bliblua\.so(?:\.\d+)*$}) { 128 | my $path = $&; 129 | 130 | if ($luajit20) { 131 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 132 | } 133 | 134 | $lua_path = $path; 135 | last; 136 | 137 | } elsif (m{\S+\blibluajit-(\d+\.\d+)\.so(?:\.\d+)*$}) { 138 | my ($path, $ver) = ($&, $1); 139 | 140 | if ($lua51) { 141 | die "The --lua51 option is specified but seen the LuaJIT library: $path\n"; 142 | } 143 | 144 | if ($ver ne '5.1') { 145 | die "Nginx server $pid uses a Lua $ver compatible LuaJIT library ", 146 | "but only Lua 5.1 is supported.\n"; 147 | } 148 | 149 | $lua_path = $path; 150 | last; 151 | } 152 | } 153 | 154 | close $in; 155 | 156 | if (!defined $lua_path) { 157 | $lua_path = $nginx_path; 158 | } 159 | } 160 | 161 | my $stap_src; 162 | 163 | my $preamble = <<_EOC_; 164 | probe begin 165 | { 166 | printf("Tracing %d ($nginx_path) for $context...\\n\\n", target()) 167 | } 168 | _EOC_ 169 | chop $preamble; 170 | 171 | my $L = qq{\@cast(L, "lua_State", "$nginx_path")}; 172 | my $pool = qq{\@cast(pool, "ngx_http_lua_socket_pool_t", "$nginx_path")}; 173 | my $pool_offset = qq{&\@cast(0, 174 | "ngx_http_lua_socket_pool_item_t", 175 | "$nginx_path")->queue}; 176 | 177 | my $print_dist_func_def; 178 | 179 | if ($distr) { 180 | $print_dist_func_def = <<_EOC_; 181 | printf("\t reused times distribution:\\n") 182 | print(\@hist_log(reused_time)) 183 | 184 | _EOC_ 185 | 186 | } else { 187 | $print_dist_func_def = ""; 188 | } 189 | 190 | my $ngx_queue_stat_func_def = <<_EOC_; 191 | function queue_len(queue) { 192 | length = 0 193 | head = queue 194 | start = \@cast(queue, "ngx_queue_t", "$nginx_path")->prev 195 | 196 | curr = start 197 | while (curr != head) { 198 | curr = \@cast(curr, "ngx_queue_t", "$nginx_path")->prev 199 | length++ 200 | } 201 | 202 | return length 203 | } 204 | 205 | function traverse_queue(queue) { 206 | head = queue 207 | start = \@cast(queue, "ngx_queue_t", "$nginx_path")->prev 208 | curr = start 209 | 210 | if (curr == head) { 211 | is_queue_empty = 1 212 | } else { 213 | is_queue_empty = 0 214 | } 215 | 216 | while (curr != head) { 217 | pool_item = curr - $pool_offset 218 | reused = \@cast(pool_item, 219 | "ngx_http_lua_socket_pool_item_t", 220 | "$nginx_path")->reused 221 | reused_time <<< reused 222 | curr = \@cast(curr, "ngx_queue_t", "$nginx_path")->prev 223 | length = length + 1 224 | } 225 | 226 | if (!is_queue_empty) { 227 | printf("\\t reused times (max/avg/min): %d/%d/%d\\n", 228 | \@max(reused_time), 229 | \@avg(reused_time), 230 | \@min(reused_time)) 231 | 232 | $print_dist_func_def 233 | } 234 | delete reused_time 235 | } 236 | 237 | _EOC_ 238 | 239 | if ($lua51) { 240 | my $USERDATA_TYPE = 7; 241 | my $sizeof_Udata = qq{&\@cast(0, "Udata", "$lua_path")[1]}; 242 | my $node = qq{\@cast(node, "Node", "$lua_path")}; 243 | $stap_src = <<_EOC_; 244 | $preamble 245 | global reused_time 246 | 247 | function size_node(table) { 248 | res = \@cast(table, "Table", "$lua_path")->lsizenode 249 | res = 1 << res 250 | return res 251 | } 252 | 253 | function hash_mod(table, key) { 254 | size = size_node(table) 255 | i = key % ((size - 1) | 1) 256 | return i 257 | } 258 | 259 | $ngx_queue_stat_func_def 260 | 261 | function print_pools_stat(pools) { 262 | pool_count = 0 263 | for (i = 0; i < size_node(pools); i++) { 264 | node = &\@cast(pools, "Table", "$lua_path")->node[i] 265 | i_val = &$node->i_val 266 | if (\@cast(i_val, "TValue", "$lua_path")->tt == $USERDATA_TYPE) { 267 | pool_count++ 268 | pool = &$node->i_val->value->gc->u 269 | pool += $sizeof_Udata 270 | pool_key = &$pool->key 271 | cache = &$pool->cache 272 | free = &$pool->free 273 | active_cnt = $pool->active_connections 274 | in_pool_cnt = queue_len(cache) 275 | 276 | printf("pool \\"%s\\"\\n", text_str(user_string(pool_key))) 277 | printf("\tout-of-pool reused connections: %d\\n", active_cnt - in_pool_cnt) 278 | printf("\tin-pool connections: %d\\n", in_pool_cnt) 279 | traverse_queue(cache) 280 | printf("\tpool capacity: %d\\n\\n", 281 | queue_len(cache) + queue_len(free)) 282 | } 283 | } 284 | if (pool_count > 0) { 285 | printf("For total %d connection pool(s) found.\\n", pool_count) 286 | exit() 287 | } 288 | } 289 | 290 | probe process("$nginx_path").function("ngx_http_lua_socket_tcp_send"), 291 | process("$lua_path").function("lua_resume") 292 | { 293 | L = \$L 294 | if (L) { 295 | begin = gettimeofday_us() 296 | 297 | key = &\@var("ngx_http_lua_socket_pool_key", "$nginx_path") 298 | lg = \@cast(L, "lua_State", "$lua_path")->l_G 299 | registrytv = &\@cast(lg, "global_State", "$lua_path")->l_registry 300 | table = &\@cast(registrytv, "TValue", "$lua_path")->value->gc->h 301 | index = hash_mod(table, key) 302 | pools_table = &\@cast(table, "Table", "$lua_path")->node[index]->i_val->value->gc->h 303 | pool_count = 0 304 | 305 | if (pools_table) { 306 | pool_count = print_pools_stat(pools_table) 307 | } 308 | 309 | if (pool_count > 0) { 310 | elapsed = gettimeofday_us() - begin 311 | printf("%d microseconds elapsed in the probe handler.\\n", elapsed) 312 | } 313 | } 314 | } 315 | _EOC_ 316 | 317 | } else { 318 | # LuaJIT 2.0 319 | my $HASH_BIAS = -0x04c11db7; 320 | my $HASH_ROT1 = 14; 321 | my $HASH_ROT2 = 5; 322 | my $HASH_ROT3 = 13; 323 | my $USERDATA_TYPE = ~12 & 0xFFFFFFFF; 324 | my $node = qq{\@cast(node, "Node", "$lua_path")}; 325 | my $pools = qq{\@cast(pools, "GCtab", "$lua_path")}; 326 | my $t = qq{\@cast(t, "GCtab", "$lua_path")}; 327 | my $sizeof_node = qq{&\@cast(0, "Node", "$lua_path")[1]}; 328 | my $sizeof_ud = qq{&\@cast(0, "GCudata", "$lua_path")[1]}; 329 | my $sizeof_x = qq{&\@cast(0, "uint32_t", "$nginx_path")[1]}; 330 | 331 | $stap_src = <<_EOC_; 332 | $preamble 333 | 334 | global reused_time 335 | 336 | function lj_rol(x, n) { 337 | return ((x << n) |(x >> ((8 * $sizeof_x - n)))) 338 | } 339 | 340 | function hashrot(low, high) { 341 | lo = low 342 | hi = high & 0xFFFFFFFF 343 | lo ^= hi 344 | lo = lo & 0xFFFFFFFF 345 | hi = lj_rol(hi, $HASH_ROT1) 346 | hi = hi & 0xFFFFFFFF 347 | lo -= hi 348 | lo = lo & 0xFFFFFFFF 349 | hi = lj_rol(hi, $HASH_ROT2) 350 | hi = hi & 0xFFFFFFFF 351 | hi ^= lo 352 | hi = hi & 0xFFFFFFFF 353 | hi -= lj_rol(lo, $HASH_ROT3) 354 | hi = hi & 0xFFFFFFFF 355 | return hi 356 | } 357 | 358 | function hashgcref(t, key) { 359 | low = key 360 | high = key + $HASH_BIAS 361 | hash = hashrot(low, high) 362 | n = $t->node->ptr32 363 | hmask = $t->hmask 364 | index = hash & hmask 365 | node = n + ($sizeof_node * index) 366 | return node 367 | } 368 | 369 | function lj_tab_get(t, key) { 370 | MAXACTION = 1000 371 | node = hashgcref(t, key) 372 | node_key = $node->key->gcr->gcptr32 373 | find_count = 0 374 | 375 | while (node_key != key) { 376 | node = $node->next->ptr32 377 | node_key = $node->key->gcr->gcptr32 378 | find_count++ 379 | if (find_count > MAXACTION) { 380 | return 0 381 | } 382 | } 383 | return node 384 | } 385 | 386 | $ngx_queue_stat_func_def 387 | 388 | function print_pools_stat(pools) { 389 | pool_count = 0 390 | n = $pools->node->ptr32 391 | hmask = $pools->hmask 392 | for (i = 0; i <= hmask; i++) { 393 | node = n + ($sizeof_node * i) 394 | if ($node->val->it == $USERDATA_TYPE) { 395 | pool_count++ 396 | pool_obj = $node->val->gcr->gcptr32 397 | pool = &\@cast(pool_obj, "GCobj", "$lua_path")->ud 398 | pool += $sizeof_ud 399 | pool_key = &$pool->key 400 | cache = &$pool->cache 401 | free = &$pool->free 402 | active_cnt = $pool->active_connections 403 | in_pool_cnt = queue_len(cache) 404 | 405 | printf("pool \\"%s\\"\\n", text_str(user_string(pool_key))) 406 | printf("\tout-of-pool reused connections: %d\\n", active_cnt - in_pool_cnt) 407 | printf("\tin-pool connections: %d\\n", in_pool_cnt) 408 | traverse_queue(cache) 409 | printf("\tpool capacity: %d\\n\\n", 410 | queue_len(cache) + queue_len(free)) 411 | } 412 | } 413 | 414 | if (pool_count > 0) { 415 | printf("For total %d connection pool(s) found.\\n", pool_count) 416 | exit() 417 | } 418 | return pool_count 419 | } 420 | 421 | probe process("$nginx_path").function("ngx_http_lua_socket_tcp_send") 422 | { 423 | L = \$L 424 | if (L) { 425 | begin = gettimeofday_us() 426 | 427 | key = &\@var("ngx_http_lua_socket_pool_key\@ngx_http_lua_util.c", "$nginx_path") 428 | lg = \@cast(L, "lua_State", "$lua_path")->glref->ptr32 429 | registrytv = &\@cast(lg, "global_State", "$lua_path")->registrytv 430 | gcptr = \@cast(registrytv, "TValue", "$lua_path")->gcr->gcptr32 431 | table = &\@cast(gcptr, "GCobj", "$lua_path")->tab 432 | pools_table = lj_tab_get(table, key) 433 | pool_count = 0 434 | 435 | if (pools_table) { 436 | pools_obj = \@cast(pools_table, "TValue", "$lua_path")->gcr->gcptr32 437 | pools = &\@cast(pools_obj, "GCobj", "$lua_path")->tab 438 | pool_count = print_pools_stat(pools) 439 | } 440 | 441 | if (pool_count > 0) { 442 | elapsed = gettimeofday_us() - begin 443 | printf("%d microseconds elapsed in the probe handler.\\n", elapsed) 444 | } 445 | } 446 | } 447 | _EOC_ 448 | } 449 | 450 | if ($dump_src) { 451 | print $stap_src; 452 | exit; 453 | } 454 | 455 | open my $in, "|stap --skip-badvars $stap_args -x $pid -" 456 | or die "Cannot run stap: $!\n"; 457 | 458 | print $in $stap_src; 459 | 460 | close $in; 461 | sub usage { 462 | return <<'_EOC_'; 463 | Usage: 464 | ngx-lua-conn-pools [optoins] 465 | 466 | Options: 467 | -p Specify the nginx worker process pid. 468 | -a Pass extra arguments to the stap utility. 469 | -d Dump out the systemtap script source. 470 | --distr Print the distribution of reused times of sockets. 471 | --lua51 Specify that the Nginx is using the standard Lua 5.1. 472 | interpreter. 473 | --luajit20 Specify that the Nginx is using LuaJIT 2.0. 474 | 475 | Examples: 476 | ngx-lua-conn-pools --lua51 -p 12345 477 | ngx-lua-conn-pools --luajit20 -p 12345 -a '-DMAXACTION=100000' 478 | _EOC_ 479 | } 480 | -------------------------------------------------------------------------------- /ngx-lua-shdict: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | # Copyright (C) Guanlan Dai 5 | 6 | use 5.006001; 7 | use strict; 8 | use warnings; 9 | 10 | use Getopt::Long qw( GetOptions ); 11 | 12 | GetOptions("a=s", \(my $stap_args), 13 | "d", \(my $dump_src), 14 | "h", \(my $help), 15 | "lua51", \(my $lua51), 16 | "luajit20", \(my $luajit20), 17 | "w", \(my $trace_write), 18 | "f", \(my $find_value), 19 | "p=i", \(my $pid), 20 | "raw", \(my $raw), 21 | "dict=s", \(my $dict), 22 | "key=s", \(my $key)) 23 | or die usage(); 24 | 25 | if ($help) { 26 | print usage(); 27 | exit; 28 | } 29 | 30 | if (!$pid) { 31 | die "No nginx process pid specified by the -p option\n"; 32 | } 33 | 34 | if (!defined $stap_args) { 35 | $stap_args = ''; 36 | } 37 | 38 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 39 | $stap_args .= " -DMAXACTION=100000" 40 | } 41 | 42 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 43 | $stap_args .= " -DMAXMAPENTRIES=5000" 44 | } 45 | 46 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 47 | $stap_args .= " -DMAXBACKTRACE=200" 48 | } 49 | 50 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 51 | $stap_args .= " -DMAXSTRINGLEN=2048" 52 | } 53 | 54 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSKIPPED=1024/) { 55 | $stap_args .= " -DMAXSKIPPED=1024" 56 | } 57 | 58 | if ($^O ne 'linux') { 59 | die "Only linux is supported but I am on $^O.\n"; 60 | } 61 | 62 | my $nginx_file = "/proc/$pid/exe"; 63 | if (!-f $nginx_file) { 64 | die "Nginx process $pid is not running or ", 65 | "you do not have enough permissions.\n"; 66 | } 67 | 68 | my $nginx_path = readlink $nginx_file; 69 | my $lua_path = get_lua_path(); 70 | 71 | my $ver = `stap --version 2>&1`; 72 | if (!defined $ver) { 73 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 74 | } 75 | 76 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 77 | my $v = $1; 78 | if ($v < 2.1) { 79 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 80 | } 81 | 82 | } else { 83 | die "ERROR: unknown version of systemtap:\n$ver\n"; 84 | } 85 | 86 | if (!$key) { 87 | die "ERROR: please specify a key\n"; 88 | } 89 | 90 | my $stap_src; 91 | my $preamble; 92 | 93 | if ($trace_write) { 94 | if ($dict) { 95 | die "ERROR: specify a dict when tracing writes is not supported\n"; 96 | } 97 | my $o = qq{\@cast(o, "TValue", "$lua_path")}; 98 | my $L = qq{\@cast(L, "lua_State", "$lua_path")}; 99 | my $gcr = qq{\@cast(gcr, "GCRef", "$lua_path")}; 100 | my $gcobj = qq{\@cast(gcobj, "GCobj", "$lua_path")}; 101 | my $str = qq{\@cast(str, "GCstr", "$lua_path")}; 102 | my $sizeof_TValue = qq{\&\@cast(0, "TValue", "$lua_path")[1]}; 103 | my $sizeof_GCstr = qq{\&\@cast(0, "GCstr", "$lua_path")[1]}; 104 | my $TL_TSTR = "4294967291"; 105 | my $NGX_HTTP_LUA_SHDICT_ADD = '0x0001'; 106 | my $NGX_HTTP_LUA_SHDICT_REPLACE = '0x0002'; 107 | my $NGX_HTTP_LUA_SHDICT_SAFE_STORE = '0x0004'; 108 | my $LJ_TISNUM = '0xfffeffffu'; 109 | $preamble = <<_EOC_; 110 | probe begin { 111 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 112 | printf("Hit Ctrl-C to end.\\n\\n") 113 | } 114 | _EOC_ 115 | 116 | $stap_src = <<_EOC_; 117 | $preamble 118 | 119 | function gcref(gcr) 120 | { 121 | return $gcr->gcptr32 122 | } 123 | 124 | function gcval(o) 125 | { 126 | return gcref(\&$o->gcr) 127 | } 128 | 129 | function strdata(s) 130 | { 131 | return s + $sizeof_GCstr 132 | } 133 | 134 | probe process("$nginx_path").function("ngx_http_lua_shdict_set_helper") 135 | { 136 | if (pid() == target()) { 137 | flags = \$flags 138 | if (flags & $NGX_HTTP_LUA_SHDICT_ADD) { 139 | op = "add" 140 | 141 | } else if (flags & $NGX_HTTP_LUA_SHDICT_REPLACE) { 142 | op = "replace" 143 | 144 | } else { 145 | op = "set" 146 | } 147 | 148 | if (flags & $NGX_HTTP_LUA_SHDICT_SAFE_STORE) { 149 | op = "safe_" . op 150 | } 151 | 152 | L = \$L 153 | o = $L->base + $sizeof_TValue * (2 - 1) 154 | if (o < $L->top && $o->it == $TL_TSTR) { 155 | gcobj = gcval(o) 156 | str = \&$gcobj->str 157 | //printf("gmatch regex: %s\\n", user_string_n(strdata(str), $str->len)) 158 | key = user_string_n(strdata(str), $str->len) 159 | if (key == "$key") { 160 | printf("%s %s", op, key) 161 | 162 | o = $L->base + $sizeof_TValue * (4 - 1) 163 | if (o < $L->top) { 164 | exptime = user_long(&$o->n) 165 | printf(" exptime=%d", exptime) 166 | } 167 | 168 | printf("\\n") 169 | } 170 | } 171 | } 172 | } 173 | _EOC_ 174 | 175 | } else { 176 | if (!$dict) { 177 | die "ERROR: please specify a dict zone\n"; 178 | } 179 | 180 | my $hash = crc32($key); 181 | my $dict_name_len = length $dict; 182 | my $key_len = length $key; 183 | my $quoted_zone = quote_str($dict); 184 | my $LUA_TBOOLEAN = 1; 185 | my $LUA_TNUMBER = 3; 186 | my $LUA_TSTRING = 4; 187 | my $part = qq{\@cast(part, "ngx_list_part_t")}; 188 | my $node = qq{\@cast(node, "ngx_rbtree_node_t", "$nginx_path")}; 189 | my $sd = qq{\@cast(sd, "ngx_http_lua_shdict_node_t", "$nginx_path")}; 190 | my $sh = qq{\@cast(sh, "ngx_http_lua_shdict_shctx_t", "$nginx_path")}; 191 | 192 | my $print_result_func_def; 193 | my $print_time_elasped_func_def; 194 | my $set_timer_begin; 195 | 196 | if ($raw) { 197 | $preamble = ""; 198 | 199 | $set_timer_begin = ""; 200 | 201 | $print_time_elasped_func_def = ""; 202 | 203 | $print_result_func_def = <<_EOC_; 204 | function print_result(sd) { 205 | if (sd) { 206 | value_type = $sd->value_type; 207 | data = $sd->data + $key_len; 208 | 209 | if (value_type == $LUA_TSTRING) { 210 | print(user_string(data)) 211 | 212 | } else if (value_type == $LUA_TNUMBER) { 213 | print(user_long(data)) 214 | 215 | } else if (value_type == $LUA_TBOOLEAN) { 216 | print(user_long(data)) 217 | 218 | } else { 219 | printf("ERROR: bad value type found for key $key in shared_dict $dict\\n") 220 | } 221 | 222 | } else { 223 | exit() 224 | } 225 | } 226 | _EOC_ 227 | 228 | } else { 229 | $preamble = <<_EOC_; 230 | probe begin { 231 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 232 | } 233 | _EOC_ 234 | 235 | $set_timer_begin = qq{begin = local_clock_us()}; 236 | 237 | $print_time_elasped_func_def = <<_EOC_; 238 | elapsed = local_clock_us() - begin 239 | printf("\\n%d microseconds elapsed in the probe handler.\\n", elapsed) 240 | _EOC_ 241 | 242 | $print_result_func_def = <<_EOC_; 243 | function print_result(sd) { 244 | if (sd) { 245 | value_type = $sd->value_type; 246 | data = $sd->data + $key_len; 247 | if (value_type == $LUA_TSTRING) { 248 | printf("type: LUA_TSTRING\\n") 249 | printf("value: %s\\n", text_str(user_string(data))) 250 | printf("expires: %d\\n", $sd->expires) 251 | printf("flags: 0x%x\\n", $sd->user_flags) 252 | 253 | } else if (value_type == $LUA_TNUMBER) { 254 | printf("type: LUA_TNUMBER\\n") 255 | printf("value: %d\\n", user_long(data)) 256 | printf("expires: %d\\n", $sd->expires) 257 | printf("flags: 0x%x\\n", $sd->user_flags) 258 | 259 | } else if (value_type == $LUA_TBOOLEAN) { 260 | printf("type: LUA_TBOOLEAN\\n") 261 | if ( user_long(data) == 0) { 262 | printf("value: false\\n") 263 | 264 | } else if ( user_long(data) == 1) { 265 | printf("value: true\\n") 266 | 267 | } else { 268 | printf("ERROR: value error\\n") 269 | } 270 | 271 | printf("expires: %d\\n", $sd->expires) 272 | printf("flags: 0x%x\\n", $sd->user_flags) 273 | 274 | } else { 275 | error("Bad value type found for key $key in shared_dict $dict\\n") 276 | } 277 | 278 | } else { 279 | printf("ERROR: key not found\\n"); 280 | } 281 | } 282 | _EOC_ 283 | 284 | } 285 | chop $preamble; 286 | 287 | $stap_src = <<_EOC_; 288 | $preamble 289 | 290 | function ngx_http_lua_shdict_lookup(zone, hash, key) { 291 | ctx = \@cast(zone, "ngx_shm_zone_t", "$nginx_path")->data 292 | sh = \@cast(ctx, "ngx_http_lua_shdict_ctx_t", "$nginx_path")->sh 293 | node = $sh->rbtree->root 294 | sentinel = $sh->rbtree->sentinel 295 | 296 | while (node != sentinel) { 297 | 298 | if (hash < $node->key) { 299 | node = $node->left 300 | continue 301 | } 302 | 303 | if (hash > $node->key) { 304 | node = $node->right 305 | continue 306 | } 307 | 308 | sd = &$node->color 309 | data = $sd->data 310 | data_str = text_str(user_string(data)) 311 | data_key = substr(data_str, 0 , $key_len) 312 | 313 | if (data_key == "$key"){ 314 | return sd 315 | 316 | } else { 317 | if (data_key < "$key") { 318 | node = $node->left 319 | 320 | } else { 321 | node = $node->right 322 | } 323 | } 324 | } 325 | return 0 326 | } 327 | 328 | $print_result_func_def 329 | 330 | probe process("$nginx_path").function("ngx_process_events_and_timers"), 331 | process("$nginx_path").function("ngx_http_init_request")!, 332 | process("$nginx_path").function("ngx_http_init_connection") 333 | { 334 | if (pid() == target()) { 335 | 336 | zone_found = 0 337 | 338 | $set_timer_begin 339 | 340 | part = &\@var("ngx_cycle\@ngx_cycle.c")->shared_memory->part 341 | zone = $part->elts 342 | 343 | for (i = 0; ; i++) { 344 | 345 | if (i >= $part->nelts) { 346 | if ($part->next == 0) { 347 | break 348 | } 349 | 350 | part = $part->next 351 | zone = $part->elts 352 | i = 0 353 | } 354 | 355 | shm = &\@cast(zone, "ngx_shm_zone_t")[i]->shm 356 | name = &\@cast(shm, "ngx_shm_t")->name 357 | 358 | if (\@cast(name, "ngx_str_t")->len != $dict_name_len) { 359 | continue; 360 | } 361 | 362 | zone_name = user_string_n(\@cast(name, "ngx_str_t")->data, $dict_name_len) 363 | if (zone_name != $quoted_zone) { 364 | continue; 365 | } 366 | 367 | sd = ngx_http_lua_shdict_lookup(zone, $hash, "$key") 368 | print_result(sd) 369 | 370 | zone_found = 1 371 | break 372 | } 373 | 374 | if (!zone_found) { 375 | printf("dict \\"%s\\" not found.\\n", $quoted_zone) 376 | } 377 | 378 | $print_time_elasped_func_def 379 | 380 | exit() 381 | 382 | } /* pid() == target() */ 383 | } 384 | _EOC_ 385 | 386 | } 387 | 388 | if ($dump_src) { 389 | print $stap_src; 390 | exit; 391 | } 392 | 393 | open my $in, "|stap $stap_args -x $pid -" 394 | or die "Cannot run stap: $!\n"; 395 | 396 | print $in $stap_src; 397 | 398 | close $in; 399 | 400 | sub usage { 401 | return <<'_EOC_'; 402 | Usage: 403 | ngx-lua-shdict [optoins] 404 | 405 | Options: 406 | -p Specify the nginx worker process pid. 407 | -a Pass extra arguments to the stap utility. 408 | -d Dump out the systemtap script source. 409 | -f Find value of specify key. 410 | -w Trace writes to the dict. 411 | --lua51 The target Nginx is using the standard Lua 5.1 interpreter. 412 | --luajit20 The target Nginx is using the LuaJIT 2.0. 413 | --dict Specify the dict. 414 | --key Specify the key. 415 | --raw Raw output. 416 | 417 | Examples: 418 | ngx-lua-shdict -p 12345 419 | ngx-lua-shdict -p 12345 -f --dict dogs --key Jim --luajit20 420 | ngx-lua-shdict -p 12345 -f --dict dogs --key Jim --luajit20 --raw 421 | ngx-lua-shdict -p 12345 -w --key Jim --luajit 422 | _EOC_ 423 | } 424 | 425 | sub quote_str { 426 | my $s = shift; 427 | $s =~ s/\\/\\\\/g; 428 | $s =~ s/"/\\"/g; 429 | $s =~ s/\n/\\n/g; 430 | $s =~ s/\t/\\t/g; 431 | $s =~ s/\r/\\r/g; 432 | return qq{"$s"}; 433 | } 434 | 435 | sub get_lua_path { 436 | my $lua_path; 437 | 438 | if (!defined $lua51 && !defined $luajit20) { 439 | die "Neither --lua51 nor --luajit20 options are specified.\n"; 440 | } 441 | 442 | my $maps_file = "/proc/$pid/maps"; 443 | open my $in, $maps_file 444 | or die "Cannot open $maps_file for reading: $!\n"; 445 | 446 | while (<$in>) { 447 | if (m{\S+\bliblua-(\d+\.\d+)\.so(?:\.\d+)*$}) { 448 | my ($path, $ver) = ($&, $1); 449 | 450 | if ($luajit20) { 451 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 452 | } 453 | 454 | if ($ver ne '5.1') { 455 | die "Nginx server $pid uses a Lua $ver library ", 456 | "but only Lua 5.1 is supported.\n"; 457 | } 458 | 459 | $lua_path = $path; 460 | last; 461 | 462 | } elsif (m{\S+\bliblua\.so(?:\.\d+)*$}) { 463 | my $path = $&; 464 | 465 | if ($luajit20) { 466 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 467 | } 468 | 469 | $lua_path = $path; 470 | last; 471 | 472 | } elsif (m{\S+\blibluajit-(\d+\.\d+)\.so(?:\.\d+)*$}) { 473 | my ($path, $ver) = ($&, $1); 474 | 475 | if ($lua51) { 476 | die "The --lua51 option is specified but seen the LuaJIT library: $path\n"; 477 | } 478 | 479 | if ($ver ne '5.1') { 480 | die "Nginx server $pid uses a Lua $ver compatible LuaJIT library ", 481 | "but only Lua 5.1 is supported.\n"; 482 | } 483 | 484 | $lua_path = $path; 485 | last; 486 | } 487 | } 488 | 489 | close $in; 490 | 491 | if (!defined $lua_path) { 492 | #warn "FALL BACK TO NGINX PATH"; 493 | $lua_path = $nginx_path; 494 | } 495 | 496 | return $lua_path; 497 | } 498 | 499 | sub crc32 { 500 | my ($input, $init_value, $polynomial) = @_; 501 | 502 | $init_value = 0 unless (defined $init_value); 503 | $polynomial = 0xedb88320 unless (defined $polynomial); 504 | 505 | my @lookup_table; 506 | 507 | for (my $i = 0; $i < 256; $i++) { 508 | my $x = $i; 509 | for (my $j = 0; $j < 8; $j++) { 510 | if ($x & 1) { 511 | $x = ($x >> 1) ^ $polynomial; 512 | 513 | } else { 514 | $x = $x >> 1; 515 | } 516 | } 517 | push @lookup_table, $x; 518 | } 519 | my $crc = $init_value ^ 0xffffffff; 520 | foreach my $x (unpack ('C*', $input)) { 521 | $crc = (($crc >> 8) & 0xffffff) ^ $lookup_table[ ($crc ^ $x) & 0xff ]; 522 | } 523 | $crc = $crc ^ 0xffffffff; 524 | 525 | return $crc; 526 | } 527 | -------------------------------------------------------------------------------- /ngx-pcre-stats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Long qw( GetOptions ); 10 | 11 | GetOptions("a=s", \(my $stap_args), 12 | "d", \(my $dump_src), 13 | "h", \(my $help), 14 | "p=i", \(my $pid), 15 | "exec-time-dist", \(my $exec_time_dist), 16 | "worst-time-top", \(my $worst_time_top), 17 | "t=i", \(my $time), 18 | "total-time-top", \(my $total_time_top), 19 | "data-len-dist", \(my $data_len_dist), 20 | "luajit20", \(my $luajit20), 21 | "lua51", \(my $lua51)) 22 | or die usage(); 23 | 24 | if ($help) { 25 | print usage(); 26 | exit; 27 | } 28 | 29 | if (!$pid) { 30 | die "No nginx process pid specified by the -p option\n"; 31 | } 32 | 33 | if (!defined $stap_args) { 34 | $stap_args = ''; 35 | } 36 | 37 | if ($^O ne 'linux') { 38 | die "Only linux is supported but I am on $^O.\n"; 39 | } 40 | 41 | my $exec_file = "/proc/$pid/exe"; 42 | if (!-f $exec_file) { 43 | die "Nginx process $pid is not running or ", 44 | "you do not have enough permissions.\n"; 45 | } 46 | 47 | my $nginx_path = readlink $exec_file; 48 | 49 | my $maps_file = "/proc/$pid/maps"; 50 | if (!-f $maps_file) { 51 | die "Nginx process $pid is not running or ", 52 | "you do not have enough permissions.\n"; 53 | } 54 | 55 | open my $in, $maps_file or 56 | die "Cannot open $maps_file for reading: $!\n"; 57 | 58 | my $pcre_path; 59 | while (<$in>) { 60 | if (m{\s+(/\S*libpcre\.so\S*)}) { 61 | $pcre_path = $1; 62 | } 63 | } 64 | close $in; 65 | 66 | #warn "pcre path: $pcre_path\n"; 67 | 68 | my $ver = `stap --version 2>&1`; 69 | if (!defined $ver) { 70 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 71 | } 72 | 73 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 74 | my $v = $1; 75 | if ($v < 2.1) { 76 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 77 | } 78 | 79 | } else { 80 | die "ERROR: unknown version of systemtap:\n$ver\n"; 81 | } 82 | 83 | my $limit = 10; 84 | my $stap_src; 85 | 86 | my $guide; 87 | if (defined $time) { 88 | $guide = "Please wait for $time seconds."; 89 | 90 | } else { 91 | $guide = "Hit Ctrl-C to end."; 92 | } 93 | 94 | my $preamble = <<_EOC_; 95 | probe begin 96 | { 97 | printf("Tracing %d ($nginx_path)...\\n$guide\\n", target()) 98 | } 99 | _EOC_ 100 | chop $preamble; 101 | 102 | my $process_path = $pcre_path || $nginx_path; 103 | 104 | if ($exec_time_dist) { 105 | $stap_src = <<_EOC_; 106 | $preamble 107 | 108 | global exectimes 109 | global begin 110 | 111 | probe process("$process_path").function("pcre_exec") 112 | { 113 | if (target() == pid()) { 114 | begin = gettimeofday_us() 115 | } 116 | } 117 | 118 | probe process("$process_path").function("pcre_exec").return 119 | { 120 | if (target() == pid() && begin) { 121 | elapsed = gettimeofday_us() - begin 122 | exectimes <<< elapsed 123 | } 124 | } 125 | 126 | probe end 127 | { 128 | if (!begin) { 129 | println("\\nNo pcre_exec() calls found so far.") 130 | 131 | } else { 132 | println("\\nLogarithmic histogram for pcre_exec running time distribution (us):") 133 | print(\@hist_log(exectimes)) 134 | } 135 | } 136 | _EOC_ 137 | 138 | } elsif ($data_len_dist) { 139 | $stap_src = <<_EOC_; 140 | $preamble 141 | 142 | global datalens 143 | global found 144 | 145 | probe process("$process_path").function("pcre_exec") 146 | { 147 | if (target() == pid()) { 148 | found = 1 149 | datalens <<< \$length 150 | //printf("len: %d, ofs: %d", \$length, \$start_offset) 151 | } 152 | } 153 | 154 | probe end 155 | { 156 | if (!found) { 157 | println("\\nNo pcre_exec() calls found so far.") 158 | 159 | } else { 160 | println("\\nLogarithmic histogram for data length distribution:") 161 | print(\@hist_log(datalens)) 162 | } 163 | } 164 | _EOC_ 165 | 166 | } elsif ($worst_time_top || $total_time_top) { 167 | my $lua_path = get_lua_path(); 168 | 169 | if (!$luajit20) { 170 | die "Only LuaJIT 2.0 is currently supported.\n"; 171 | } 172 | 173 | my $LJ_TUDATA = "4294967283"; 174 | my $TL_TSTR = "4294967291"; 175 | 176 | my $sizeof_TValue = qq{\&\@cast(0, "TValue", "$lua_path")[1]}; 177 | my $sizeof_GCstr = qq{\&\@cast(0, "GCstr", "$lua_path")[1]}; 178 | my $o = qq{\@cast(o, "TValue", "$lua_path")}; 179 | my $L = qq{\@cast(L, "lua_State", "$lua_path")}; 180 | my $gcr = qq{\@cast(gcr, "GCRef", "$lua_path")}; 181 | my $gcobj = qq{\@cast(gcobj, "GCobj", "$lua_path")}; 182 | my $str = qq{\@cast(str, "GCstr", "$lua_path")}; 183 | my $fn = qq{\@cast(fn, "GCfunc", "$lua_path")}; 184 | my $ud = qq{\@cast(ud, "GCudata", "$lua_path")}; 185 | 186 | my $common = <<_EOC_; 187 | $preamble 188 | 189 | global regex 190 | global gmatch_regex 191 | global datalen 192 | global begin 193 | global exectimes 194 | global datalens 195 | global compiled 196 | 197 | function gcref(gcr) 198 | { 199 | return $gcr->gcptr32 200 | } 201 | 202 | function gcval(o) 203 | { 204 | return gcref(\&$o->gcr) 205 | } 206 | 207 | function strdata(s) 208 | { 209 | return s + $sizeof_GCstr 210 | } 211 | 212 | function curr_func(L) 213 | { 214 | o = $L->base - 1 * $sizeof_TValue 215 | gcobj = gcref(\&$o->fr->func) 216 | return \&$gcobj->fn 217 | } 218 | 219 | probe process("$nginx_path").function("ngx_http_lua_ngx_re_gmatch_iterator") 220 | { 221 | if (target() == pid()) { 222 | L = \$L 223 | fn = curr_func(L) 224 | //printf("upvalues: %d\\n", $fn->c->nupvalues) 225 | if ($fn->c->nupvalues >= 3) { 226 | o = \&$fn->c->upvalue[1] // the 2nd upvalue 227 | //printf("upvalue type: %d == $LJ_TUDATA\\n", $o->it) 228 | if ($o->it == $LJ_TUDATA) { 229 | gcobj = gcval(o) 230 | ud = \&$gcobj->ud 231 | regex = compiled[ud] 232 | } 233 | } 234 | } 235 | } 236 | 237 | probe process("$nginx_path").function("ngx_http_lua_ngx_re_gmatch") 238 | { 239 | if (target() == pid()) { 240 | delete compiled // XXX this is a hack 241 | L = \$L 242 | o = $L->base + $sizeof_TValue * (2 - 1) 243 | if (o < $L->top && $o->it == $TL_TSTR) { 244 | gcobj = gcval(o) 245 | str = \&$gcobj->str 246 | //printf("gmatch regex: %s\\n", user_string_n(strdata(str), $str->len)) 247 | gmatch_regex = user_string_n(strdata(str), $str->len) 248 | } else { 249 | gmatch_regex = "" 250 | } 251 | } 252 | } 253 | 254 | probe process("$lua_path").function("lua_pushcclosure") { 255 | if (target() == pid() && gmatch_regex != "") { 256 | L = \$L 257 | //compiled[\$ctx->regex] = gmatch_regex 258 | o = $L->top + $sizeof_TValue * -2 259 | //printf("type %d == $LJ_TUDATA\\n", $o->it) 260 | if ($o->it == $LJ_TUDATA) { 261 | gcobj = gcval(o) 262 | ud = \&$gcobj->ud 263 | compiled[ud] = gmatch_regex 264 | } 265 | gmatch_regex = "" 266 | } 267 | } 268 | 269 | probe process("$nginx_path").function("ngx_http_lua_ngx_re_gmatch").return 270 | { 271 | if (target() == pid()) { 272 | gmatch_regex = "" 273 | } 274 | } 275 | 276 | probe process("$nginx_path").function("ngx_http_lua_ngx_re_sub_helper"), 277 | process("$nginx_path").function("ngx_http_lua_ngx_re_match_helper") !, 278 | process("$nginx_path").function("ngx_http_lua_ngx_re_match") 279 | { 280 | if (target() == pid()) { 281 | L = \$L 282 | o = $L->base + $sizeof_TValue * (2 - 1) 283 | if (o < $L->top && $o->it == $TL_TSTR) { 284 | gcobj = gcval(o) 285 | str = \&$gcobj->str 286 | //printf("regex: %s\\n", user_string_n(strdata(str), $str->len)) 287 | regex = user_string_n(strdata(str), $str->len) 288 | } 289 | } 290 | } 291 | 292 | probe process("$process_path").function("pcre_exec") 293 | { 294 | if (target() == pid()) { 295 | begin = gettimeofday_us() 296 | datalen = \$length 297 | } 298 | } 299 | _EOC_ 300 | 301 | if ($worst_time_top) { 302 | $stap_src = <<_EOC_; 303 | $common 304 | 305 | probe process("$process_path").function("pcre_exec").return 306 | { 307 | if (target() == pid() && begin && regex != "") { 308 | elapsed = gettimeofday_us() - begin 309 | max = exectimes[regex] 310 | if (max < elapsed) { 311 | exectimes[regex] = elapsed 312 | datalens[regex] = datalen 313 | } 314 | regex = "" 315 | elapsed = 0 316 | } 317 | } 318 | 319 | probe end 320 | { 321 | if (!begin) { 322 | println("\\nNo pcre_exec() calls found so far.") 323 | 324 | } else { 325 | println("\\nTop N regexes with worst running time:") 326 | 327 | i = 0 328 | foreach (regex in exectimes- limit $limit) { 329 | i++ 330 | printf("%d. pattern /%s/: %dus (data size: %d)\\n", 331 | i, regex, exectimes[regex], datalens[regex]) 332 | } 333 | } 334 | } 335 | _EOC_ 336 | 337 | } else { 338 | # $total_time_top 339 | 340 | $stap_src = <<_EOC_; 341 | $common 342 | 343 | probe process("$process_path").function("pcre_exec").return 344 | { 345 | if (target() == pid() && begin && regex != "") { 346 | elapsed = gettimeofday_us() - begin 347 | exectimes[regex] += elapsed 348 | datalens[regex] += datalen 349 | regex = "" 350 | elapsed = 0 351 | } 352 | } 353 | 354 | probe end 355 | { 356 | if (!begin) { 357 | println("\\nNo pcre_exec() calls found so far.") 358 | 359 | } else { 360 | println("\\nTop N regexes with longest total running time:") 361 | 362 | i = 0 363 | foreach (regex in exectimes- limit $limit) { 364 | i++ 365 | printf("%d. pattern /%s/: %dus (total data size: %d)\\n", 366 | i, regex, exectimes[regex], datalens[regex]) 367 | } 368 | } 369 | } 370 | _EOC_ 371 | } 372 | 373 | } else { 374 | die "You must specify one of the --exec-time-dist, --worst-time-top, " 375 | ."--total-time-top, and --data-len-dist options.\n"; 376 | } 377 | 378 | if (defined $time) { 379 | $stap_src .= <<_EOC_; 380 | probe timer.s($time) { 381 | exit() 382 | } 383 | _EOC_ 384 | } 385 | 386 | if ($dump_src) { 387 | print $stap_src; 388 | exit; 389 | } 390 | 391 | open $in, "|stap --skip-badvars -d '$nginx_path' $stap_args -x $pid -" 392 | or die "Cannot run stap: $!\n"; 393 | 394 | print $in $stap_src; 395 | 396 | close $in; 397 | 398 | sub usage { 399 | return <<'_EOC_'; 400 | Usage: 401 | ngx-pcre-stats [optoins] 402 | 403 | Options: 404 | -a Pass extra arguments to the stap utility. 405 | --exec-time-dist pcre_exec running time distribution. 406 | -d Dump out the systemtap script source. 407 | --data-len-dist Data length distribution. 408 | --lua51 The target Nginx is using the standard Lua 5.1 interpreter. 409 | --luajit20 The target Nginx is using the LuaJIT 2.0. 410 | -h Print this usage. 411 | -p Specify the nginx worker process pid. 412 | -t Specify the time period in seconds for sampling 413 | --total-time-top The top N regexes with longest total running time. 414 | --worst-time-top The top N regexes with worst running time. 415 | 416 | Examples: 417 | ngx-pcre-stats -p 12345 --exec-time-dist 418 | ngx-pcre-stats -p 12345 --data-len-dist 419 | ngx-pcre-stats -p 12345 --worst-time-top --luajit20 420 | ngx-pcre-stats -p 12345 --total-time-top --luajit20 421 | ngx-pcre-stats -p 12345 --total-time-top --luajit20 -t 10 422 | _EOC_ 423 | } 424 | 425 | sub get_lua_path { 426 | my $lua_path; 427 | 428 | if (!defined $lua51 && !defined $luajit20) { 429 | die "Neither --lua51 nor --luajit20 options are specified.\n"; 430 | } 431 | 432 | my $maps_file = "/proc/$pid/maps"; 433 | open my $in, $maps_file 434 | or die "Cannot open $maps_file for reading: $!\n"; 435 | 436 | while (<$in>) { 437 | if (m{\S+\bliblua-(\d+\.\d+)\.so(?:\.\d+)*$}) { 438 | my ($path, $ver) = ($&, $1); 439 | 440 | if ($luajit20) { 441 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 442 | } 443 | 444 | if ($ver ne '5.1') { 445 | die "Nginx server $pid uses a Lua $ver library ", 446 | "but only Lua 5.1 is supported.\n"; 447 | } 448 | 449 | $lua_path = $path; 450 | last; 451 | 452 | } elsif (m{\S+\bliblua\.so(?:\.\d+)*$}) { 453 | my $path = $&; 454 | 455 | if ($luajit20) { 456 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 457 | } 458 | 459 | $lua_path = $path; 460 | last; 461 | 462 | } elsif (m{\S+\blibluajit-(\d+\.\d+)\.so(?:\.\d+)*$}) { 463 | my ($path, $ver) = ($&, $1); 464 | 465 | if ($lua51) { 466 | die "The --lua51 option is specified but seen the LuaJIT library: $path\n"; 467 | } 468 | 469 | if ($ver ne '5.1') { 470 | die "Nginx server $pid uses a Lua $ver compatible LuaJIT library ", 471 | "but only Lua 5.1 is supported.\n"; 472 | } 473 | 474 | $lua_path = $path; 475 | last; 476 | } 477 | } 478 | 479 | close $in; 480 | 481 | if (!defined $lua_path) { 482 | #warn "FALL BACK TO NGINX PATH"; 483 | $lua_path = $nginx_path; 484 | } 485 | 486 | return $lua_path; 487 | } 488 | 489 | -------------------------------------------------------------------------------- /ngx-pcrejit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | if ($^O ne 'linux') { 31 | die "Only linux is supported but I am on $^O.\n"; 32 | } 33 | 34 | my $verbose = $opts{v}; 35 | 36 | my $exec_file = "/proc/$pid/exe"; 37 | if (!-f $exec_file) { 38 | die "Nginx process $pid is not running or ", 39 | "you do not have enough permissions.\n"; 40 | } 41 | 42 | my $nginx_path = readlink $exec_file; 43 | 44 | my $maps_file = "/proc/$pid/maps"; 45 | if (!-f $maps_file) { 46 | die "Nginx process $pid is not running or ", 47 | "you do not have enough permissions.\n"; 48 | } 49 | 50 | open my $in, $maps_file or 51 | die "Cannot open $maps_file for reading: $!\n"; 52 | 53 | my $pcre_path; 54 | while (<$in>) { 55 | if (m{\s+(/\S*libpcre\.so\S*)}) { 56 | $pcre_path = $1; 57 | } 58 | } 59 | close $in; 60 | 61 | #warn "pcre path: $pcre_path\n"; 62 | 63 | my $ver = `stap --version 2>&1`; 64 | if (!defined $ver) { 65 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 66 | } 67 | 68 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 69 | my $v = $1; 70 | if ($v < 2.1) { 71 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 72 | } 73 | 74 | } else { 75 | die "ERROR: unknown version of systemtap:\n$ver\n"; 76 | } 77 | 78 | my $stap_src; 79 | 80 | my $preamble = <<_EOC_; 81 | probe begin 82 | { 83 | printf("Tracing %d ($nginx_path)...\\nHit Ctrl-C to end.\\n", target()) 84 | } 85 | _EOC_ 86 | chop $preamble; 87 | 88 | my $process_path = $pcre_path || $nginx_path; 89 | 90 | $stap_src = <<_EOC_; 91 | $preamble 92 | 93 | global jits 94 | global jit_disabled = 0 95 | 96 | probe process("$process_path").function("pcre_exec") 97 | { 98 | if (target() == pid()) { 99 | /* PCRE_EXTRA_EXECUTABLE_JIT = 0x0040 */ 100 | if (\@defined(\$extra_data->executable_jit) 101 | && (\@defined(\@var("ucp_typerange\@pcre_tables.c")) 102 | || \@defined(\@var("_pcre_ucp_typerange\@pcre_tables.c")))) 103 | { 104 | jit = (\$extra_data && (\$extra_data->flags & 0x0040) 105 | && \$extra_data->executable_jit) 106 | 107 | } else { 108 | jit_disabled = 1 109 | jit = 0 110 | } 111 | 112 | caller = usymname(ustack(1)) 113 | jits[caller] <<< jit 114 | } 115 | } 116 | 117 | probe end { 118 | found = 0 119 | printf("\\n") 120 | foreach (caller in jits- limit 10) { 121 | if (!found) { 122 | found = 1 123 | } 124 | printf("%s: %d of %d are PCRE JITted.\\n", caller, 125 | \@sum(jits[caller]), \@count(jits[caller])) 126 | } 127 | if (!found) { 128 | println("No pcre_exec() invocations found.") 129 | } 130 | if (jit_disabled) { 131 | println("\\nLooks like JIT support is missing in your PCRE build.") 132 | } 133 | } 134 | _EOC_ 135 | 136 | if ($opts{d}) { 137 | print $stap_src; 138 | exit; 139 | } 140 | 141 | open $in, "|stap --skip-badvars -d '$nginx_path' $stap_args -x $pid -" 142 | or die "Cannot run stap: $!\n"; 143 | 144 | print $in $stap_src; 145 | 146 | close $in; 147 | 148 | sub usage { 149 | return <<'_EOC_'; 150 | Usage: 151 | ngx-pcrejit [optoins] 152 | 153 | Options: 154 | -a Pass extra arguments to the stap utility. 155 | -d Dump out the systemtap script source. 156 | -h Print this usage. 157 | -p Specify the nginx worker process pid. 158 | 159 | Examples: 160 | ngx-pcrejit -p 12345 161 | _EOC_ 162 | } 163 | 164 | -------------------------------------------------------------------------------- /ngx-phase-handlers: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | # Copyright (C) Guanlan Dai 5 | 6 | use 5.006001; 7 | use strict; 8 | use warnings; 9 | 10 | use Getopt::Std qw( getopts ); 11 | 12 | my %opts; 13 | 14 | getopts("a:dhp:", \%opts) 15 | or die usage(); 16 | 17 | if ($opts{h}) { 18 | print usage(); 19 | exit; 20 | } 21 | 22 | my $pid = $opts{p} 23 | or die "No nginx process pid specified by the -p option\n"; 24 | 25 | if ($pid !~ /^\d+$/) { 26 | die "Bad -p option value \"$pid\": not look like a pid\n"; 27 | } 28 | 29 | my $stap_args = $opts{a} || ''; 30 | 31 | if ($^O ne 'linux') { 32 | die "Only linux is supported but I am on $^O.\n"; 33 | } 34 | 35 | my $exec_file = "/proc/$pid/exe"; 36 | if (!-f $exec_file) { 37 | die "Nginx process $pid is not running or ", 38 | "you do not have enough permissions.\n"; 39 | } 40 | 41 | my $nginx_path = readlink $exec_file; 42 | 43 | my $ver = `stap --version 2>&1`; 44 | if (!defined $ver) { 45 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 46 | } 47 | 48 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 49 | my $v = $1; 50 | if ($v < 2.1) { 51 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 52 | } 53 | 54 | } else { 55 | die "ERROR: unknown version of systemtap:\n$ver\n"; 56 | } 57 | 58 | my $stap_src; 59 | 60 | my $cmcf = qq{\@cast(cmcf, "ngx_http_core_main_conf_t", "$nginx_path")}; 61 | my $ph = qq{\@cast(ph, "ngx_http_phase_handler_t", "$nginx_path")}; 62 | my $handler_addr = qq{\@cast(handler_addr, "ngx_http_handler_pt", "$nginx_path")}; 63 | my $sizeof_ngx_http_phase_handler_t = qq{&\@cast(0, 64 | "ngx_http_phase_handler_t", 65 | "$nginx_path")[1]}; 66 | my $sizeof_ngx_http_handler_pt = qq{&\@cast(0, "ngx_int_t", "$nginx_path")[1]}; 67 | 68 | my $NGX_HTTP_POST_READ_PHASE = 0; 69 | my $NGX_HTTP_SERVER_REWRITE_PHASE = 1; 70 | my $NGX_HTTP_FIND_CONFIG_PHASE = 2; 71 | my $NGX_HTTP_REWRITE_PHASE = 3; 72 | my $NGX_HTTP_POST_REWRITE_PHASE = 4; 73 | my $NGX_HTTP_PREACCESS_PHASE = 5; 74 | my $NGX_HTTP_ACCESS_PHASE = 6; 75 | my $NGX_HTTP_POST_ACCESS_PHASE = 7; 76 | my $NGX_HTTP_TRY_FILES_PHASE = 8; 77 | my $NGX_HTTP_CONTENT_PHASE = 9; 78 | my $NGX_HTTP_LOG_PHASE = 10; 79 | 80 | my $preamble = <<_EOC_; 81 | probe begin { 82 | printf("Tracing %d ($nginx_path)...\\n", target()) 83 | } 84 | _EOC_ 85 | chop $preamble; 86 | 87 | $stap_src = <<_EOC_; 88 | $preamble 89 | 90 | function print_phase(cmcf, phase_num) { 91 | n = $cmcf->phases[phase_num]->handlers->nelts 92 | handler_addr = $cmcf->phases[phase_num]->handlers->elts 93 | handler = $handler_addr 94 | for (i = 0; i < n; i++) { 95 | printf(" %s\\n", usymname(handler)) 96 | handler_addr += $sizeof_ngx_http_handler_pt 97 | handler = $handler_addr 98 | } 99 | } 100 | 101 | probe process("$nginx_path").function("ngx_http_log_request") 102 | { 103 | if (pid() == target()) { 104 | begin = gettimeofday_us() 105 | ngx_http_core_module = &\@var("ngx_http_core_module") 106 | ctx_index = \@cast(ngx_http_core_module, "ngx_module_t")->ctx_index 107 | cmcf = \$r->main_conf[ctx_index] 108 | use_rewrite = $cmcf->phases[$NGX_HTTP_REWRITE_PHASE]->handlers->nelts ? 1 : 0 109 | use_access = $cmcf->phases[$NGX_HTTP_ACCESS_PHASE]->handlers->nelts ? 1 : 0 110 | n = 0 111 | 112 | for (i = 0; i < $NGX_HTTP_LOG_PHASE; i++) { 113 | if (i == $NGX_HTTP_FIND_CONFIG_PHASE) { 114 | n++ 115 | continue 116 | } 117 | 118 | if (i == $NGX_HTTP_POST_REWRITE_PHASE) { 119 | if (use_rewrite) { 120 | n++ 121 | } 122 | continue 123 | } 124 | 125 | if (i == $NGX_HTTP_POST_ACCESS_PHASE) { 126 | if (use_access) { 127 | n++ 128 | } 129 | continue 130 | } 131 | 132 | if (i == $NGX_HTTP_TRY_FILES_PHASE) { 133 | if ($cmcf->try_files) { 134 | n++ 135 | } 136 | continue 137 | } 138 | 139 | phase_len = $cmcf->phases[i]->handlers->nelts 140 | if (phase_len == 0) { 141 | continue 142 | } 143 | 144 | if (i == $NGX_HTTP_POST_READ_PHASE) { 145 | println("post-read phase") 146 | 147 | } else if (i == $NGX_HTTP_SERVER_REWRITE_PHASE) { 148 | println("server-rewrite phase") 149 | 150 | } else if (i == $NGX_HTTP_REWRITE_PHASE) { 151 | println("rewrite phase") 152 | 153 | } else if (i == $NGX_HTTP_PREACCESS_PHASE) { 154 | println("pre-access phase") 155 | 156 | } else if (i == $NGX_HTTP_ACCESS_PHASE) { 157 | println("access phase") 158 | 159 | } else if (i == $NGX_HTTP_CONTENT_PHASE) { 160 | println("content phase") 161 | 162 | if (\$r->content_handler) { 163 | printf(" %s (request content handler)\\n", 164 | usymname(\$r->content_handler)) 165 | } 166 | 167 | } else { 168 | printf("unknown phase (%d)\\n", i) 169 | } 170 | 171 | for (j = 0; j < phase_len; j++) { 172 | handler = $cmcf->phase_engine->handlers[n++]->handler 173 | if (handler == 0) { 174 | continue 175 | } 176 | printf(" %s\\n", usymname(handler)) 177 | } 178 | 179 | printf("\\n") 180 | } 181 | 182 | println("log phase") 183 | print_phase(cmcf, $NGX_HTTP_LOG_PHASE) 184 | 185 | elapsed = gettimeofday_us() - begin 186 | printf("\\n%d microseconds elapsed in the probe handler.\\n", elapsed) 187 | exit() 188 | } 189 | } 190 | _EOC_ 191 | 192 | if ($opts{d}) { 193 | print $stap_src; 194 | exit; 195 | } 196 | 197 | open my $in, "|stap --skip-badvars $stap_args -x $pid -" 198 | or die "Cannot run stap: $!\n"; 199 | 200 | print $in $stap_src; 201 | 202 | close $in; 203 | 204 | sub usage { 205 | return <<'_EOC_'; 206 | Usage: 207 | ngx-phase-handlers [optoins] 208 | 209 | Options: 210 | -a Pass extra arguments to the stap utility. 211 | -d Dump out the systemtap script source. 212 | -h Print this usage. 213 | -p Specify the nginx worker process pid. 214 | 215 | Examples: 216 | ngx-phase-handlers -p 12345 217 | _EOC_ 218 | } 219 | 220 | -------------------------------------------------------------------------------- /ngx-req-distr: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhm:c", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $master_pid = $opts{m} 22 | or die "No nginx master process pid specified by the -m option\n"; 23 | 24 | if ($master_pid !~ /^\d+$/) { 25 | die "Bad -m option value \"$master_pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | my $analyze_conn = $opts{c}; 31 | 32 | if ($^O ne 'linux') { 33 | die "Only linux is supported but I am on $^O.\n"; 34 | } 35 | 36 | my $exec_file = "/proc/$master_pid/exe"; 37 | if (!-f $exec_file) { 38 | die "Nginx process $master_pid is not running or ", 39 | "you do not have enough permissions.\n"; 40 | } 41 | 42 | my $nginx_path = readlink $exec_file; 43 | 44 | my $ver = `stap --version 2>&1`; 45 | if (!defined $ver) { 46 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 47 | } 48 | 49 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 50 | my $v = $1; 51 | if ($v < 2.1) { 52 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 53 | } 54 | 55 | } else { 56 | die "ERROR: unknown version of systemtap:\n$ver\n"; 57 | } 58 | 59 | my $child_pids = get_child_processes($master_pid); 60 | if (!@$child_pids) { 61 | push @$child_pids, $master_pid; 62 | } 63 | 64 | my $condition = gen_pid_test_condition($child_pids); 65 | 66 | #warn "Nginx worker processes: @$child_pids\n"; 67 | 68 | my $stap_src; 69 | 70 | my $preamble = <<_EOC_; 71 | probe begin { 72 | printf("Tracing @$child_pids ($nginx_path)...\\nHit Ctrl-C to end.\\n") 73 | } 74 | _EOC_ 75 | 76 | if ($analyze_conn) { 77 | my $output_code = ''; 78 | for my $pid (@$child_pids) { 79 | $output_code .= <<_EOC_; 80 | printf("worker $pid:\\t%d reqs,\\t%d conns\\n", \@count(reqs[$pid]), \@count(conns[$pid])) 81 | _EOC_ 82 | } 83 | 84 | $stap_src = <<_EOC_; 85 | $preamble 86 | global reqs 87 | global conns 88 | 89 | probe process("$nginx_path").function("ngx_http_create_request")!, 90 | process("$nginx_path").function("ngx_http_init_request") 91 | { 92 | my_pid = pid() 93 | if ($condition) { 94 | reqs[my_pid] <<< 1 95 | } 96 | } 97 | 98 | probe process("$nginx_path").function("ngx_http_init_connection") 99 | { 100 | my_pid = pid() 101 | if ($condition) { 102 | conns[my_pid] <<< 1 103 | } 104 | } 105 | 106 | probe end { 107 | printf("\\n") 108 | $output_code} 109 | _EOC_ 110 | 111 | } else { 112 | my $output_code = ''; 113 | for my $pid (@$child_pids) { 114 | $output_code .= <<_EOC_; 115 | printf("worker $pid:\\t%d reqs\\n", \@count(reqs[$pid])) 116 | _EOC_ 117 | } 118 | 119 | $stap_src = <<_EOC_; 120 | $preamble 121 | global reqs 122 | 123 | probe process("$nginx_path").function("ngx_http_create_request")!, 124 | process("$nginx_path").function("ngx_http_init_request") 125 | { 126 | my_pid = pid() 127 | if ($condition) { 128 | reqs[my_pid] <<< 1 129 | } 130 | } 131 | 132 | probe end { 133 | printf("\\n") 134 | $output_code} 135 | _EOC_ 136 | } 137 | 138 | if ($opts{d}) { 139 | print $stap_src; 140 | exit; 141 | } 142 | 143 | open my $in, "|stap --skip-badvars $stap_args -" 144 | or die "Cannot run stap: $!\n"; 145 | 146 | print $in $stap_src; 147 | 148 | close $in; 149 | 150 | sub usage { 151 | return <<'_EOC_'; 152 | Usage: 153 | ngx-req-distr [optoins] 154 | 155 | Options: 156 | -a Pass extra arguments to the stap utility. 157 | -c analyze connections at the same time. 158 | -d Dump out the systemtap script source. 159 | -h Print this usage. 160 | -m Specify the nginx master process pid. 161 | 162 | Examples: 163 | ngx-req-distr -m 12345 164 | ngx-req-distr -m 12345 -a '-DMAXACTION=100000' 165 | _EOC_ 166 | } 167 | 168 | sub get_child_processes { 169 | my $pid = shift; 170 | my @files = glob "/proc/[0-9]*/stat"; 171 | my @children; 172 | for my $file (@files) { 173 | #print "file: $file\n"; 174 | if ($file =~ m{^/proc/$pid/}) { 175 | next; 176 | } 177 | 178 | open my $in, $file or next; 179 | my $line = <$in>; 180 | close $in; 181 | if ($line =~ /^(\d+) \S+ \S+ (\d+)/) { 182 | my ($child, $parent) = ($1, $2); 183 | if ($parent eq $pid) { 184 | push @children, $child; 185 | } 186 | } 187 | } 188 | 189 | @children = sort { $a <=> $b } @children; 190 | return \@children; 191 | } 192 | 193 | sub gen_pid_test_condition { 194 | my $pids = shift; 195 | my @c; 196 | for my $pid (@$pids) { 197 | push @c, "my_pid == $pid"; 198 | } 199 | return '(' . join(" || ", @c) . ')'; 200 | } 201 | -------------------------------------------------------------------------------- /ngx-sample-lua-bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Long qw( GetOptions ); 10 | 11 | GetOptions("a=s", \(my $stap_args), 12 | "d", \(my $dump_src), 13 | "h", \(my $help), 14 | "p=i", \(my $pid), 15 | "l=i", \(my $limit), 16 | "t=i", \(my $time), 17 | "lua51", \(my $lua51), 18 | "luajit20", \(my $luajit20)) 19 | or die usage(); 20 | 21 | if ($help) { 22 | print usage(); 23 | exit; 24 | } 25 | 26 | if (!$luajit20 && !$lua51) { 27 | die "You have to specify either the --lua51 or --luajit20 options.\n"; 28 | } 29 | 30 | if ($luajit20 && $lua51) { 31 | die "You cannot specify --lua51 and --luajit20 at the same time.\n"; 32 | } 33 | 34 | if (!defined $pid) { 35 | die "No nginx process pid specified by the -p option.\n"; 36 | } 37 | 38 | if (!defined $time) { 39 | die "No -t option specified.\n"; 40 | } 41 | 42 | if (!defined $limit) { 43 | $limit = 1024; 44 | } 45 | 46 | if (!defined $stap_args) { 47 | $stap_args = ''; 48 | } 49 | 50 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 51 | $stap_args .= " -DMAXACTION=100000" 52 | } 53 | 54 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 55 | $stap_args .= " -DMAXMAPENTRIES=5000" 56 | } 57 | 58 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 59 | $stap_args .= " -DMAXBACKTRACE=200" 60 | } 61 | 62 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 63 | $stap_args .= " -DMAXSTRINGLEN=2048" 64 | } 65 | 66 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSKIPPED=1024/) { 67 | $stap_args .= " -DMAXSKIPPED=1024" 68 | } 69 | 70 | if ($^O ne 'linux') { 71 | die "Only linux is supported but I am on $^O.\n"; 72 | } 73 | 74 | my $exec_file = "/proc/$pid/exe"; 75 | if (!-f $exec_file) { 76 | die "Nginx process $pid is not running or ", 77 | "you do not have enough permissions.\n"; 78 | } 79 | 80 | my $nginx_path = readlink $exec_file; 81 | 82 | my $child_pids = get_child_processes($pid); 83 | if (@$child_pids == 0) { 84 | push @$child_pids, $pid; 85 | } 86 | 87 | my $condition = gen_pid_test_condition($child_pids); 88 | 89 | my $ver = `stap --version 2>&1`; 90 | if (!defined $ver) { 91 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 92 | } 93 | 94 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 95 | my $v = $1; 96 | if ($v < 2.1) { 97 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 98 | } 99 | 100 | } else { 101 | die "ERROR: unknown version of systemtap:\n$ver\n"; 102 | } 103 | 104 | #warn "Nginx worker processes: @$child_pids\n"; 105 | 106 | my $context; 107 | if ($lua51) { 108 | $context = "standard Lua 5.1"; 109 | 110 | } elsif ($luajit20) { 111 | $context = "LuaJIT 2.0"; 112 | } 113 | 114 | my $preamble = <<_EOC_; 115 | probe begin { 116 | warn("Tracing @$child_pids ($nginx_path) for $context...\\n") 117 | } 118 | _EOC_ 119 | 120 | my $stap_src; 121 | 122 | my $postamble = <<_EOC_; 123 | probe timer.s($time) 124 | { 125 | warn("Time's up. Quitting now...\\n") 126 | foreach (bt in bts- limit $limit) { 127 | printf("%s\\t%d\\n", bt, \@count(bts[bt])) 128 | } 129 | exit() 130 | } 131 | _EOC_ 132 | 133 | my $lua_path; 134 | 135 | { 136 | my $maps_file = "/proc/$pid/maps"; 137 | open my $in, $maps_file 138 | or die "Cannot open $maps_file for reading: $!\n"; 139 | 140 | while (<$in>) { 141 | if (m{\S+\bliblua-(\d+\.\d+)\.so(?:\.\d+)*$}) { 142 | my ($path, $ver) = ($&, $1); 143 | 144 | if ($luajit20) { 145 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 146 | } 147 | 148 | if ($ver ne '5.1') { 149 | die "Nginx server $pid uses a Lua $ver library ", 150 | "but only Lua 5.1 is supported.\n"; 151 | } 152 | 153 | $lua_path = $path; 154 | last; 155 | 156 | } elsif (m{\S+\bliblua(?:5\.1)?\.so(?:\.\d+)*$}) { 157 | my $path = $&; 158 | 159 | if ($luajit20) { 160 | die "The --luajit20 option is specified but seen standard Lua library: $path\n"; 161 | } 162 | 163 | $lua_path = $path; 164 | last; 165 | 166 | } elsif (m{\S+\blibluajit-(\d+\.\d+)\.so(?:\.\d+)*$}) { 167 | my ($path, $ver) = ($&, $1); 168 | 169 | if ($lua51) { 170 | die "The --lua51 option is specified but seen the LuaJIT library: $path\n"; 171 | } 172 | 173 | if ($ver ne '5.1') { 174 | die "Nginx server $pid uses a Lua $ver compatible LuaJIT library ", 175 | "but only Lua 5.1 is supported.\n"; 176 | } 177 | 178 | $lua_path = $path; 179 | last; 180 | } 181 | } 182 | 183 | close $in; 184 | 185 | if (!defined $lua_path) { 186 | #warn "FALL BACK TO NGINX PATH"; 187 | $lua_path = $nginx_path; 188 | } 189 | } 190 | 191 | if ($lua51) { 192 | $stap_src = <<_EOC_; 193 | $preamble 194 | 195 | global bts 196 | global lua_states 197 | global ci_offset = 0 198 | global cfuncs 199 | 200 | 201 | probe process("$lua_path").function("lua_resume"), 202 | process("$lua_path").function("lua_pcall") 203 | { 204 | //printf("L = %p\\n", \$L) 205 | my_pid = pid() 206 | if ($condition) { 207 | lua_states[my_pid] = \$L 208 | } 209 | } 210 | 211 | 212 | probe process("$lua_path").function("lua_yield") 213 | { 214 | //printf("L = %p\\n", \$L) 215 | my_pid = pid() 216 | if ($condition) { 217 | lua_states[my_pid] = 0 218 | } 219 | } 220 | 221 | 222 | probe process("$nginx_path").function("ngx_http_lua_del_thread") 223 | { 224 | my_pid = pid() 225 | if ($condition) { 226 | lua_states[my_pid] = 0 227 | } 228 | } 229 | 230 | 231 | function clvalue(o) { 232 | return &\@cast(o, "TValue", "$lua_path")->value->gc->cl; 233 | } 234 | 235 | 236 | function ci_func(ci) { 237 | return clvalue(\@cast(ci, "CallInfo", "$lua_path")->func) 238 | } 239 | 240 | 241 | function f_isLua(ci) { 242 | f = ci_func(ci) 243 | if (f == 0) { 244 | return 0; 245 | } 246 | 247 | //printf("f_isLua: ci=%x, f=%x, c=%p\\n", ci, f, &\@cast(f, "Closure", "$lua_path")->c) 248 | //print_ubacktrace() 249 | return !\@cast(f, "Closure", "$lua_path")->c->isC 250 | } 251 | 252 | 253 | function getstr(ts) { 254 | return user_string_warn(&\@cast(ts, "TString", "$lua_path")[1]) 255 | } 256 | 257 | 258 | function funcinfo(cl) { 259 | //printf("funcinfo: cl: %x\\n", cl) 260 | if (\@cast(cl, "Closure", "$lua_path")->c->isC) { 261 | cfunc = \@cast(cl, "Closure", "$lua_path")->c->f 262 | sym = cfuncs[cfunc] 263 | if (sym != "") { 264 | info = sym 265 | 266 | } else { 267 | sym = "C:" . usymname(cfunc) 268 | cfuncs[cfunc] = sym 269 | info = sym 270 | } 271 | 272 | } else { 273 | src = \@cast(cl, "Closure", "$lua_path")->l->p->source 274 | info = sprintf("%s:%d", getstr(src), \@cast(cl, "Closure", "$lua_path")->l->p->linedefined); 275 | } 276 | 277 | return info 278 | } 279 | 280 | 281 | function lua_getinfo(L, i_ci) { 282 | ci = 0 /* CallInfo *ci */ 283 | f = 0 /* Closure *f */ 284 | 285 | if (i_ci != 0) { 286 | base_ci = \@cast(L, "lua_State", "$lua_path")->base_ci 287 | ci = base_ci + i_ci; 288 | f = ci_func(ci) 289 | //printf("lua_getinfo: ci=%x, f=%x, isLua=%d\\n", ci, f, f_isLua(ci)); 290 | } 291 | 292 | if (f == 0) { 293 | /* info_tailcall() */ 294 | return "[tail]" 295 | } 296 | 297 | /* f != 0 */ 298 | finfo = funcinfo(f) 299 | 300 | //fname = getfuncname(L, ci) 301 | 302 | return finfo 303 | } 304 | 305 | 306 | function lua_getstack(L, level) { 307 | ci = \@cast(L, "lua_State", "$lua_path")->ci 308 | base_ci = \@cast(L, "lua_State", "$lua_path")->base_ci 309 | 310 | //printf("L=%x, ci=%x, base_ci=%x\\n", L, ci, base_ci) 311 | if (ci_offset == 0) { 312 | ci_offset = &\@cast(0, "CallInfo", "$lua_path")[1] 313 | } 314 | 315 | //printf("ci offset: %d\\n", ci_offset) 316 | 317 | for (; level > 0 && ci > base_ci; ci -= ci_offset) { 318 | level--; 319 | 320 | //tt = \@cast(ci, "CallInfo", "$lua_path")->func->tt 321 | //printf("ci tt: %d\\n", tt) 322 | 323 | if (f_isLua(ci)) { /* Lua function? */ 324 | tailcalls = \@cast(ci, "CallInfo", "$lua_path")->tailcalls 325 | //printf("it is a lua func! tailcalls=%d\\n", tailcalls) 326 | level -= tailcalls; /* skip lost tail calls */ 327 | } 328 | } 329 | 330 | if (level == 0 && ci > base_ci) { /* level found? */ 331 | //printf("lua_getstack: ci=%x\\n", ci); 332 | 333 | //tt = \@cast(ci, "CallInfo", "$lua_path")->func->tt 334 | //printf("ci tt: %d\\n", tt) 335 | 336 | //ff = &\@cast(ci, "CallInfo", "$lua_path")->func->value->gc->cl 337 | 338 | //isC = \@cast(ci, "CallInfo", "$lua_path")->func->value->gc->cl->c->isC 339 | //printf("isC: %d, %d ff=%x\\n", isC, \@cast(ff, "Closure", "$lua_path")->c->isC, ff) 340 | 341 | //f = ci_func(ci) 342 | //printf("lua_getstack: ci=%x, f=%x, isLua=%d\\n", ci, f, f_isLua(ci)); 343 | 344 | return ci - base_ci; 345 | } 346 | 347 | if (level < 0) { /* level is of a lost tail call? */ 348 | return 0; 349 | } 350 | 351 | return -1; 352 | } 353 | 354 | 355 | probe timer.profile { 356 | my_pid = pid() 357 | if ($condition) { 358 | L = lua_states[my_pid] 359 | //printf("HERE: %d\\n", L) 360 | if (L) { 361 | //println("===============") 362 | stack = "" 363 | level = 0 364 | prev_is_tail = 0 365 | while (1) { 366 | //printf("--- begin: l=%d, u=%d\\n", level, user_mode()) 367 | i_ci = lua_getstack(L, level++) 368 | 369 | //printf("lua_getstack returned: %d\\n", i_ci) 370 | 371 | if (i_ci < 0 || level > 200) { 372 | break 373 | } 374 | 375 | //printf("%d: i_ci: %s\\n", level, lua_getinfo(L, i_ci)) 376 | frame = lua_getinfo(L, i_ci) 377 | if (frame == "[tail]") { 378 | if (prev_is_tail) { 379 | continue 380 | } 381 | 382 | prev_is_tail = 1 383 | 384 | } else { 385 | prev_is_tail = 0 386 | } 387 | 388 | stack .= frame . "\\n" 389 | } 390 | 391 | if (stack != "") { 392 | bts[stack] <<< 1 393 | } 394 | } 395 | } 396 | } 397 | 398 | $postamble 399 | _EOC_ 400 | 401 | } else { 402 | # LuaJIT 2.0 403 | my $L = qq{\@cast(L, "lua_State", "$lua_path")}; 404 | my $mref = qq{\@cast(mref, "MRef", "$lua_path")}; 405 | my $tvalue = qq{\@cast(tvalue, "TValue", "$lua_path")}; 406 | my $gcr = qq{\@cast(gcr, "GCRef", "$lua_path")}; 407 | #my $pc = qq{\@cast(pc, "uint32_t", "$lua_path")}; # BCins is uint32_t 408 | my $sizeof_TValue = qq{\&\@cast(0, "TValue", "$lua_path")[1]}; 409 | my $fn = qq{\@cast(fn, "GCfunc", "$lua_path")}; 410 | my $pt = qq{\@cast(pt, "GCproto", "$lua_path")}; 411 | my $gcobj = qq{\@cast(gcobj, "GCobj", "$lua_path")}; 412 | my $sizeof_GCproto = qq{\&\@cast(0, "GCproto", "$lua_path")[1]}; 413 | my $sizeof_GCstr = qq{\&\@cast(0, "GCstr", "$lua_path")[1]}; 414 | 415 | $stap_src = <<_EOC_; 416 | $preamble 417 | 418 | global bts 419 | global lua_states 420 | global cfuncs 421 | 422 | 423 | probe process("$lua_path").function("lua_resume"), 424 | process("$lua_path").function("lua_pcall") 425 | { 426 | //printf("L = %p\\n", \$L) 427 | my_pid = pid() 428 | if ($condition) { 429 | lua_states[my_pid] = \$L 430 | } 431 | } 432 | 433 | 434 | probe process("$lua_path").function("lua_yield") { 435 | //printf("L = %p\\n", \$L) 436 | my_pid = pid() 437 | if ($condition) { 438 | lua_states[my_pid] = 0 439 | } 440 | } 441 | 442 | 443 | probe process("$nginx_path").function("ngx_http_lua_del_thread") { 444 | my_pid = pid() 445 | if ($condition) { 446 | lua_states[my_pid] = 0 447 | } 448 | } 449 | 450 | 451 | /* 452 | function dd(s) { 453 | //printf("--- %s\\n", s) 454 | } 455 | */ 456 | 457 | 458 | function tvref(mref) { 459 | return $mref->ptr32 460 | } 461 | 462 | 463 | function gcref(gcr) { 464 | return $gcr->gcptr32 465 | } 466 | 467 | 468 | function frame_gc(tvalue) { 469 | return gcref(\&$tvalue->fr->func) 470 | } 471 | 472 | 473 | function frame_ftsz(tvalue) { 474 | return $tvalue->fr->tp->ftsz 475 | } 476 | 477 | 478 | function frame_type(f) { 479 | /* FRAME_TYPE == 3 */ 480 | return frame_ftsz(f) & 3 481 | } 482 | 483 | 484 | function frame_typep(f) { 485 | /* FRAME_TYPEP == 7 */ 486 | return frame_ftsz(f) & 7 487 | } 488 | 489 | 490 | function frame_islua(f) { 491 | /* FRAME_LUA == 0 */ 492 | return frame_type(f) == 0 493 | } 494 | 495 | 496 | function frame_pc(tvalue) { 497 | return $tvalue->fr->tp->pcr->ptr32 498 | } 499 | 500 | 501 | function bc_a(i) { 502 | //dd(sprintf("instruction %d", i)) 503 | return (i >> 8) & 0xff 504 | } 505 | 506 | 507 | function frame_prevl(f) { 508 | pc = frame_pc(f) 509 | return f - (1 + bc_a(user_uint32(pc - 4))) * $sizeof_TValue 510 | } 511 | 512 | 513 | function frame_isvarg(f) { 514 | /* FRAME_VARG == 3 */ 515 | return frame_typep(f) == 3 516 | } 517 | 518 | 519 | function frame_sized(f) { 520 | /* 521 | * FRAME_TYPE == 3 522 | * FRAME_P == 4 523 | * FRAME_TYPEP == (FRAME_TYPE | FRAME_P) 524 | */ 525 | 526 | return frame_ftsz(f) & ~(3|4) 527 | } 528 | 529 | 530 | function frame_prevd(f) { 531 | return f - frame_sized(f) 532 | } 533 | 534 | 535 | function lua_getstack(L, level) { 536 | /* code from function lj_debug_frame in LuaJIT 2.0 */ 537 | 538 | /* TValue *frame, *nextframe, *bot; */ 539 | bot = tvref(\&$L->stack) // TValue * 540 | found_frame = 0 541 | 542 | for (nextframe = frame = $L->base - $sizeof_TValue; frame > bot; ) { 543 | //dd(sprintf("checking frame: %d, level: %d", frame, level)) 544 | 545 | /* Traverse frames backwards */ 546 | if (frame_gc(frame) == L) { 547 | //dd("Skip dummy frames. See lj_meta_call") 548 | level++ 549 | } 550 | 551 | if (level-- == 0) { 552 | //dd(sprintf("Level found, frame=%p, nextframe=%p", frame, nextframe)) 553 | size = (nextframe - frame) / $sizeof_TValue 554 | found_frame = 1 555 | break 556 | } 557 | 558 | nextframe = frame 559 | if (frame_islua(frame)) { 560 | frame = frame_prevl(frame) 561 | 562 | } else { 563 | if (frame_isvarg(frame)) { 564 | //dd("Skip vararg pseudo-frame") 565 | level++ 566 | } 567 | 568 | frame = frame_prevd(frame) 569 | } 570 | } 571 | 572 | if (!found_frame) { 573 | //dd("Level not found") 574 | size = level 575 | frame = 0 576 | } 577 | 578 | /* code from function lua_getstatck in LuaJIT 2.0 */ 579 | 580 | if (frame) { 581 | i_ci = (size << 16) + (frame - bot) / $sizeof_TValue 582 | return i_ci 583 | } 584 | 585 | return -1 586 | } 587 | 588 | 589 | function frame_func(f) { 590 | gcobj = frame_gc(f) 591 | return \&$gcobj->fn 592 | } 593 | 594 | 595 | function isluafunc(fn) { 596 | /* FF_LUA == 0 */ 597 | return $fn->c->ffid == 0 598 | } 599 | 600 | 601 | function funcproto(fn) { 602 | return $fn->l->pc->ptr32 - $sizeof_GCproto 603 | } 604 | 605 | 606 | function strref(r) { 607 | gcobj = gcref(r) 608 | return \&$gcobj->str 609 | } 610 | 611 | 612 | function proto_chunkname(pt) { 613 | return strref(\&$pt->chunkname) 614 | } 615 | 616 | 617 | function strdata(s) { 618 | return s + $sizeof_GCstr 619 | } 620 | 621 | 622 | function lua_getinfo(L, i_ci) { 623 | /* code from function lj_debug_getinfo in LuaJIT 2.0 */ 624 | 625 | offset = (i_ci & 0xffff) 626 | if (offset == 0) { 627 | //dd(sprintf("assertion failed: offset == 0: i_ci=%x", i_ci)) 628 | return "" 629 | } 630 | 631 | frame = tvref(\&$L->stack) + offset * $sizeof_TValue 632 | 633 | size = (i_ci >> 16) 634 | if (size) { 635 | nextframe = frame + size * $sizeof_TValue 636 | 637 | } else { 638 | nextframe = 0 639 | } 640 | 641 | //dd(sprintf("getinfo: frame=%p nextframe=%p", frame, nextframe)) 642 | 643 | maxstack = tvref(\&$L->maxstack) 644 | if (!(frame <= maxstack && (!nextframe || nextframe <= maxstack))) { 645 | //dd("assertion failed: frame <= maxstack && (!nextframe || nextframe <= maxstack)") 646 | return "" 647 | } 648 | 649 | fn = frame_func(frame) 650 | 651 | /* LJ_TFUNC == ~8u */ 652 | if (!($fn->c->gct == 8)) { 653 | //dd(sprintf("assertion failed: fn->c.gct == ~LJ_TFUNC: %d", $fn->c->gct)) 654 | return "" 655 | } 656 | 657 | if (isluafunc(fn)) { 658 | pt = funcproto(fn) 659 | firstline = $pt->firstline 660 | name = proto_chunkname(pt) /* GCstr *name */ 661 | src = strdata(name) 662 | return sprintf("%s:%d", user_string(src), firstline) 663 | } 664 | 665 | /* being a C function */ 666 | 667 | cfunc = $fn->c->f 668 | sym = cfuncs[cfunc] 669 | if (sym != "") { 670 | return sym 671 | } 672 | 673 | sym = "C:" . usymname(cfunc) 674 | cfuncs[cfunc] = sym 675 | return sym 676 | } 677 | 678 | 679 | probe timer.profile { 680 | my_pid = pid() 681 | if ($condition) { 682 | L = lua_states[my_pid] 683 | //printf("HERE: %d\\n", L) 684 | if (L) { 685 | //println("===============") 686 | stack = "" 687 | level = 0 688 | prev_is_tail = 0 689 | while (1) { 690 | //dd(sprintf("begin: l=%d, u=%d", level, user_mode())) 691 | i_ci = lua_getstack(L, level++) 692 | 693 | //printf("lua_getstack returned: %d\\n", i_ci) 694 | 695 | if (i_ci < 0 || level > 100) { 696 | break 697 | } 698 | 699 | //printf("%d: i_ci: %s\\n", level, lua_getinfo(L, i_ci)) 700 | frame = lua_getinfo(L, i_ci) 701 | if (frame == "") { 702 | stack = "" 703 | break 704 | } 705 | 706 | //dd(sprintf("got frame: %s", frame)) 707 | 708 | if (frame == "[tail]") { 709 | if (prev_is_tail) { 710 | continue 711 | } 712 | 713 | prev_is_tail = 1 714 | 715 | } else { 716 | prev_is_tail = 0 717 | } 718 | 719 | stack .= frame . "\\n" 720 | } 721 | 722 | if (stack != "") { 723 | bts[stack] <<< 1 724 | } 725 | } 726 | } 727 | } 728 | 729 | $postamble 730 | _EOC_ 731 | } 732 | 733 | if ($dump_src) { 734 | print $stap_src; 735 | exit; 736 | } 737 | 738 | my %so_files; 739 | { 740 | for my $pid (@$child_pids) { 741 | my $maps_file = "/proc/$pid/maps"; 742 | open my $in, $maps_file 743 | or die "Cannot open $maps_file for reading: $!\n"; 744 | 745 | while (<$in>) { 746 | if (/\s+(\/\S+\.so)$/) { 747 | if (!exists $so_files{$1}) { 748 | $so_files{$1} = 1; 749 | #warn $1, "\n"; 750 | } 751 | } 752 | } 753 | 754 | close $in; 755 | } 756 | } 757 | 758 | my $d_so_args; 759 | if (%so_files) { 760 | $d_so_args = join " ", map { "-d $_" } sort keys %so_files; 761 | 762 | } else { 763 | $d_so_args = ''; 764 | } 765 | 766 | my $extra_args = ''; 767 | if (@$child_pids == 1) { 768 | $extra_args .= " -x " . $child_pids->[0]; 769 | } 770 | 771 | my $cmd = "stap --skip-badvars --all-modules -d '$nginx_path' --ldd $d_so_args $extra_args $stap_args -"; 772 | #warn $cmd; 773 | open my $in, "|$cmd" 774 | or die "Cannot run stap: $!\n"; 775 | 776 | print $in $stap_src; 777 | 778 | close $in; 779 | 780 | sub usage { 781 | return <<'_EOC_'; 782 | Usage: 783 | ngx-sample-lua-bt [optoins] 784 | 785 | Options: 786 | -a Pass extra arguments to the stap utility. 787 | -d Dump out the systemtap script source. 788 | -h Print this usage. 789 | -l Only output most frenquent backtrace samples. 790 | (Default to 1024) 791 | --lua51 Specify that the Nginx is using the standard Lua 5.1. 792 | interpreter. 793 | --luajit20 Specify that the Nginx is using LuaJIT 2.0. 794 | -p Specify the nginx process pid. 795 | -t Specify the number of seconds for sampling. 796 | 797 | Examples: 798 | ngx-sample-lua-bt --lua51 -p 12345 -t 10 799 | ngx-sample-lua-bt --luajit20 -p 12345 -t 5 -a '-DMAXACTION=100000' 800 | _EOC_ 801 | } 802 | 803 | sub get_child_processes { 804 | my $pid = shift; 805 | my @files = glob "/proc/[0-9]*/stat"; 806 | my @children; 807 | for my $file (@files) { 808 | #print "file: $file\n"; 809 | if ($file =~ m{^/proc/$pid/}) { 810 | next; 811 | } 812 | 813 | open my $in, $file or next; 814 | my $line = <$in>; 815 | close $in; 816 | if ($line =~ /^(\d+) \S+ \S+ (\d+)/) { 817 | my ($child, $parent) = ($1, $2); 818 | if ($parent eq $pid) { 819 | push @children, $child; 820 | } 821 | } 822 | } 823 | 824 | @children = sort { $a <=> $b } @children; 825 | return \@children; 826 | } 827 | 828 | sub gen_pid_test_condition { 829 | my $pids = shift; 830 | my @c; 831 | for my $pid (@$pids) { 832 | push @c, "my_pid == $pid"; 833 | } 834 | return join " || ", @c; 835 | } 836 | -------------------------------------------------------------------------------- /ngx-shm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhp:n:", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No nginx process pid specified by the -p option\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid\n"; 26 | } 27 | 28 | my $stap_args = $opts{a} || ''; 29 | 30 | if ($^O ne 'linux') { 31 | die "Only linux is supported but I am on $^O.\n"; 32 | } 33 | 34 | my $exec_file = "/proc/$pid/exe"; 35 | if (!-f $exec_file) { 36 | die "Nginx process $pid is not running or ", 37 | "you do not have enough permissions.\n"; 38 | } 39 | 40 | my $nginx_path = readlink $exec_file; 41 | 42 | my $ver = `stap --version 2>&1`; 43 | if (!defined $ver) { 44 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 45 | } 46 | 47 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 48 | my $v = $1; 49 | if ($v < 2.1) { 50 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 51 | } 52 | 53 | } else { 54 | die "ERROR: unknown version of systemtap:\n$ver\n"; 55 | } 56 | 57 | my $stap_src; 58 | 59 | my $zone = $opts{n}; 60 | 61 | my $preamble = <<_EOC_; 62 | probe begin { 63 | printf("Tracing %d ($nginx_path)...\\n\\n", target()) 64 | } 65 | _EOC_ 66 | chop $preamble; 67 | 68 | if ($zone) { 69 | my $quoted_zone = quote_str($zone); 70 | 71 | my $zone_name_len = length $zone; 72 | 73 | $stap_src = <<_EOC_; 74 | $preamble 75 | 76 | probe process("$nginx_path").function("ngx_process_events_and_timers"), 77 | process("$nginx_path").function("ngx_http_init_request")!, 78 | process("$nginx_path").function("ngx_http_create_request") 79 | { 80 | if (pid() == target()) { 81 | 82 | found = 0 83 | begin = local_clock_us() 84 | 85 | pagesize = \@var("ngx_pagesize\@ngx_alloc.c") 86 | part = &\@var("ngx_cycle\@ngx_cycle.c")->shared_memory->part 87 | zone = \@cast(part, "ngx_list_part_t")->elts 88 | for (i = 0; ; i++) { 89 | if (i >= \@cast(part, "ngx_list_part_t")->nelts) { 90 | if (\@cast(part, "ngx_list_part_t")->next == 0) { 91 | break 92 | } 93 | 94 | part = \@cast(part, "ngx_list_part_t")->next 95 | zone = \@cast(part, "ngx_list_part_t")->elts 96 | i = 0 97 | } 98 | 99 | shm = &\@cast(zone, "ngx_shm_zone_t")[i]->shm 100 | 101 | name = &\@cast(shm, "ngx_shm_t")->name 102 | 103 | if (\@cast(name, "ngx_str_t")->len != $zone_name_len) { 104 | continue; 105 | } 106 | 107 | zone_name = user_string_n(\@cast(name, "ngx_str_t")->data, $zone_name_len) 108 | 109 | if (zone_name != $quoted_zone) { 110 | continue; 111 | } 112 | 113 | init = \@cast(zone, "ngx_shm_zone_t")[i]->init 114 | addr = \@cast(shm, "ngx_shm_t")->addr 115 | addr2 = \@cast(addr, "ngx_slab_pool_t")->addr 116 | 117 | if (addr != addr2) { 118 | error("shm zone \\"" . zone_name . "\\" is corrupted or " 119 | . "not yet loaded into the process memory space: " 120 | . "pool != pool->addr " 121 | . sprintf("(pool->addr: %p)", addr2)) 122 | exit() 123 | } 124 | 125 | init_name = usymname(init) 126 | owner = str_replace(str_replace(init_name, "_init", ""), "_zone", "") 127 | 128 | printf("shm zone \\"%s\\"\\n", zone_name) 129 | printf(" owner: %s\\n", owner) 130 | printf(" total size: %d KB\\n", \@cast(shm, "ngx_shm_t")->size / 1024) 131 | 132 | pages = 0 133 | free = &\@cast(addr, "ngx_slab_pool_t")->free 134 | blocks = 0 135 | 136 | /* 137 | printf("shm: %p, shm->addr: %p, free: %p, addr2: %p, next: %p\\n", 138 | shm, addr, free, addr2, 139 | \@cast(free, "ngx_slab_page_t")->next) 140 | */ 141 | 142 | for (page = \@cast(free, "ngx_slab_page_t")->next; 143 | page && page != free; 144 | page = \@cast(page, "ngx_slab_page_t")->next) 145 | { 146 | //printf("page: %p\\n", page) 147 | pages += \@cast(page, "ngx_slab_page_t")->slab 148 | if (++blocks % 1000 == 0) { 149 | printf("\\r free pages: %d KB (%d pages, %d blocks)", 150 | pages * pagesize / 1024, pages, blocks) 151 | } 152 | } 153 | 154 | printf("\\r free pages: %d KB (%d pages, %d blocks)\\n", 155 | pages * pagesize / 1024, pages, blocks) 156 | 157 | found = 1 158 | break 159 | } 160 | 161 | if (!found) { 162 | printf("shm zone \\"%s\\" not found.\\n", $quoted_zone) 163 | } 164 | 165 | elapsed = local_clock_us() - begin 166 | printf("\\n%d microseconds elapsed in the probe handler.\\n", elapsed) 167 | 168 | exit() 169 | 170 | } /* pid() == target() */ 171 | } 172 | _EOC_ 173 | 174 | } else { 175 | # no zone name specified 176 | 177 | $stap_src = <<_EOC_; 178 | $preamble 179 | 180 | probe process("$nginx_path").function("ngx_process_events_and_timers"), 181 | process("$nginx_path").function("ngx_http_handler") 182 | { 183 | if (pid() == target()) { 184 | 185 | begin = local_clock_us() 186 | 187 | part = &\@var("ngx_cycle\@ngx_cycle.c")->shared_memory->part 188 | zone = \@cast(part, "ngx_list_part_t")->elts 189 | for (i = 0; ; i++) { 190 | if (i >= \@cast(part, "ngx_list_part_t")->nelts) { 191 | if (\@cast(part, "ngx_list_part_t")->next == 0) { 192 | break 193 | } 194 | 195 | part = \@cast(part, "ngx_list_part_t")->next 196 | zone = \@cast(part, "ngx_list_part_t")->elts 197 | i = 0 198 | } 199 | 200 | shm = &\@cast(zone, "ngx_shm_zone_t")[i]->shm 201 | addr = \@cast(shm, "ngx_shm_t")->addr 202 | init = \@cast(zone, "ngx_shm_zone_t")[i]->init 203 | name = &\@cast(shm, "ngx_shm_t")->name 204 | 205 | zone_name = user_string_n(\@cast(name, "ngx_str_t")->data, 206 | \@cast(name, "ngx_str_t")->len) 207 | 208 | addr2 = \@cast(addr, "ngx_slab_pool_t")->addr 209 | 210 | if (addr != addr2) { 211 | println("WARNING: shm zone \\"" . zone_name . "\\" is " 212 | . "corrupted or " 213 | . "not yet loaded into the process memory space: " 214 | . "pool != pool->addr " 215 | . sprintf("(pool->addr: %p)", addr2)) 216 | exit() 217 | } 218 | 219 | init_name = usymname(init) 220 | owner = str_replace(str_replace(init_name, "_init", ""), "_zone", "") 221 | 222 | printf("shm zone \\"%s\\"\\n owner: %s\\n total size: %d KB\\n\\n", 223 | user_string_n(\@cast(name, "ngx_str_t")->data, 224 | \@cast(name, "ngx_str_t")->len), 225 | owner, \@cast(shm, "ngx_shm_t")->size / 1024) 226 | } 227 | 228 | println("Use the -n option to see more details about each zone.") 229 | 230 | elapsed = local_clock_us() - begin 231 | printf("%d microseconds elapsed in the probe.\\n", elapsed) 232 | exit() 233 | } /* pid() == target() */ 234 | } 235 | _EOC_ 236 | } 237 | 238 | if ($opts{d}) { 239 | print $stap_src; 240 | exit; 241 | } 242 | 243 | if (!$zone) { 244 | $stap_args .= " --skip-badvars" 245 | } 246 | 247 | open my $in, "|stap $stap_args -x $pid -" 248 | or die "Cannot run stap: $!\n"; 249 | 250 | print $in $stap_src; 251 | 252 | close $in; 253 | 254 | sub usage { 255 | return <<'_EOC_'; 256 | Usage: 257 | ngx-shm [optoins] 258 | 259 | Options: 260 | -a Pass extra arguments to the stap utility. 261 | -d Dump out the systemtap script source. 262 | -h Print this usage. 263 | -n Show details about the shm zone named . 264 | -p Specify the nginx worker process pid. 265 | 266 | Examples: 267 | ngx-shm -p 12345 268 | ngx-shm -p 12345 -n dogs 269 | ngx-shm -p 12345 -n my_zone_name -a '-DMAXACTION=100000' 270 | _EOC_ 271 | } 272 | 273 | sub quote_str { 274 | my $s = shift; 275 | $s =~ s/\\/\\\\/g; 276 | $s =~ s/"/\\"/g; 277 | $s =~ s/\n/\\n/g; 278 | $s =~ s/\t/\\t/g; 279 | $s =~ s/\r/\\r/g; 280 | return qq{"$s"}; 281 | } 282 | 283 | -------------------------------------------------------------------------------- /resolve-inlines: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | my $infile = shift 7 | or die "No backtrace file specified.\n"; 8 | 9 | open my $in, $infile or 10 | die "Cannot open $infile for reading: $!\n"; 11 | 12 | my $execfile = shift 13 | or die "No exec file path specified.\n"; 14 | 15 | my $cached_limit = 10000; 16 | my $cached_count = 0; 17 | my %cached; 18 | my $tip = 1; 19 | while (<$in>) { 20 | if (/^\t\d+$/) { 21 | $tip = 1; 22 | print; 23 | next; 24 | } 25 | if (/^ (0x[0-9a-f]+)\b/i) { 26 | my $stack = $1; 27 | my $line = $_; 28 | 29 | my $frames = $cached{$stack}; 30 | if (!$frames) { 31 | $frames = []; 32 | my $addr; 33 | if (!$tip) { 34 | no warnings 'portable'; 35 | $addr = hex($stack); 36 | # XXX I don't know why I need to substract by 1, 37 | # but I need it to get the correct result at least with 38 | # gcc 4.8 39 | $addr = sprintf("%x", $addr - 1); 40 | 41 | } else { 42 | $addr = $stack; 43 | } 44 | my $out = `addr2line -fsip -e $execfile $addr`; 45 | #warn "$addr: $out"; 46 | open my $t, "<", \$out; 47 | while (<$t>) { 48 | if (/(\w+) at (\S+:\d+)/) { 49 | push @$frames, [$1, $2]; 50 | #warn "$1\n"; 51 | } 52 | } 53 | close $t; 54 | if (++$cached_count > $cached_limit) { 55 | # prevent the cache from growing forever... 56 | %cached = (); 57 | } 58 | $cached{$stack} = $frames; 59 | } 60 | if (@$frames > 1) { 61 | #warn "HIT"; 62 | for my $fr (@$frames) { 63 | my ($func, $loc) = @$fr; 64 | print " $stack : $func+0x0/0x0 [$loc]\n"; 65 | } 66 | 67 | } else { 68 | print $line; 69 | } 70 | #print "@frames\n"; 71 | #print "$cnt\n"; 72 | 73 | } else { 74 | print; 75 | } 76 | undef $tip; 77 | } 78 | 79 | close $in; 80 | -------------------------------------------------------------------------------- /resolve-src-lines: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | my $infile = shift 7 | or die "No backtrace file specified.\n"; 8 | 9 | open my $in, $infile or 10 | die "Cannot open $infile for reading: $!\n"; 11 | 12 | my $execfile = shift 13 | or die "No exec file path specified.\n"; 14 | 15 | my $cached_limit = 10000; 16 | my $cached_count = 0; 17 | my %cached; 18 | my $tip = 1; 19 | while (<$in>) { 20 | if (/^\t\d+$/) { 21 | $tip = 1; 22 | print; 23 | next; 24 | } 25 | if (/^ (0x[0-9a-f]+)\b/i) { 26 | my $stack = $1; 27 | my $line = $_; 28 | 29 | my $frames = $cached{$stack}; 30 | if (!$frames) { 31 | $frames = []; 32 | my $addr; 33 | if (!$tip) { 34 | no warnings 'portable'; 35 | $addr = hex($stack); 36 | # XXX I don't know why I need to substract by 1, 37 | # but I need it to get the correct result at least with 38 | # gcc 4.8 39 | $addr = sprintf("%x", $addr - 1); 40 | 41 | } else { 42 | $addr = $stack; 43 | } 44 | my $out = `addr2line -fsip -e $execfile $addr`; 45 | #warn "$addr: $out"; 46 | open my $t, "<", \$out; 47 | while (<$t>) { 48 | if (/(\w+) at (\S+:\d+)/) { 49 | push @$frames, [$1, $2]; 50 | #warn "$1\n"; 51 | } 52 | } 53 | close $t; 54 | if (++$cached_count > $cached_limit) { 55 | # prevent the cache from growing forever... 56 | %cached = (); 57 | } 58 | $cached{$stack} = $frames; 59 | } 60 | if (@$frames) { 61 | #warn "HIT"; 62 | for my $fr (@$frames) { 63 | my ($func, $loc) = @$fr; 64 | print " $stack : $func($loc)+0x0/0x0 [$loc]\n"; 65 | } 66 | 67 | 68 | } else { 69 | print $line; 70 | } 71 | #print "@frames\n"; 72 | #print "$cnt\n"; 73 | 74 | } else { 75 | print; 76 | } 77 | undef $tip; 78 | } 79 | 80 | close $in; 81 | -------------------------------------------------------------------------------- /reverse-bts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | my @bt; 4 | while (<>) { 5 | if (/^\t\d+/) { 6 | print reverse @bt; 7 | undef @bt; 8 | print; 9 | } else { 10 | push @bt, $_; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /sample-bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Std qw( getopts ); 10 | 11 | my %opts; 12 | 13 | getopts("a:dhl:p:t:uk", \%opts) 14 | or die usage(); 15 | 16 | if ($opts{h}) { 17 | print usage(); 18 | exit; 19 | } 20 | 21 | my $pid = $opts{p} 22 | or die "No process pid specified by the -p option.\n"; 23 | 24 | if ($pid !~ /^\d+$/) { 25 | die "Bad -p option value \"$pid\": not look like a pid.\n"; 26 | } 27 | 28 | my $condition = "pid() == target()"; 29 | 30 | my $time = $opts{t} 31 | or die "No -t option specified.\n"; 32 | 33 | my $limit = $opts{l} || 1024; 34 | 35 | my $user_space = $opts{u}; 36 | my $kernel_space = $opts{k}; 37 | 38 | if (!$user_space && !$kernel_space) { 39 | die "Neither -u nor -k is specified.\n", 40 | "(You should choose to sample in the user space or ", 41 | "in the kernel space or in both.)\n"; 42 | } 43 | 44 | if ($time !~ /^\d+$/) { 45 | die "Bad time value specified in the -t option: $time\n"; 46 | } 47 | 48 | my $stap_args = $opts{a} || ''; 49 | 50 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 51 | $stap_args .= " -DMAXACTION=100000" 52 | } 53 | 54 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 55 | $stap_args .= " -DMAXMAPENTRIES=5000" 56 | } 57 | 58 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 59 | $stap_args .= " -DMAXBACKTRACE=200" 60 | } 61 | 62 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 63 | $stap_args .= " -DMAXSTRINGLEN=2048" 64 | } 65 | 66 | #warn $stap_args; 67 | 68 | if ($^O ne 'linux') { 69 | die "Only linux is supported but I am on $^O.\n"; 70 | } 71 | 72 | my $exec_file = "/proc/$pid/exe"; 73 | if (!-f $exec_file) { 74 | die "Process $pid is not running or ", 75 | "you do not have enough permissions.\n"; 76 | } 77 | 78 | my $exec_path = readlink $exec_file; 79 | 80 | my $ver = `stap --version 2>&1`; 81 | if (!defined $ver) { 82 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 83 | } 84 | 85 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 86 | my $v = $1; 87 | if ($v < 2.1) { 88 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 89 | } 90 | 91 | } else { 92 | die "ERROR: unknown version of systemtap:\n$ver\n"; 93 | } 94 | 95 | my $context; 96 | if ($user_space) { 97 | if ($kernel_space) { 98 | $context = 'both user-space and kernel-space'; 99 | 100 | } else { 101 | $context = 'user-space only'; 102 | } 103 | 104 | } else { 105 | $context = 'kernel-space only'; 106 | } 107 | 108 | my $preamble = <<_EOC_; 109 | probe begin { 110 | warn(sprintf("Tracing %d ($exec_path) in $context...\\n", target())) 111 | } 112 | _EOC_ 113 | 114 | my $stap_src; 115 | 116 | if ($user_space) { 117 | if ($kernel_space) { 118 | # in both user-space and kernel-space 119 | $stap_src = <<_EOC_; 120 | $preamble 121 | 122 | global bts; 123 | global quit = 0 124 | 125 | probe timer.profile { 126 | if ($condition) { 127 | if (!quit) { 128 | bts[backtrace(), ubacktrace()] <<< 1 129 | 130 | } else { 131 | 132 | foreach ([sys, usr] in bts- limit $limit) { 133 | print_stack(sys) 134 | print_ustack(usr) 135 | printf("\\t%d\\n", \@count(bts[sys, usr])) 136 | } 137 | 138 | exit() 139 | } 140 | } 141 | } 142 | 143 | probe timer.s($time) { 144 | nstacks = 0 145 | foreach ([a, b] in bts limit 1) { 146 | nstacks++ 147 | } 148 | 149 | if (nstacks == 0) { 150 | warn("No backtraces found. Quitting now...\\n") 151 | exit() 152 | 153 | } else { 154 | warn("Time's up. Quitting now...(it may take a while)\\n") 155 | quit = 1 156 | } 157 | } 158 | _EOC_ 159 | 160 | } else { 161 | # in user-space only 162 | $stap_src = <<_EOC_; 163 | $preamble 164 | 165 | global bts; 166 | global quit = 0; 167 | 168 | probe timer.profile { 169 | if ($condition) { 170 | if (!quit) { 171 | bts[ubacktrace()] <<< 1; 172 | 173 | } else { 174 | 175 | foreach (bt in bts- limit $limit) { 176 | print_ustack(bt); 177 | printf("\\t%d\\n", \@count(bts[bt])); 178 | } 179 | 180 | exit() 181 | } 182 | } 183 | } 184 | 185 | probe timer.s($time) { 186 | nstacks = 0 187 | foreach (bt in bts limit 1) { 188 | nstacks++ 189 | } 190 | 191 | if (nstacks == 0) { 192 | warn("No backtraces found. Quitting now...\\n") 193 | exit() 194 | 195 | } else { 196 | warn("Time's up. Quitting now...(it may take a while)\\n") 197 | quit = 1 198 | } 199 | } 200 | _EOC_ 201 | } 202 | 203 | } else { 204 | # in kernel-space only 205 | $stap_src = <<_EOC_; 206 | $preamble 207 | 208 | global bts; 209 | 210 | probe timer.profile { 211 | if ($condition && !user_mode()) { 212 | bts[backtrace()] <<< 1 213 | } 214 | } 215 | 216 | probe end { 217 | nstacks = 0 218 | foreach (bt in bts limit 1) { 219 | nstacks++ 220 | } 221 | 222 | if (nstacks == 0) { 223 | warn("No backtraces found. Quitting now...\\n") 224 | 225 | } else { 226 | foreach (bt in bts- limit $limit) { 227 | print_stack(bt) 228 | printf("\\t%d\\n", \@count(bts[bt])) 229 | } 230 | } 231 | } 232 | 233 | probe timer.s($time) { 234 | warn("Time's up. Quitting now...(it may take a while)\\n") 235 | exit() 236 | } 237 | _EOC_ 238 | } 239 | 240 | if ($opts{d}) { 241 | print $stap_src; 242 | exit; 243 | } 244 | 245 | my %so_files; 246 | { 247 | my $maps_file = "/proc/$pid/maps"; 248 | open my $in, $maps_file 249 | or die "Cannot open $maps_file for reading: $!\n"; 250 | 251 | while (<$in>) { 252 | if (/\s+(\/\S+\.so(?:\.\S+)?)$/) { 253 | if (!exists $so_files{$1}) { 254 | $so_files{$1} = 1; 255 | #warn $1, "\n"; 256 | } 257 | } 258 | } 259 | 260 | close $in; 261 | } 262 | 263 | my $d_so_args; 264 | if (%so_files) { 265 | $d_so_args = join " ", map { "-d $_" } sort keys %so_files; 266 | 267 | } else { 268 | $d_so_args = ''; 269 | } 270 | 271 | open my $in, "|stap --skip-badvars --all-modules -x $pid -d '$exec_path' --ldd $d_so_args $stap_args -" 272 | or die "Cannot run stap: $!\n"; 273 | 274 | print $in $stap_src; 275 | 276 | close $in; 277 | 278 | sub usage { 279 | return <<'_EOC_'; 280 | Usage: 281 | sample-bt [optoins] 282 | 283 | Options: 284 | -a Pass extra arguments to the stap utility. 285 | -d Dump out the systemtap script source. 286 | -h Print this usage. 287 | -l Only output most frenquent backtrace samples. 288 | (Default to 1024) 289 | -p Specify the user process pid. 290 | -t Specify the number of seconds for sampling. 291 | -u Sample in the user-space. 292 | -k Sample in the kernel-space. 293 | 294 | Examples: 295 | sample-bt -p 12345 -t 10 296 | sample-bt -p 12345 -t 5 -a '-DMAXACTION=100000' 297 | _EOC_ 298 | } 299 | -------------------------------------------------------------------------------- /sample-bt-off-cpu: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | # Thanks Brendan Gregg for the inspiration given here: 6 | # http://dtrace.org/blogs/brendan/2011/07/08/off-cpu-performance-analysis/ 7 | 8 | use 5.006001; 9 | use strict; 10 | use warnings; 11 | 12 | use Getopt::Long qw( GetOptions ); 13 | 14 | GetOptions("a=s", \(my $stap_args), 15 | "d", \(my $dump_src), 16 | "distr", \(my $check_distr), 17 | "h", \(my $help), 18 | "k", \(my $check_kernel), 19 | "l=i", \(my $limit), 20 | "min=i", \(my $min_elapsed), 21 | "p=i", \(my $pid), 22 | "t=i", \(my $time), 23 | "u", \(my $check_user)) 24 | or die usage(); 25 | 26 | if ($help) { 27 | print usage(); 28 | exit; 29 | } 30 | 31 | if (!defined $min_elapsed) { 32 | $min_elapsed = 4; 33 | } 34 | 35 | if (!defined $pid) { 36 | die "No process pid specified by the -p option.\n"; 37 | } 38 | 39 | my $condition = "pid() == target()"; 40 | 41 | if (!defined $time) { 42 | die "No -t option specified.\n"; 43 | } 44 | 45 | if (!defined $limit) { 46 | $limit = 1024; 47 | } 48 | 49 | if (!defined $stap_args) { 50 | $stap_args = ''; 51 | } 52 | 53 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 54 | $stap_args .= " -DMAXACTION=100000"; 55 | } 56 | 57 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 58 | $stap_args .= " -DMAXMAPENTRIES=3000"; 59 | } 60 | 61 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 62 | $stap_args .= " -DMAXBACKTRACE=200"; 63 | } 64 | 65 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 66 | $stap_args .= " -DMAXSTRINGLEN=2048"; 67 | } 68 | 69 | $stap_args .= " -DSTP_NO_OVERLOAD"; 70 | 71 | if ($^O ne 'linux') { 72 | die "Only linux is supported but I am on $^O.\n"; 73 | } 74 | 75 | my $exec_file = "/proc/$pid/exe"; 76 | if (!-f $exec_file) { 77 | die "User process $pid is not running or ", 78 | "you do not have enough permissions.\n"; 79 | } 80 | 81 | my $exec_path = readlink $exec_file; 82 | 83 | my $ver = `stap --version 2>&1`; 84 | if (!defined $ver) { 85 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 86 | } 87 | 88 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 89 | my $v = $1; 90 | if ($v < 2.1) { 91 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 92 | } 93 | 94 | } else { 95 | die "ERROR: unknown version of systemtap:\n$ver\n"; 96 | } 97 | 98 | my $preamble = <<_EOC_; 99 | global quit = 0; 100 | global found 101 | 102 | probe begin { 103 | warn(sprintf("Tracing %d ($exec_path)...\\n", target())) 104 | } 105 | _EOC_ 106 | 107 | my $postamble = <<_EOC_; 108 | probe timer.s($time) { 109 | if (!found) { 110 | warn("No backtraces found. Quitting now...\\n") 111 | exit() 112 | 113 | } else { 114 | warn("Time's up. Quitting now...(it may take a while)\\n") 115 | quit = 1 116 | } 117 | } 118 | _EOC_ 119 | 120 | my $stap_src; 121 | my $d_so_args = ''; 122 | 123 | if ($check_distr) { 124 | $stap_src = <<_EOC_; 125 | global start_time 126 | global elapsed_times 127 | 128 | $preamble 129 | 130 | probe end { 131 | if (!found) { 132 | println("\\nNo samples found yet.") 133 | 134 | } else { 135 | println("=== Off-CPU time distribution (in us) ===") 136 | printf("min/avg/max: %d/%d/%d\\n", 137 | \@min(elapsed_times), \@avg(elapsed_times), \@max(elapsed_times)) 138 | println(\@hist_log(elapsed_times)) 139 | } 140 | } 141 | 142 | probe scheduler.cpu_off { 143 | if ($condition) { 144 | if (!quit) { 145 | start_time[tid()] = gettimeofday_us() 146 | 147 | } else { 148 | exit() 149 | } 150 | } 151 | } 152 | 153 | probe scheduler.cpu_on { 154 | if ($condition && !quit) { 155 | t = tid() 156 | begin = start_time[t] 157 | if (begin > 0) { 158 | elapsed = gettimeofday_us() - begin 159 | if (elapsed >= $min_elapsed) { 160 | found = 1 161 | elapsed_times <<< elapsed 162 | } 163 | delete start_time[t] 164 | } 165 | } 166 | } 167 | 168 | probe timer.s($time) { 169 | println("Exiting...Please wait...") 170 | quit = 1 171 | } 172 | _EOC_ 173 | 174 | } else { 175 | # for sampling backtraces 176 | 177 | my ($gen_bt_code, $print_bt_code); 178 | 179 | if ($check_kernel) { 180 | if ($check_user) { 181 | $gen_bt_code = "backtrace(), ubacktrace()"; 182 | $print_bt_code = <<_EOC_; 183 | foreach ([kbt, ubt] in bts- limit $limit) { 184 | print_stack(kbt) 185 | print_ustack(ubt) 186 | printf("\\t%d\\n", \@sum(bts[kbt, ubt])) 187 | } 188 | _EOC_ 189 | 190 | } else { 191 | $gen_bt_code = "backtrace()"; 192 | $print_bt_code = <<_EOC_; 193 | foreach (bt in bts- limit $limit) { 194 | print_stack(bt) 195 | printf("\\t%d\\n", \@sum(bts[bt])) 196 | } 197 | _EOC_ 198 | } 199 | 200 | } else { 201 | $gen_bt_code = "ubacktrace()"; 202 | $print_bt_code = <<_EOC_; 203 | foreach (bt in bts- limit $limit) { 204 | print_ustack(bt) 205 | printf("\\t%d\\n", \@sum(bts[bt])) 206 | } 207 | _EOC_ 208 | } 209 | 210 | $stap_src = <<_EOC_; 211 | global bts 212 | global start_time 213 | 214 | $preamble 215 | 216 | probe scheduler.cpu_off { 217 | if ($condition) { 218 | if (!quit) { 219 | start_time[tid()] = gettimeofday_us() 220 | 221 | } else { 222 | $print_bt_code 223 | exit() 224 | } 225 | } 226 | } 227 | 228 | probe scheduler.cpu_on { 229 | if ($condition && !quit) { 230 | t = tid() 231 | begin = start_time[t] 232 | if (begin > 0) { 233 | elapsed = gettimeofday_us() - begin 234 | if (elapsed >= $min_elapsed) { 235 | bts[$gen_bt_code] <<< elapsed 236 | found = 1 237 | } 238 | delete start_time[t] 239 | } 240 | } 241 | } 242 | 243 | $postamble 244 | _EOC_ 245 | 246 | my %so_files; 247 | { 248 | my $maps_file = "/proc/$pid/maps"; 249 | open my $in, $maps_file 250 | or die "Cannot open $maps_file for reading: $!\n"; 251 | 252 | while (<$in>) { 253 | if (/\s+(\/\S+\.so(?:\.\S+)?)$/) { 254 | if (!exists $so_files{$1}) { 255 | $so_files{$1} = 1; 256 | #warn $1, "\n"; 257 | } 258 | } 259 | } 260 | 261 | close $in; 262 | } 263 | 264 | if (%so_files) { 265 | $d_so_args = join " ", map { "-d $_" } sort keys %so_files; 266 | } 267 | } 268 | 269 | if ($dump_src) { 270 | print $stap_src; 271 | exit; 272 | } 273 | 274 | 275 | #warn "$d_so_args\n"; 276 | 277 | open my $in, "|stap --skip-badvars --all-modules -x $pid -d '$exec_path' --ldd $d_so_args $stap_args -" 278 | or die "Cannot run stap: $!\n"; 279 | 280 | print $in $stap_src; 281 | 282 | close $in; 283 | 284 | sub usage { 285 | return <<'_EOC_'; 286 | Usage: 287 | sample-bt-off-cpu [optoins] 288 | 289 | Options: 290 | -a Pass extra arguments to the stap utility. 291 | -d Dump out the systemtap script source. 292 | --distr Analyze the distribution of the elapsed off-CPU times only. 293 | -h Print this usage. 294 | -k Analyze kernelspace backtraces. 295 | -l Only output most frenquent backtrace samples. 296 | (Default to 1024) 297 | --min= Minimal elapsed off-CPU time to be tracked. 298 | (Default to 4us) 299 | -p Specify the user process pid. 300 | -t Specify the number of seconds for sampling. 301 | -u Analyze userspace backtraces. 302 | 303 | Examples: 304 | sample-bt-off-cpu -p 12345 -t 10 305 | sample-bt-off-cpu -p 12345 -t 5 -a '-DMAXACTION=100000' 306 | sample-bt-off-cpu --distr -p 12345 -t 10 --min=1 307 | _EOC_ 308 | } 309 | -------------------------------------------------------------------------------- /sample-bt-vfs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Long qw( GetOptions ); 10 | 11 | GetOptions("a=s", \(my $stap_args), 12 | "d", \(my $dump_src), 13 | "h", \(my $help), 14 | "l=i", \(my $limit), 15 | "latency", \(my $check_latency), 16 | "p=i", \(my $pid), 17 | "r", \(my $readonly), 18 | "w", \(my $writeonly), 19 | "t=i", \(my $time)) 20 | or die usage(); 21 | 22 | if ($help) { 23 | print usage(); 24 | exit; 25 | } 26 | 27 | if (!defined $pid) { 28 | die "No process pid specified by the -p option.\n"; 29 | } 30 | 31 | my $condition = "pid() == target()"; 32 | 33 | if (!defined $time) { 34 | die "No -t option specified.\n"; 35 | } 36 | 37 | if (!defined $limit) { 38 | $limit = 1024; 39 | } 40 | 41 | if (!defined $stap_args) { 42 | $stap_args = ''; 43 | } 44 | 45 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 46 | $stap_args .= " -DMAXACTION=100000"; 47 | } 48 | 49 | if ($stap_args !~ /(?:^|\s)-D\s*MAXMAPENTRIES=/) { 50 | $stap_args .= " -DMAXMAPENTRIES=5000"; 51 | } 52 | 53 | if ($stap_args !~ /(?:^|\s)-D\s*MAXBACKTRACE=/) { 54 | $stap_args .= " -DMAXBACKTRACE=200"; 55 | } 56 | 57 | if ($stap_args !~ /(?:^|\s)-D\s*MAXSTRINGLEN=2048/) { 58 | $stap_args .= " -DMAXSTRINGLEN=2048"; 59 | } 60 | 61 | $stap_args .= " -DSTP_NO_OVERLOAD"; 62 | 63 | if ($^O ne 'linux') { 64 | die "Only linux is supported but I am on $^O.\n"; 65 | } 66 | 67 | my $exec_file = "/proc/$pid/exe"; 68 | if (!-f $exec_file) { 69 | die "Nginx process $pid is not running or ", 70 | "you do not have enough permissions.\n"; 71 | } 72 | 73 | my $exec_path = readlink $exec_file; 74 | 75 | my $ver = `stap --version 2>&1`; 76 | if (!defined $ver) { 77 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 78 | } 79 | 80 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 81 | my $v = $1; 82 | if ($v < 2.1) { 83 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 84 | } 85 | 86 | } else { 87 | die "ERROR: unknown version of systemtap:\n$ver\n"; 88 | } 89 | 90 | my $preamble = <<_EOC_; 91 | global bts; 92 | global quit = 0; 93 | 94 | probe begin { 95 | warn(sprintf("Tracing %d ($exec_path)...\\n", target())) 96 | } 97 | _EOC_ 98 | 99 | my $postamble = <<_EOC_; 100 | probe timer.s($time) { 101 | nstacks = 0 102 | foreach (bt in bts limit 10) { 103 | nstacks++ 104 | } 105 | 106 | if (nstacks == 0) { 107 | warn(sprintf("Too few backtraces (%d) found. Quitting now...\\n", nstacks)) 108 | exit() 109 | 110 | } else { 111 | warn("Time's up. Quitting now...(it may take a while)\\n") 112 | quit = 1 113 | } 114 | } 115 | _EOC_ 116 | 117 | my @probes; 118 | if ($readonly) { 119 | @probes = qw(vfs.read); 120 | 121 | } elsif ($writeonly) { 122 | @probes = qw(vfs.write); 123 | 124 | } else { 125 | @probes = qw(vfs.read vfs.write); 126 | } 127 | 128 | my $return_probes = join ", ", map { "$_.return" } @probes; 129 | 130 | my $stap_src; 131 | if ($check_latency) { 132 | # analyze I/O latency 133 | my $entry_probes = join ", ", @probes; 134 | $stap_src = <<_EOC_; 135 | $preamble 136 | 137 | global start_time 138 | 139 | probe $entry_probes { 140 | if ($condition) { 141 | if (!quit) { 142 | if (devname != "N/A") { 143 | start_time = gettimeofday_us() 144 | } 145 | 146 | } else { 147 | foreach (bt in bts- limit $limit) { 148 | print_ustack(bt) 149 | printf("\\t%d\\n", \@sum(bts[bt])) 150 | } 151 | 152 | exit() 153 | } 154 | } 155 | } 156 | 157 | probe $return_probes { 158 | if ($condition) { 159 | if (!quit) { 160 | if (\$return > 0 && devname != "N/A" && start_time) { 161 | bts[ubacktrace()] <<< gettimeofday_us() - start_time 162 | start_time = 0; 163 | } 164 | } 165 | } 166 | } 167 | 168 | $postamble 169 | _EOC_ 170 | 171 | } else { 172 | # analyze I/O data volumn 173 | $stap_src = <<_EOC_; 174 | $preamble 175 | 176 | probe $return_probes { 177 | if ($condition) { 178 | if (!quit) { 179 | if (\$return > 0 && devname != "N/A") { 180 | bts[ubacktrace()] <<< \$return 181 | } 182 | 183 | } else { 184 | 185 | foreach (bt in bts- limit $limit) { 186 | print_ustack(bt); 187 | printf("\\t%d\\n", \@sum(bts[bt])) 188 | } 189 | 190 | exit() 191 | } 192 | } 193 | } 194 | 195 | $postamble 196 | _EOC_ 197 | } 198 | 199 | if ($dump_src) { 200 | print $stap_src; 201 | exit; 202 | } 203 | 204 | open my $in, "|stap --skip-badvars --all-modules -x $pid -d '$exec_path' --ldd $stap_args -" 205 | or die "Cannot run stap: $!\n"; 206 | 207 | print $in $stap_src; 208 | 209 | close $in; 210 | 211 | sub usage { 212 | return <<'_EOC_'; 213 | Usage: 214 | sample-bt-vfs [optoins] 215 | 216 | Options: 217 | -a Pass extra arguments to the stap utility. 218 | -d Dump out the systemtap script source. 219 | -h Print this usage. 220 | --latency Analyze the VFS kernel calls' latency instead 221 | of I/O data volumn. 222 | -l Only output most frenquent backtrace samples. 223 | (Default to 1024) 224 | -p Specify the user process pid. 225 | -r Probe file reading operations only. 226 | -w Probe file writing operations only. 227 | -t Specify the number of seconds for sampling. 228 | 229 | Examples: 230 | sample-bt-vfs -p 12345 -t 10 231 | sample-bt-vfs -p 12345 -t 5 -a '-DMAXACTION=100000' 232 | _EOC_ 233 | } 234 | -------------------------------------------------------------------------------- /tcp-accept-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | # Based on the systemtap script first shown by chaoslawful. 6 | 7 | use 5.006001; 8 | use strict; 9 | use warnings; 10 | 11 | use Getopt::Long qw( GetOptions ); 12 | 13 | GetOptions("a=s", \(my $stap_args), 14 | "d", \(my $dump_src), 15 | "distr", \(my $show_distr), 16 | "h", \(my $help), 17 | "limit=i", \(my $limit), 18 | "port=i", \(my $port), 19 | "time=i", \(my $time), 20 | "latency", \(my $check_latency)) 21 | or die usage(); 22 | 23 | if ($help) { 24 | print usage(); 25 | exit; 26 | } 27 | 28 | if (!defined $port) { 29 | die "No listening port specified by the -p option.\n"; 30 | } 31 | 32 | if (!defined $limit) { 33 | $limit = 10; 34 | } 35 | 36 | if (!defined $stap_args) { 37 | $stap_args = ''; 38 | } 39 | 40 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 41 | $stap_args .= " -DMAXACTION=100000"; 42 | } 43 | 44 | my $ver = `stap --version 2>&1`; 45 | if (!defined $ver) { 46 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 47 | } 48 | 49 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 50 | my $v = $1; 51 | if ($v < 2.1) { 52 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 53 | } 54 | 55 | } else { 56 | die "ERROR: unknown version of systemtap:\n$ver\n"; 57 | } 58 | 59 | my $stap_src; 60 | 61 | if ($show_distr) { 62 | # show queue length distribution 63 | 64 | my $summary_code = <<_EOC_; 65 | if (max_syn_qlen == 0) { 66 | println("\\nNo new connections found yet.") 67 | 68 | } else { 69 | println("\\n=== SYN Queue ===") 70 | printf("min/avg/max: %d/%d/%d\\n", 71 | \@min(syn_qlen_stats), \@avg(syn_qlen_stats), \@max(syn_qlen_stats)) 72 | println(\@hist_log(syn_qlen_stats)) 73 | 74 | println("=== Accept Queue ===") 75 | printf("min/avg/max: %d/%d/%d\\n", 76 | \@min(acc_qlen_stats), \@avg(acc_qlen_stats), \@max(acc_qlen_stats)) 77 | println(\@hist_log(acc_qlen_stats)) 78 | } 79 | _EOC_ 80 | 81 | my $postamble = <<_EOC_; 82 | probe end { 83 | $summary_code 84 | } 85 | _EOC_ 86 | 87 | my $tip; 88 | if (!defined $time) { 89 | $tip = "Hit Ctrl-C to end." 90 | 91 | } else { 92 | $tip = "Sampling for $time seconds."; 93 | $postamble .= <<_EOC_; 94 | probe timer.s($time) { 95 | exit() 96 | } 97 | _EOC_ 98 | } 99 | 100 | $stap_src = <<_EOC_; 101 | global syn_qlen_stats 102 | global acc_qlen_stats 103 | global max_syn_qlen 104 | global max_acc_qlen 105 | 106 | probe begin { 107 | warn("Tracing SYN & ACK backlog queue length distribution on the listening port $port...\\n$tip\\n") 108 | } 109 | 110 | probe kernel.function("tcp_v4_conn_request") { 111 | tcphdr = __get_skb_tcphdr(\$skb) 112 | dport = __tcp_skb_dport(tcphdr) 113 | 114 | if (dport == $port) { 115 | syn_qlen = \@cast(\$sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->qlen 116 | syn_qlen_stats <<< syn_qlen 117 | 118 | if (max_syn_qlen == 0) { 119 | max_qlen_log = \@cast(\$sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->max_qlen_log 120 | max_syn_qlen = (1 << max_qlen_log) 121 | printf("SYN queue length limit: %d\\n", max_syn_qlen) 122 | } 123 | 124 | acc_qlen_stats <<< \$sk->sk_ack_backlog 125 | 126 | if (max_acc_qlen == 0) { 127 | max_acc_qlen = \$sk->sk_max_ack_backlog 128 | printf("Accept queue length limit: %d\\n", max_acc_qlen) 129 | } 130 | } 131 | } 132 | 133 | $postamble 134 | _EOC_ 135 | 136 | } elsif ($check_latency) { 137 | 138 | my $summary_code = <<_EOC_; 139 | if (!found) { 140 | println("\\nNo new connections found yet.") 141 | 142 | } else { 143 | println("\\n=== Accept Queueing Latency Distribution (microsends) ===") 144 | printf("min/avg/max: %d/%d/%d\\n", 145 | \@min(latency_stats), \@avg(latency_stats), \@max(latency_stats)) 146 | println(\@hist_log(latency_stats)) 147 | } 148 | _EOC_ 149 | 150 | my $postamble = <<_EOC_; 151 | probe end { 152 | $summary_code 153 | } 154 | _EOC_ 155 | 156 | my $tip; 157 | if (!defined $time) { 158 | $tip = "Hit Ctrl-C to end." 159 | 160 | } else { 161 | $tip = "Sampling for $time seconds."; 162 | $postamble .= <<_EOC_; 163 | probe timer.s($time) { 164 | exit() 165 | } 166 | _EOC_ 167 | } 168 | 169 | $stap_src = <<_EOC_; 170 | global begin_times 171 | global latency_stats 172 | global found 173 | 174 | probe begin { 175 | warn("Tracing accept queueing latency on the listening port $port...\\n$tip\\n") 176 | } 177 | 178 | probe kernel.function("tcp_openreq_init") { 179 | tcphdr = __get_skb_tcphdr(\$skb) 180 | dport = __tcp_skb_dport(tcphdr) 181 | 182 | if (dport == $port) { 183 | begin_times[\$req] = gettimeofday_us() 184 | //printf("tcp openreq init: %p %d\\n", \$req, dport) 185 | } 186 | } 187 | 188 | probe kernel.function("inet_csk_accept") { 189 | req = \@cast(\$sk, "struct inet_connection_sock")->icsk_accept_queue->rskq_accept_head 190 | begin = begin_times[req] 191 | if (begin) { 192 | elapsed = gettimeofday_us() - begin 193 | /* 194 | printf("inet csk accept: sk=%p, req=%p, latency=%d\\n", \$sk, req, 195 | elapsed) 196 | */ 197 | latency_stats <<< elapsed 198 | delete begin_times[req] 199 | found = 1 200 | } 201 | } 202 | 203 | $postamble 204 | _EOC_ 205 | 206 | } else { 207 | # trace queue overflows 208 | 209 | $stap_src = <<_EOC_; 210 | global count 211 | 212 | probe begin { 213 | warn("Tracing SYN & ACK backlog queue overflows on the listening port $port...\\n") 214 | } 215 | 216 | probe kernel.function("tcp_v4_conn_request") { 217 | tcphdr = __get_skb_tcphdr(\$skb) 218 | dport = __tcp_skb_dport(tcphdr) 219 | 220 | if (dport == $port) { 221 | syn_qlen = \@cast(\$sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->qlen 222 | max_syn_qlen_log = \@cast(\$sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->max_qlen_log 223 | max_syn_qlen = (2 << max_syn_qlen_log) 224 | 225 | if (syn_qlen > max_syn_qlen) { 226 | now = tz_ctime(gettimeofday_s()) 227 | printf("[%s] SYN queue is overflown: %d > %d\\n", now, syn_qlen, max_syn_qlen) 228 | count++ 229 | } 230 | 231 | //printf("syn queue: %d <= %d\\n", qlen, max_qlen) 232 | 233 | ack_backlog = \$sk->sk_ack_backlog 234 | max_ack_backlog = \$sk->sk_max_ack_backlog 235 | 236 | if (ack_backlog > max_ack_backlog) { 237 | now = tz_ctime(gettimeofday_s()) 238 | printf("[%s] ACK backlog queue is overflown: %d > %d\\n", now, ack_backlog, max_ack_backlog) 239 | count++ 240 | } 241 | 242 | //printf("ACK backlog queue: %d <= %d\\n", ack_backlog, max_ack_backlog) 243 | 244 | if (count >= $limit) { 245 | exit() 246 | } 247 | } 248 | } 249 | _EOC_ 250 | } 251 | 252 | if ($dump_src) { 253 | print $stap_src; 254 | exit; 255 | } 256 | 257 | open my $in, "|stap --all-modules $stap_args -" 258 | or die "Cannot run stap: $!\n"; 259 | 260 | print $in $stap_src; 261 | 262 | close $in; 263 | 264 | sub usage { 265 | return <<'_EOC_'; 266 | Usage: 267 | tcp-accept-queue [optoins] 268 | 269 | Options: 270 | -a Pass extra arguments to the stap utility. 271 | -d Dump out the systemtap script source. 272 | --distr Show queue length distribution only. 273 | -h Print this usage. 274 | --limit= Exit when queue overflowing issues are found. 275 | (Default to 10) 276 | --port= Specify the listening port to be analyzed. 277 | --time= Time to wait before printing out the report and exiting 278 | (Only meaningful with --distr) 279 | 280 | Examples: 281 | tcp-accept-queue --port 11211 282 | tcp-accept-queue --port 6379 --limit 10 -a '-DMAXACTION=100000' 283 | _EOC_ 284 | } 285 | -------------------------------------------------------------------------------- /tcp-recv-queue: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Copyright (C) Yichun Zhang (agentzh) 4 | 5 | use 5.006001; 6 | use strict; 7 | use warnings; 8 | 9 | use Getopt::Long qw( GetOptions ); 10 | 11 | GetOptions("a=s", \(my $stap_args), 12 | "d", \(my $dump_src), 13 | "h", \(my $help), 14 | "dport=i", \(my $dport), 15 | "time=i", \(my $time)) 16 | or die usage(); 17 | 18 | if ($help) { 19 | print usage(); 20 | exit; 21 | } 22 | 23 | if (!defined $dport) { 24 | die "No destination port specified by the --dport option.\n"; 25 | } 26 | 27 | if (!defined $stap_args) { 28 | $stap_args = ''; 29 | } 30 | 31 | if ($stap_args !~ /(?:^|\s)-D\s*MAXACTION=/) { 32 | $stap_args .= " -DMAXACTION=100000"; 33 | } 34 | 35 | my $ver = `stap --version 2>&1`; 36 | if (!defined $ver) { 37 | die "Systemtap not installed or its \"stap\" utility is not visible to the PATH environment: $!\n"; 38 | } 39 | 40 | if ($ver =~ /version\s+(\d+\.\d+)/i) { 41 | my $v = $1; 42 | if ($v < 2.1) { 43 | die "ERROR: at least systemtap 2.1 is required but found $v\n"; 44 | } 45 | 46 | } else { 47 | die "ERROR: unknown version of systemtap:\n$ver\n"; 48 | } 49 | 50 | my $stap_src; 51 | 52 | my $summary_code = <<_EOC_; 53 | if (!found) { 54 | println("\\nNo queued received packets found yet.") 55 | 56 | } else { 57 | println("\\n=== Distribution of First-In-First-Out Latency (us) in TCP Receive Queue ===") 58 | printf("min/avg/max: %d/%d/%d\\n", 59 | \@min(latency_stats), \@avg(latency_stats), \@max(latency_stats)) 60 | println(\@hist_log(latency_stats)) 61 | } 62 | _EOC_ 63 | 64 | my $postamble = <<_EOC_; 65 | probe end { 66 | $summary_code 67 | } 68 | _EOC_ 69 | 70 | my $tip; 71 | if (!defined $time) { 72 | $tip = "Hit Ctrl-C to end." 73 | 74 | } else { 75 | $tip = "Sampling for $time seconds."; 76 | $postamble .= <<_EOC_; 77 | probe timer.s($time) { 78 | exit() 79 | } 80 | _EOC_ 81 | } 82 | 83 | $stap_src = <<_EOC_; 84 | global latency_stats 85 | global start_times 86 | global found 87 | 88 | probe begin { 89 | warn("Tracing the TCP receive queues for packets to the port $dport...\\n$tip\\n") 90 | } 91 | 92 | /* TODO: take into account the TCP out_of_order queue (i.e., tcp_ofo_queue). */ 93 | 94 | probe kernel.function("tcp_queue_rcv")!, 95 | kernel.function("tcp_data_queue") 96 | { 97 | tcphdr = __get_skb_tcphdr(\$skb) 98 | dport = __tcp_skb_dport(tcphdr) 99 | sport = __tcp_skb_sport(tcphdr) 100 | 101 | if (dport == $dport && start_times[\$sk, sport] == 0) { 102 | //printf("tcp_queue_rcv: queue=%p sk=%p sport=%d\\n", 103 | //&\$sk->sk_receive_queue, \$sk, sport) 104 | if (\$skb->len > 0) { 105 | //println("probe func: ", probefunc()) 106 | if (\@cast(\$skb->cb, "tcp_skb_cb")->seq != \@cast(\$skb->cb, "tcp_skb_cb")->end_seq) { 107 | start_times[\$sk, sport] = gettimeofday_us() 108 | 109 | } else { 110 | //println("found seq == end_seq") 111 | } 112 | } 113 | } 114 | } 115 | 116 | probe kernel.function("tcp_recvmsg"), kernel.function("tcp_recv_skb") { 117 | q = &\$sk->sk_receive_queue 118 | skb = \$sk->sk_receive_queue->next 119 | if (q != skb) { 120 | /* queue is not empty */ 121 | tcphdr = __get_skb_tcphdr(skb) 122 | dport = __tcp_skb_dport(tcphdr) 123 | 124 | if (dport == $dport) { 125 | sport = __tcp_skb_sport(tcphdr) 126 | begin = start_times[\$sk, sport] 127 | if (begin > 0) { 128 | //printf("tcp recvmsg: port=$dport sk=%p\\n", \$sk) 129 | latency_stats <<< (gettimeofday_us() - begin) 130 | found = 1 131 | delete start_times[\$sk, sport] 132 | } 133 | } 134 | } 135 | } 136 | 137 | probe kernel.function("tcp_close"), kernel.function("tcp_disconnect") { 138 | q = &\$sk->sk_receive_queue 139 | skb = \$sk->sk_receive_queue->next 140 | if (q != skb) { 141 | /* queue is not empty */ 142 | tcphdr = __get_skb_tcphdr(skb) 143 | dport = __tcp_skb_dport(tcphdr) 144 | 145 | if (dport == $dport) { 146 | sport = __tcp_skb_sport(tcphdr) 147 | delete start_times[\$sk, sport] 148 | } 149 | } 150 | } 151 | 152 | $postamble 153 | _EOC_ 154 | 155 | if ($dump_src) { 156 | print $stap_src; 157 | exit; 158 | } 159 | 160 | open my $in, "|stap --all-modules $stap_args -" 161 | or die "Cannot run stap: $!\n"; 162 | 163 | print $in $stap_src; 164 | 165 | close $in; 166 | 167 | sub usage { 168 | return <<'_EOC_'; 169 | Usage: 170 | tcp-recv-queue [optoins] 171 | 172 | Options: 173 | -a Pass extra arguments to the stap utility. 174 | -d Dump out the systemtap script source. 175 | -h Print this usage. 176 | --dport= Specify the destination port to be analyzed. 177 | --time= Time to wait before printing out the report and exiting 178 | (Only meaningful with --distr) 179 | 180 | Examples: 181 | tcp-recv-queue --dport=11211 182 | tcp-recv-queue --dport=6379 --time=10 -a '-DMAXACTION=100000' 183 | _EOC_ 184 | } 185 | --------------------------------------------------------------------------------