├── .gitignore ├── README.md ├── apache ├── apparmor ├── io ├── php ├── php-memcache ├── profile ├── stap ├── __init__.py ├── d.py ├── h │ ├── __init__.py │ └── php.py └── log.py └── tcp /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | systemtap cookbook 2 | ================== 3 | 4 | Various scripts using systemtap for analysis and diagnostics. This is 5 | inspired by [nginx-systemtap-toolkit][1] but scripts are written in 6 | Python instead of Perl. 7 | 8 | [1]: https://github.com/agentzh/nginx-systemtap-toolkit 9 | 10 | Prerequisites 11 | ------------- 12 | 13 | You need at least systemtap 2.3. Maybe it works with less recent 14 | version but I did not test them. You also need Python 2.7. For most 15 | scripts, you should ensure that the appropriate DWARF debuginfo have 16 | been installed. For Debian, this means to either install `-dbg` 17 | package or install from source without stripping debug symbols. For 18 | Ubuntu, you can use [`dbgsym` packages][2]. 19 | 20 | [2]: https://wiki.ubuntu.com/DebuggingProgramCrash#Debug_Symbol_Packages 21 | 22 | You need at least Linux kernel 3.5 with uprobes API for userspace 23 | tracing. A reasonable setup for Ubuntu Precise is the following one: 24 | 25 | $ apt-get install linux-image-3.11.0-13-generic{,-dbgsym} 26 | $ apt-get install linux-headers-3.11.0-13-generic 27 | $ apt-get install make gcc 28 | 29 | Previous kernels are missing important symbols. Ubuntu Precise has a 30 | buggy GCC which is not able to handle kernels using `-mfentry` 31 | flag. You can workaround this by setting `PR15123_ASSUME_MFENTRY` 32 | environment variable to 1 with systemtap 2.4. 33 | 34 | Permissions 35 | ----------- 36 | 37 | Running systemtap-based tools requires special user permissions. To 38 | prevent running these tools with the root account, you can add your 39 | own (non-root) account name to the `stapusr` and `staprun` user 40 | groups. But it is usually easier to just use `sudo`. 41 | 42 | Tools 43 | ----- 44 | 45 | Each script comes with appropriate help. Just run it with `--help` to 46 | get comprehensive help with examples. 47 | 48 | Some tools are emitting warnings: 49 | 50 | - `untested`: not really tested 51 | - `buggy`: can crash or produce no results 52 | 53 | License 54 | ------- 55 | 56 | > Permission to use, copy, modify, and/or distribute this software for any 57 | > purpose with or without fee is hereby granted, provided that the above 58 | > copyright notice and this permission notice appear in all copies. 59 | > 60 | > THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 61 | > WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 62 | > MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 63 | > ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 64 | > WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 65 | > ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 66 | > OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 67 | -------------------------------------------------------------------------------- /apache: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """PHP related tools 5 | ============================ 6 | 7 | This script will handle various instrumentation related to PHP interpreter. 8 | 9 | """ 10 | 11 | import sys 12 | import os 13 | import stap 14 | import jinja2 15 | 16 | 17 | @stap.d.enable 18 | @stap.d.arg("--apr", type=str, default="/usr/lib/libapr-1.so.0", 19 | help="path to libapr-1 library") 20 | @stap.d.arg("--interval", default=1000, type=int, metavar="MS", 21 | help="delay between screen updates in milliseconds") 22 | @stap.d.arg("--time", default=0, type=int, metavar="S", 23 | help="run at most S seconds") 24 | @stap.d.arg("--big", type=int, default=0, metavar="N", 25 | help="display the N biggest cookies") 26 | @stap.d.arg("--max", type=int, default=(1 << 14), metavar="M", 27 | help="maximum cookie size") 28 | def cookies(options): 29 | """Show distribution of cookie sizes.""" 30 | probe = jinja2.Template(ur""" 31 | global sizes; 32 | {%- if options.big %} 33 | global biggest%; 34 | {%- endif %} 35 | 36 | probe process("{{ options.apr }}").function("apr_table_addn") { 37 | if (user_string2($key, "") == "Cookie") { 38 | size = strlen(user_string2($val, "")); 39 | sizes <<< size; 40 | {%- if options.big %} 41 | biggest[size] = user_string_n2($val, 80, ""); 42 | {%- endif %} 43 | } 44 | } 45 | 46 | probe timer.ms({{ options.interval }}) { 47 | ansi_clear_screen(); 48 | print(@hist_log(sizes)); 49 | printf(" — min:%s avg:%s max:%s count:%d sum:%s\n\n", 50 | bytes_to_string(@min(sizes)), 51 | bytes_to_string(@avg(sizes)), 52 | bytes_to_string(@max(sizes)), 53 | @count(sizes), 54 | bytes_to_string(@sum(sizes))); 55 | {%- if options.big %} 56 | foreach (s- in biggest limit {{ options.big }}) { 57 | printf("%10s: %s%s\n", bytes_to_string(s), 58 | substr(biggest[s], 0, 60), (strlen(biggest[s])>60)?"…":""); 59 | } 60 | {%- endif %} 61 | } 62 | 63 | {%- if options.time %} 64 | probe timer.s({{ options.time }}) { 65 | printf("Exit after {{ options.time }} seconds\n"); 66 | exit(); 67 | } 68 | {%- endif %} 69 | """) 70 | probe = probe.render(options=options).encode("utf-8") 71 | stap.execute(probe, options, 72 | "-DMAXSTRINGLEN={}".format(options.max)) 73 | 74 | 75 | stap.run(sys.modules[__name__]) 76 | -------------------------------------------------------------------------------- /apparmor: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """AppArmor related tools 5 | ============================ 6 | 7 | This script will interact with AppArmor to extract some 8 | information. Notably, it will display more information about denied 9 | requests. 10 | 11 | """ 12 | 13 | import sys 14 | import stap 15 | import jinja2 16 | 17 | 18 | @stap.d.enable 19 | @stap.d.linux("3.2") 20 | @stap.d.warn("untested") 21 | def audit(options): 22 | """Audit backtraces 23 | 24 | For each audit event, display it with the associated 25 | backtrace. This should give some additional information on the 26 | denied operation. 27 | 28 | """ 29 | probe = jinja2.Template(ur""" 30 | // enum audit_type { 31 | // AUDIT_APPARMOR_AUDIT, 32 | // AUDIT_APPARMOR_ALLOWED, 33 | // AUDIT_APPARMOR_DENIED, 34 | // AUDIT_APPARMOR_HINT, 35 | // AUDIT_APPARMOR_STATUS, 36 | // AUDIT_APPARMOR_ERROR, 37 | // AUDIT_APPARMOR_KILL, 38 | // AUDIT_APPARMOR_AUTO 39 | // }; 40 | global aa_type[12] 41 | probe begin { 42 | aa_type[0] = "AUDIT/AUTO" 43 | aa_type[1] = "ALLOWED" 44 | aa_type[2] = "DENIED" 45 | aa_type[3] = "HINT" 46 | aa_type[4] = "STATUS" 47 | aa_type[5] = "ERROR" 48 | aa_type[6] = "KILL" 49 | aa_type[7] = "AUTO" 50 | } 51 | 52 | function aa_type2str:string(type:long) { 53 | return (type in aa_type ? aa_type[type] : "UNDEF") 54 | } 55 | 56 | probe kernel.function("aa_audit_msg") { 57 | print("type=%s error=%d op=%d name=%s info=%s\n", 58 | aa_type2str($type), 59 | $sa->apparmor_audit_data->error, $sa->apparmor_audit_data->op, 60 | kernel_string($sa->apparmor_audit_data->name), 61 | kernel_string($sa->apparmor_audit_data->info)) 62 | print("exec=%s pid=%s pp=%s\n", execname(), pid(), pp()) 63 | print_backtrace() 64 | } 65 | 66 | """) 67 | probe = probe.render(options=options).encode("utf-8") 68 | stap.execute(probe, options, "--all-modules") 69 | 70 | 71 | stap.run(sys.modules[__name__]) 72 | -------------------------------------------------------------------------------- /io: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """IO tools 5 | ============================ 6 | 7 | This script groups IO related tools 8 | 9 | """ 10 | 11 | import sys 12 | import os 13 | import stap 14 | import jinja2 15 | 16 | 17 | @stap.d.enable 18 | @stap.d.linux("3.11") 19 | @stap.d.arg("--limit", "-l", default=20, type=int, metavar="N", 20 | help="display N top processes") 21 | @stap.d.arg("--by-command", action="store_true", 22 | help="aggregate using command line instead of PID") 23 | @stap.d.arg("--sort", "-s", 24 | choices=["reads", "writes", 25 | "rbytes", "wbytes", 26 | "tio", "tbytes"], 27 | default="tbytes", 28 | help="sort results using the specified metric") 29 | def top(options): 30 | """iotop-like tool. 31 | 32 | Display top users of IO disks. Those are IO as seen from the VFS 33 | point of view. 34 | 35 | """ 36 | probe = jinja2.Template(ur""" 37 | global ioreads, iowrites, breads, bwrites, all; 38 | 39 | probe vfs.read.return { 40 | breads[{{pid}},cmdline_str()] += bytes_read; 41 | ioreads[{{pid}},cmdline_str()] += 1; 42 | } 43 | 44 | probe vfs.write.return { 45 | bwrites[{{pid}},cmdline_str()] += bytes_written; 46 | iowrites[{{pid}},cmdline_str()] += 1; 47 | } 48 | 49 | function human_bytes:string(bytes:long) { 50 | return sprintf("%sB/s", bytes_to_string(bytes)); 51 | } 52 | function human_iops:string(iops:long) { 53 | prefix = " "; 54 | if (iops > 10000000000) { 55 | prefix = "G"; 56 | iops /= 10000000000; 57 | } else if (iops > 1000000) { 58 | iops /= 10000000; 59 | prefix = "M"; 60 | } else if (iops > 10000) { 61 | iops /= 1000; 62 | prefix = "K"; 63 | } 64 | return sprintf("%d%s/s", iops, prefix); 65 | } 66 | function average:string(bytes:long, iops:long) { 67 | if (iops == 0) return "-"; 68 | return bytes_to_string(bytes/iops); 69 | } 70 | 71 | probe timer.s(1) { 72 | foreach ([t,s] in breads) { 73 | tbreads += breads[t,s]; 74 | {%- if options.sort in ["rbytes", "tbytes"] %} 75 | all[t,s] += breads[t,s]; 76 | {%- endif %} 77 | } 78 | foreach ([t,s] in bwrites) { 79 | tbwrites += bwrites[t,s]; 80 | {%- if options.sort in ["wbytes", "tbytes"] %} 81 | all[t,s] += bwrites[t,s]; 82 | {%- endif %} 83 | } 84 | foreach ([t,s] in ioreads) { 85 | tioreads += ioreads[t,s]; 86 | {%- if options.sort in ["reads", "tio"] %} 87 | all[t,s] += ioreads[t,s]; 88 | {%- endif %} 89 | } 90 | foreach ([t,s] in iowrites) { 91 | tiowrites += iowrites[t,s]; 92 | {%- if options.sort in ["writes", "tio"] %} 93 | all[t,s] += iowrites[t,s]; 94 | {%- endif %} 95 | } 96 | ansi_clear_screen(); 97 | printf("Total read: %10s / %10s (avg req size: %10s) \n", 98 | bytes_to_string(tbreads), human_iops(tioreads), 99 | average(tbreads, tioreads)); 100 | printf("Total write: %10s / %10s (avg req size: %10s) \n", 101 | bytes_to_string(tbwrites), human_iops(tiowrites), 102 | average(tbwrites, tiowrites)); 103 | ansi_set_color2(30, 46); 104 | printf("{% if not options.by_command %}%5s {% endif %}%10s %10s %8s %10s %10s %8s %-30s\n", 105 | {%- if not options.by_command %} 106 | "PID", 107 | {%- endif %} 108 | "RBYTES/s", "READ/s", "rAVG", 109 | "WBYTES/s", "WRITE/s", "wAVG", "COMMAND"); 110 | ansi_reset_color(); 111 | foreach ([t,s] in all- limit {{ options.limit }}) { 112 | cmd = substr(s, 0, 30); 113 | printf("{% if not options.by_command %}%5d {% endif %}%10s %10s %8s %10s %10s %8s %s\n", 114 | {%- if not options.by_command %} 115 | t, 116 | {%- endif %} 117 | human_bytes(breads[t,s]), 118 | human_iops(ioreads[t,s]), 119 | average(breads[t,s], ioreads[t,s]), 120 | human_bytes(bwrites[t,s]), 121 | human_iops(iowrites[t,s]), 122 | average(bwrites[t,s], iowrites[t,s]), 123 | cmd); 124 | } 125 | delete all; 126 | delete ioreads; 127 | delete iowrites; 128 | delete breads; 129 | delete bwrites; 130 | } 131 | """) 132 | pid = "pid()" 133 | if options.by_command: 134 | pid = "0" 135 | probe = probe.render(options=options, pid=pid).encode("utf-8") 136 | stap.execute(probe, options) 137 | 138 | 139 | @stap.d.enable 140 | @stap.d.linux("3.11") 141 | @stap.d.arg_pid 142 | @stap.d.arg_process 143 | @stap.d.arg("--limit", "-l", default=20, type=int, metavar="N", 144 | help="display N top processes") 145 | @stap.d.arg("--sort", "-s", 146 | choices=["reads", "writes", "total"], 147 | default="total", 148 | help="sort results using the specified metric") 149 | def files(options): 150 | """Display most read/written files. 151 | 152 | The filename is only available when opening the file. If the 153 | displayed filename is an inode number, you need to search it 154 | yourself on the file system. 155 | """ 156 | probe = jinja2.Template(ur""" 157 | global files%, all, ioreads, iowrites, breads, bwrites; 158 | 159 | probe generic.fop.open { 160 | if (!{{ options.condition }}) next; 161 | files[dev,ino] = filename; 162 | } 163 | 164 | function fname:string(dev:long, ino:long) { 165 | try { 166 | if (files[dev,ino] != "") return files[dev,ino]; 167 | return sprintf("dev:%d ino:%d", dev, ino); 168 | } catch { 169 | return "?????"; 170 | } 171 | } 172 | 173 | probe vfs.read.return { 174 | if (!{{ options.condition }}) next; 175 | breads[fname(dev, ino)] += bytes_read; 176 | ioreads[fname(dev, ino)] += 1; 177 | } 178 | 179 | probe vfs.write.return { 180 | if (!{{ options.condition }}) next; 181 | bwrites[fname(dev, ino)] += bytes_written; 182 | iowrites[fname(dev, ino)] += 1; 183 | } 184 | 185 | function human_bytes:string(bytes:long) { 186 | return sprintf("%sB/s", bytes_to_string(bytes)); 187 | } 188 | function human_iops:string(iops:long) { 189 | prefix = " "; 190 | if (iops > 10000000000) { 191 | prefix = "G"; 192 | iops /= 10000000000; 193 | } else if (iops > 1000000) { 194 | iops /= 10000000; 195 | prefix = "M"; 196 | } else if (iops > 10000) { 197 | iops /= 1000; 198 | prefix = "K"; 199 | } 200 | return sprintf("%d%s/s", iops, prefix); 201 | } 202 | 203 | probe timer.s(1) { 204 | {%- if options.sort in ["reads", "total"] %} 205 | foreach (f in breads) all[f] += breads[f]; 206 | {%- endif %} 207 | {%- if options.sort in ["writes", "total"] %} 208 | foreach (f in bwrites) all[f] += bwrites[f]; 209 | {%- endif %} 210 | ansi_clear_screen(); 211 | ansi_set_color2(30, 46); 212 | printf("%10s %10s %10s %10s %-40s\n", 213 | "RBYTES/s", "READ/s", 214 | "WBYTES/s", "WRITE/s", "FILE"); 215 | ansi_reset_color(); 216 | foreach (f in all- limit {{ options.limit }}) { 217 | file = f; 218 | if (strlen(file) > 40) { 219 | file = sprintf("%s…%s", substr(f, 0, 10), substr(f, strlen(f) - 29, strlen(f))); 220 | } 221 | printf("%10s %10s %10s %10s %s\n", 222 | human_bytes(breads[f]), 223 | human_iops(ioreads[f]), 224 | human_bytes(bwrites[f]), 225 | human_iops(iowrites[f]), 226 | file); 227 | } 228 | delete all; 229 | delete ioreads; 230 | delete iowrites; 231 | delete breads; 232 | delete bwrites; 233 | } 234 | """) 235 | probe = probe.render(options=options).encode("utf-8") 236 | stap.execute(probe, options) 237 | 238 | 239 | @stap.d.enable 240 | @stap.d.linux("4.1") 241 | @stap.d.arg("-T", "--timestamp", action="store_true", 242 | help="include timestamp on output") 243 | @stap.d.arg("--queue", "-Q", action="store_true", 244 | help="include OS queued time in IO time") 245 | @stap.d.arg("--milliseconds", "-m", action="store_true", 246 | help="millisecond histogram") 247 | @stap.d.arg("interval", nargs="?", default=99999999, 248 | type=int, 249 | help="output interval, in seconds") 250 | @stap.d.arg("count", nargs="?", default=99999999, 251 | type=int, 252 | help="number of outputs") 253 | def latency(options): 254 | """Summarize block device I/O latency as a histogram. 255 | 256 | This is a port of Brendan Gregg's biolatency tool available here: 257 | https://github.com/iovisor/bcc/blob/master/tools/biolatency 258 | """ 259 | probe = jinja2.Template(ur""" 260 | global count; 261 | global latency_stats; 262 | global start%; 263 | 264 | {%- if options.queue %} 265 | probe kernel.function("blk_account_io_start") { 266 | start[@choose_defined($rq,$req)] = gettimeofday_us(); 267 | } 268 | {%- else %} 269 | probe kernel.function("blk_start_request") { 270 | start[@choose_defined($rq,$req)] = gettimeofday_us(); 271 | } 272 | probe kernel.function("blk_mq_start_request") { 273 | start[@choose_defined($rq,$req)] = gettimeofday_us(); 274 | } 275 | {% endif %} 276 | 277 | probe kernel.function("blk_account_io_completion") { 278 | s = start[@choose_defined($rq,$req)]; 279 | if (s == 0) { 280 | next; 281 | } 282 | delta = gettimeofday_us() - s; 283 | {%- if options.milliseconds %} 284 | delta /= 1000; 285 | {%- endif %} 286 | delete start[@choose_defined($rq,$req)]; 287 | 288 | latency_stats <<< delta; 289 | } 290 | 291 | function display_histogram() { 292 | {%- if options.timestamp %} 293 | t = gettimeofday_s(); 294 | println(ctime(t)); 295 | {%- endif %} 296 | if (@count(latency_stats)) { 297 | print(@hist_log(latency_stats)); 298 | } 299 | delete latency_stats; 300 | delete start; 301 | } 302 | 303 | probe timer.s({{ options.interval }}) { 304 | display_histogram(); 305 | if (++count >= {{ options.count }}) { 306 | exit(); 307 | } 308 | } 309 | 310 | probe end { 311 | display_histogram(); 312 | } 313 | """) 314 | probe = probe.render(options=options).encode("utf-8") 315 | stap.execute(probe, options) 316 | 317 | 318 | stap.run(sys.modules[__name__]) 319 | -------------------------------------------------------------------------------- /php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """PHP related tools 5 | ============================ 6 | 7 | This script will handle various instrumentation related to PHP interpreter. 8 | 9 | """ 10 | 11 | import sys 12 | import os 13 | import stap 14 | import jinja2 15 | 16 | 17 | @stap.d.enable 18 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 19 | help="path to PHP process or module") 20 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 21 | help="restrict the profiling to URI prefixed by PREFIX") 22 | @stap.d.arg("--interval", default=1000, type=int, 23 | help="delay between screen updates in milliseconds") 24 | @stap.d.arg("--log", action="store_true", 25 | help="display a logarithmic histogram") 26 | @stap.d.arg("--step", type=int, default=10, metavar="MS", 27 | help="each bucket represents MS milliseconds") 28 | @stap.d.arg("--slow", action="store_true", 29 | help="log slowest requests") 30 | @stap.d.arg("--function", default="", type=str, metavar="FN", 31 | help="profile FN instead of the whole request") 32 | def time(options): 33 | """Distributions of response time for PHP requests. 34 | """ 35 | probe = jinja2.Template(ur""" 36 | global start%, intervals; 37 | {%- if options.slow %} 38 | global slow%; 39 | {%- endif %} 40 | {%- if options.function %} 41 | global request%; 42 | {%- endif %} 43 | 44 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 45 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") { 46 | {%- if options.function %} 47 | request[pid()] = sprintf("%5s %s", user_string($arg3), user_string($arg2)); 48 | {%- else %} 49 | start[pid()] = gettimeofday_ms(); 50 | {%- endif %} 51 | } 52 | } 53 | 54 | {% if options.function %} 55 | function fn_name:string(name:long, class:long) { 56 | try { 57 | fn = sprintf("%s::%s", 58 | user_string(class), 59 | user_string(name)); 60 | } catch { 61 | fn = ""; 62 | } 63 | return fn; 64 | } 65 | 66 | probe process("{{ options.php }}").provider("php").mark("function__entry") { 67 | if (request[pid()] != "") { 68 | fn = fn_name($arg1, $arg4); 69 | if (fn == "{{ options.function }}") 70 | start[pid()] = gettimeofday_ms(); 71 | } 72 | } 73 | 74 | probe process("{{ options.php }}").provider("php").mark("function__return") { 75 | if (start[pid()] && request[pid()] != "") { 76 | fn = fn_name($arg1, $arg4); 77 | if (fn == "{{ options.function }}") 78 | record(request[pid()]); 79 | } 80 | } 81 | {% endif %} 82 | 83 | function record(uri:string) { 84 | t = gettimeofday_ms(); 85 | old_t = start[pid()]; 86 | if (old_t > 1 && t > 1) { 87 | intervals <<< t - old_t; 88 | {%- if options.slow %} 89 | // We may miss some values... 90 | slow[t - old_t] = uri 91 | {%- endif %} 92 | } 93 | delete start[pid()]; 94 | } 95 | 96 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 97 | {%- if options.function %} 98 | delete request[pid()]; 99 | {%- else %} 100 | record(sprintf("%5s %s", user_string($arg3), user_string($arg2))); 101 | {%- endif %} 102 | } 103 | 104 | probe timer.ms({{ options.interval }}) { 105 | ansi_clear_screen(); 106 | if (@count(intervals) == 0) { 107 | printf("No data yet...\n"); 108 | next; 109 | } 110 | {%- if options.log %} 111 | print(@hist_log(intervals)); 112 | {%- else %} 113 | print(@hist_linear(intervals, 0, {{ options.step * 20 }}, {{ options.step }})); 114 | {%- endif %} 115 | printf(" — URI prefix: %s\n", "{{ options.uri }}"); 116 | {%- if options.function %} 117 | printf(" — Function: %s\n", "{{ options.function }}"); 118 | {%- endif %} 119 | printf(" — min:%dms avg:%dms max:%dms count:%d\n", 120 | @min(intervals), @avg(intervals), 121 | @max(intervals), @count(intervals)); 122 | {% if options.slow %} 123 | printf(" — slowest requests:\n"); 124 | foreach (t- in slow limit 10) { 125 | printf(" %6dms: %s\n", t, slow[t]); 126 | } 127 | delete slow; 128 | {% endif %} 129 | } 130 | """) 131 | probe = probe.render(options=options).encode("utf-8") 132 | stap.execute(probe, options) 133 | 134 | 135 | @stap.d.enable 136 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 137 | help="path to PHP process or module") 138 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 139 | help="restrict requests to URI prefixed by PREFIX") 140 | @stap.d.arg("--interval", default=500, type=int, metavar="MS", 141 | help="delay between screen updates in milliseconds") 142 | @stap.d.arg("--limit", default=20, type=int, metavar="N", 143 | help="display the top N active requests") 144 | def activereqs(options): 145 | """Display active PHP requests. 146 | 147 | Active PHP requests are displayed in order of execution length. A 148 | summary is also provided to show the number of active requests 149 | over the number of total requests. 150 | 151 | """ 152 | probe = jinja2.Template(ur""" 153 | global request%, since%, requests, actives; 154 | 155 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 156 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") { 157 | request[pid()] = sprintf("%5s %s", user_string($arg3), user_string($arg2)); 158 | since[pid()] = gettimeofday_ms(); 159 | requests++; actives++; 160 | } 161 | } 162 | 163 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 164 | if (since[pid()]) { 165 | delete request[pid()]; 166 | delete since[pid()]; 167 | actives--; 168 | } 169 | } 170 | 171 | probe timer.ms({{ options.interval }}) { 172 | t = gettimeofday_ms(); 173 | ansi_clear_screen(); 174 | foreach (p in since+ limit {{ options.limit }}) { 175 | r = request[p]; 176 | if (strlen(r) > 60) 177 | r = sprintf("%s...%s", substr(r, 0, 45), substr(r, strlen(r) - 15, strlen(r))); 178 | printf(" %6dms: %s\n", t - since[p], r); 179 | } 180 | 181 | for (i = actives; i < {{ options.limit }}; i++) printf("\n"); 182 | printf("\n"); 183 | ansi_set_color2(30, 46); 184 | printf(" ♦ Active requests: %-6d \n", actives); 185 | printf(" ♦ Total requests: %-6d \n", requests); 186 | ansi_reset_color(); 187 | 188 | requests = 0; 189 | } 190 | """) 191 | probe = probe.render(options=options).encode("utf-8") 192 | stap.execute(probe, options) 193 | 194 | 195 | @stap.d.enable 196 | @stap.d.warn("buggy") 197 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 198 | help="path to PHP process or module") 199 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 200 | help="restrict the profiling to URI prefixed by PREFIX") 201 | @stap.d.arg("--interval", default=1000, type=int, 202 | help="delay between screen updates in milliseconds") 203 | @stap.d.arg("--log", action="store_true", 204 | help="display a logarithmic histogram") 205 | @stap.d.arg("--step", type=int, default=500000, metavar="BYTES", 206 | help="each bucket represents BYTES bytes") 207 | @stap.d.arg("--big", action="store_true", 208 | help="log bigger memory users") 209 | @stap.d.arg("--absolute", action="store_true", 210 | help="log absolute memory usage") 211 | @stap.d.arg("--memtype", choices=["data", "rss", "shr", "txt" "total"], 212 | default="total", 213 | help="memory type to watch for") 214 | def memory(options): 215 | """Display memory usage of PHP requests. 216 | 217 | This is not reliable as it seems that memory is allocated 218 | early. The usage is therefore lower than it should be. You can use 219 | :option:`--absolute` to get absolute memory usage instead. This 220 | time, this overestimate the memory use. 221 | 222 | """ 223 | probe = jinja2.Template(ur""" 224 | global mem, pagesize, track; 225 | {%- if not options.absolute %} 226 | global memusage; 227 | {%- endif %} 228 | {%- if options.big %} 229 | global big%; 230 | {%- endif %} 231 | 232 | probe begin { 233 | pagesize = {{ memfunc }}(); 234 | } 235 | 236 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 237 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") { 238 | {%- if not options.absolute %} 239 | memusage[pid()] = {{ memfunc }}() * pagesize; 240 | {%- endif %} 241 | track[pid()] = 1; 242 | } 243 | } 244 | 245 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 246 | if (!track[pid()]) next; 247 | m = proc_mem_size() * pagesize; 248 | {%- if not options.absolute %} 249 | old_m = memusage[pid()]; 250 | delete memusage[pid()]; 251 | if (old_m && m && m - old_m > 0) { 252 | {%- else %} 253 | old_m = 0; 254 | if (m) { 255 | {%- endif %} 256 | mem <<< m - old_m; 257 | {%- if options.big %} 258 | request = sprintf("%5s %s", user_string($arg3), user_string($arg2)); 259 | big[m - old_m] = request; 260 | {%- endif %} 261 | } 262 | delete track[pid()]; 263 | } 264 | 265 | probe timer.ms({{ options.interval }}) { 266 | ansi_clear_screen(); 267 | {%- if options.log %} 268 | print(@hist_log(mem)); 269 | {%- else %} 270 | print(@hist_linear(mem, 0, {{ options.step * 20 }}, {{ options.step }})); 271 | {%- endif %} 272 | printf(" — URI prefix: %s\n", "{{ options.uri }}"); 273 | printf(" — min:%s avg:%s max:%s count:%d\n", 274 | bytes_to_string(@min(mem)), 275 | bytes_to_string(@avg(mem)), 276 | bytes_to_string(@max(mem)), 277 | @count(mem)); 278 | {% if options.big %} 279 | printf(" — biggest users:\n"); 280 | foreach (t- in big limit 10) { 281 | printf(" %10s: %s\n", bytes_to_string(t), big[t]); 282 | } 283 | delete big; 284 | {% endif %} 285 | } 286 | """) 287 | memfunc = "proc_mem_{}".format( 288 | dict(total="size").get(options.memtype, options.memtype)) 289 | probe = probe.render(options=options, 290 | memfunc=memfunc).encode("utf-8") 291 | stap.execute(probe, options) 292 | 293 | 294 | @stap.d.enable 295 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 296 | help="path to PHP process or module") 297 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 298 | help="restrict the profiling to URI prefixed by PREFIX") 299 | @stap.d.arg("--filter-uri", type=str, action="append", dest="filtered", 300 | help="don't log the given URI", metavar="URI") 301 | @stap.d.arg("--interval", default=1000, type=int, 302 | help="delay between screen updates in milliseconds") 303 | @stap.d.arg("--log", action="store_true", 304 | help="display a logarithmic histogram") 305 | @stap.d.arg("--step", type=int, default=2000000, metavar="BYTES", 306 | help="each bucket represents BYTES bytes") 307 | @stap.d.arg("--big", action="store_true", 308 | help="log bigger memory users") 309 | def peak(options): 310 | """Display memory peak usage for each requests. 311 | 312 | This examines PHP heap. See: https://wiki.php.net/internals/zend_mm. 313 | 314 | """ 315 | probe = jinja2.Template(ur""" 316 | global mem; 317 | {%- if options.big %} 318 | global big%; 319 | {%- endif %} 320 | 321 | {#- We don't use request__shutdown probe: it is too late #} 322 | probe process("{{ options.php }}").function("php_request_shutdown") { 323 | uri = user_string2(@var("sapi_globals", "{{ options.php }}")->request_info->request_uri, "(unknown)"); 324 | if (substr(uri, 0, {{ options.uri|length() }}) != "{{ options.uri }}") next; 325 | {% for uri in options.filtered %} 326 | if (uri == "{{ uri }}") next; 327 | {% endfor %} 328 | peak = @var("alloc_globals", "{{ php }}")->mm_heap->real_peak; 329 | mem <<< peak; 330 | {%- if options.big %} 331 | big[peak] = uri; 332 | {%- endif %} 333 | } 334 | 335 | probe timer.ms({{ options.interval }}) { 336 | ansi_clear_screen(); 337 | {%- if options.log %} 338 | print(@hist_log(mem)); 339 | {%- else %} 340 | print(@hist_linear(mem, 0, {{ options.step * 20 }}, {{ options.step }})); 341 | {%- endif %} 342 | printf(" — URI prefix: %s\n", "{{ options.uri }}"); 343 | printf(" — min:%s avg:%s max:%s count:%d\n", 344 | bytes_to_string(@min(mem)), 345 | bytes_to_string(@avg(mem)), 346 | bytes_to_string(@max(mem)), 347 | @count(mem)); 348 | {% if options.big %} 349 | printf(" — biggest users:\n"); 350 | foreach (t- in big limit 10) { 351 | printf(" %10s: %s\n", bytes_to_string(t), big[t]); 352 | } 353 | {% endif %} 354 | } 355 | """) 356 | probe = probe.render(options=options).encode("utf-8") 357 | stap.execute(probe, options) 358 | 359 | @stap.d.enable 360 | @stap.d.warn("buggy") 361 | @stap.d.warn("untested") 362 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 363 | help="path to PHP process or module") 364 | @stap.d.arg("--uri", type=str, default="", metavar="PREFIX", 365 | help="restrict the profiling to URI prefixed by PREFIX") 366 | @stap.d.arg("--time", type=int, default=10, metavar="S", 367 | help="how much time we should run") 368 | @stap.d.arg("--depth", type=int, default=1, metavar="D", 369 | help="PHP backtrace depth to consider") 370 | def malloc(options): 371 | """Display most frequent memory allocation backtraces. 372 | 373 | """ 374 | probe = jinja2.Template(ur""" 375 | {{ backtrace.init() }} 376 | 377 | global allocs%; 378 | 379 | {%- if options.uri %} 380 | global enabled; 381 | 382 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 383 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") 384 | enabled[pid()] = 1; 385 | } 386 | 387 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 388 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") 389 | delete enabled[pid()]; 390 | } 391 | {%- endif %} 392 | 393 | probe process("{{ options.php }}").function("_zend_mm_realloc_int").return, 394 | process("{{ options.php }}").function("_zend_mm_alloc_int").return { 395 | {%- if options.uri %} 396 | if (!enabled[pid()]) next; 397 | {%- endif %} 398 | if ($return != 0) 399 | allocs[phpstack_n({{ options.depth }})] <<< $size; 400 | } 401 | 402 | probe timer.s({{ options.time }}) { 403 | foreach (stack in allocs- limit 10) { 404 | ansi_set_color2(30, 46); 405 | printf(" Allocated %s (%d times): \n", 406 | bytes_to_string(@sum(allocs[stack])), 407 | @count(allocs[stack])); 408 | ansi_reset_color(); 409 | printf("%s\n\n", stack); 410 | } 411 | exit(); 412 | } 413 | 414 | """) 415 | probe = probe.render(options=options, 416 | backtrace=stap.h.php.Backtrace(options.php)).encode("utf-8") 417 | stap.execute(probe, options) 418 | 419 | 420 | @stap.d.enable 421 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 422 | help="path to PHP process or module") 423 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 424 | help="restrict the profiling to URI prefixed by PREFIX") 425 | @stap.d.arg("--interval", default=1000, type=int, 426 | help="delay between screen updates in milliseconds") 427 | @stap.d.arg("--busy", action="store_true", 428 | help="log busiest requests") 429 | @stap.d.arg("--function", default="", type=str, metavar="FN", 430 | help="profile FN instead of the whole request") 431 | def cpu(options): 432 | """Display CPU usage 433 | 434 | Return distributions of CPU usage for PHP requests. 435 | """ 436 | probe = jinja2.Template(ur""" 437 | global start%, use%, usage; 438 | {%- if options.busy %} 439 | global busy%; 440 | {%- endif %} 441 | {%- if options.function %} 442 | global request%; 443 | {%- endif %} 444 | 445 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 446 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") { 447 | {%- if options.function %} 448 | request[pid()] = sprintf("%5s %s", user_string($arg3), user_string($arg2)); 449 | {%- else %} 450 | start[pid()] = gettimeofday_ms(); 451 | use[pid()] = task_stime() + task_utime(); 452 | {%- endif %} 453 | } 454 | } 455 | 456 | {% if options.function %} 457 | function fn_name:string(name:long, class:long) { 458 | try { 459 | fn = sprintf("%s::%s", 460 | user_string(class), 461 | user_string(name)); 462 | } catch { 463 | fn = ""; 464 | } 465 | return fn; 466 | } 467 | 468 | probe process("{{ options.php }}").provider("php").mark("function__entry") { 469 | if (request[pid()] != "") { 470 | fn = fn_name($arg1, $arg4); 471 | if (fn == "{{ options.function }}") { 472 | start[pid()] = gettimeofday_ms(); 473 | use[pid()] = task_stime() + task_utime(); 474 | } 475 | } 476 | } 477 | 478 | probe process("{{ options.php }}").provider("php").mark("function__return") { 479 | if (start[pid()] && request[pid()] != "") { 480 | fn = fn_name($arg1, $arg4); 481 | if (fn == "{{ options.function }}") 482 | record(request[pid()]); 483 | } 484 | } 485 | {% endif %} 486 | 487 | function record(uri:string) { 488 | u = task_stime() + task_utime(); 489 | t = gettimeofday_ms(); 490 | old_u = use[pid()]; 491 | old_t = start[pid()]; 492 | if (old_t && t && old_u && u && t != old_t) { 493 | percent = cputime_to_msecs(u - old_u) * 100 / (t - old_t); 494 | if (percent > 100) percent = 100; 495 | usage <<< percent; 496 | {%- if options.busy %} 497 | // We may miss some values... 498 | busy[percent] = uri; 499 | {%- endif %} 500 | } 501 | delete start[pid()]; 502 | delete use[pid()]; 503 | } 504 | 505 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 506 | {%- if options.function %} 507 | delete request[pid()]; 508 | {%- else %} 509 | record(sprintf("%5s %s", user_string($arg3), user_string($arg2))); 510 | {%- endif %} 511 | } 512 | 513 | probe timer.ms({{ options.interval }}) { 514 | ansi_clear_screen(); 515 | print(@hist_linear(usage, 0, 100, 10)); 516 | printf(" — URI prefix: %s\n", "{{ options.uri }}"); 517 | {%- if options.function %} 518 | printf(" — Function: %s\n", "{{ options.function }}"); 519 | {%- endif %} 520 | printf(" — min:%d%% avg:%d%% max:%d%% count:%d\n", 521 | @min(usage), @avg(usage), 522 | @max(usage), @count(usage)); 523 | {% if options.busy %} 524 | printf(" — busyest requests:\n"); 525 | foreach (t- in busy limit 10) { 526 | printf(" %6d%%: %s\n", t, busy[t]); 527 | } 528 | delete busy; 529 | {% endif %} 530 | } 531 | """) 532 | probe = probe.render(options=options).encode("utf-8") 533 | stap.execute(probe, options) 534 | 535 | 536 | @stap.d.enable 537 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 538 | help="path to PHP process or module") 539 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 540 | help="restrict the profiling to URI prefixed by PREFIX") 541 | @stap.d.arg("--interval", default=5000, type=int, 542 | help="delay between screen updates in milliseconds") 543 | @stap.d.arg("--step", type=int, default=1, metavar="N", 544 | help="each bucket represents N calls") 545 | @stap.d.arg("--buckets", type=int, default=20, metavar="N", 546 | help="how many buckets to display") 547 | @stap.d.arg("--disable-hist", dest="hist", 548 | action="store_false", 549 | help="disable display of distribution histogram") 550 | @stap.d.arg("functions", nargs="+", 551 | type=str, metavar="FN", 552 | help="functions to count") 553 | def count(options): 554 | """Distributions of PHP function calls per requests. 555 | 556 | """ 557 | probe = jinja2.Template(ur""" 558 | global fns; 559 | global count%; 560 | global countfn; 561 | global acountfn; 562 | 563 | probe begin { 564 | {%- for item in options.functions %} 565 | fns["{{ item }}"] = 1; 566 | {%- endfor %} 567 | } 568 | 569 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 570 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") { 571 | count[pid(), ""] = 1; 572 | foreach (fn in fns) count[pid(), fn] = 0; 573 | } 574 | } 575 | 576 | function fn_name:string(name:long, class:long) { 577 | try { 578 | fn = sprintf("%s::%s", 579 | user_string(class), 580 | user_string(name)); 581 | } catch { 582 | fn = ""; 583 | } 584 | return fn; 585 | } 586 | 587 | probe process("{{ options.php }}").provider("php").mark("function__entry") { 588 | if (count[pid(), ""] != 1) next; 589 | fn = fn_name($arg1, $arg4); 590 | if ([fn] in fns) { 591 | count[pid(), fn]++ 592 | } 593 | } 594 | 595 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 596 | if (count[pid(), ""] != 1) next; 597 | foreach ([p, fn] in count) { 598 | if (p != pid()) continue; 599 | if (fn == "") continue; 600 | countfn[fn] += count[p, fn]; 601 | } 602 | foreach (fn in countfn) { 603 | acountfn[fn] <<< countfn[fn]; 604 | } 605 | delete countfn; 606 | delete count[pid(), ""]; 607 | } 608 | 609 | probe timer.ms({{ options.interval }}) { 610 | ansi_clear_screen(); 611 | foreach (fn in acountfn) { 612 | ansi_set_color2(30, 46); 613 | printf(" — Function %-30s: \n", fn); 614 | ansi_reset_color(); 615 | {%- if options.hist %} 616 | print(@hist_linear(acountfn[fn], 0, {{ options.step * options.buckets }}, {{ options.step }})); 617 | {%- endif %} 618 | printf(" min:%d avg:%d max:%d count:%d\n\n", 619 | @min(acountfn[fn]), @avg(acountfn[fn]), 620 | @max(acountfn[fn]), @count(acountfn[fn])); 621 | } 622 | delete acountfn; 623 | } 624 | """) 625 | probe = probe.render(options=options).encode("utf-8") 626 | stap.execute(probe, options) 627 | 628 | 629 | @stap.d.enable 630 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 631 | help="path to PHP process or module") 632 | @stap.d.arg("--uri", type=str, default="/", metavar="PREFIX", 633 | help="restrict the profiling to URI prefixed by PREFIX") 634 | @stap.d.arg("--limit", type=int, default=10, metavar="N", 635 | help="show the most N frequent backtraces") 636 | @stap.d.arg("--depth", type=int, default=10, metavar="N", 637 | help="limit the backtraces to N function calls") 638 | @stap.d.arg("--time", "-t", default=10, metavar="S", type=int, 639 | help="sample during S seconds") 640 | @stap.d.arg("--frequency", type=int, default=97, 641 | help="sample frequency") 642 | @stap.d.arg("--c", action="store_true", 643 | help="also display C backtrace") 644 | def profile(options): 645 | """Sample backtraces to find the most used ones. 646 | """ 647 | probe = jinja2.Template(ur""" 648 | {{ backtrace.init() }} 649 | 650 | global cantrace; 651 | global traces%; 652 | 653 | probe process("{{ options.php }}").provider("php").mark("request__startup") { 654 | if (user_string_n($arg2, {{ options.uri|length() }}) == "{{ options.uri }}") 655 | cantrace[pid()] = 1; 656 | } 657 | 658 | 659 | probe process("{{ options.php }}").provider("php").mark("request__shutdown") { 660 | delete cantrace[pid()]; 661 | } 662 | 663 | probe timer.us({{ (1000000 / options.frequency)|int() }}) { 664 | if (!cantrace[pid()]) next; 665 | {%- if options.c %} 666 | traces[phpstack_n({{ options.depth }}), sprint_ubacktrace()] <<< 1; 667 | {%- else %} 668 | traces[phpstack_n({{ options.depth }}), 1] <<< 1; 669 | {%- endif %} 670 | } 671 | 672 | probe timer.s({{ options.time }}) { 673 | foreach ([php, c] in traces- limit {{ options.limit }}) { 674 | printf("--- PHP backtrace ---\n"); 675 | printf("%s\n", php); 676 | {%- if options.c %} 677 | printf("---- C backtrace ----\n"); 678 | printf("%s\n", c); 679 | {%- endif %} 680 | ansi_set_color2(30, 46); 681 | printf(" ♦ Number of occurrences: %-6d \n", @count(traces[php, c])); 682 | ansi_reset_color(); 683 | } 684 | exit(); 685 | } 686 | """) 687 | probe = probe.render(options=options, 688 | backtrace=stap.h.php.Backtrace(options.php)).encode("utf-8") 689 | stap.execute(probe, options, "-DMAXSTRINGLEN={}".format(options.depth*200)) 690 | 691 | 692 | stap.run(sys.modules[__name__]) 693 | -------------------------------------------------------------------------------- /php-memcache: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """PHP memcache related tools 5 | ============================ 6 | 7 | This script will handle various instrumentation related to PHP 8 | memcache extension. 9 | 10 | """ 11 | 12 | import sys 13 | import os 14 | import stap 15 | import jinja2 16 | 17 | 18 | @stap.d.enable 19 | @stap.d.arg("--interval", default=5000, type=int, 20 | help="delay between screen updates in milliseconds") 21 | @stap.d.arg("--limit", default=10, type=int, metavar="N", 22 | help="don't display more than N keys") 23 | @stap.d.arg("--server", action="store_true", 24 | help="display top keys per server") 25 | @stap.d.arg("--keys", default=100000, type=int, metavar="N", 26 | help="allocate space for N keys") 27 | def topkeys(options): 28 | """Display the top requested keys. 29 | 30 | No difference is made between a GET or a SET. The :option:`server` 31 | option allows one to display the top requests for each server 32 | instead of a global top. 33 | 34 | """ 35 | probe = jinja2.Template(ur""" 36 | global keys[{{ options.keys }}]; 37 | global total; 38 | 39 | probe process("{{ memcache_so }}").function("mmc_pool_schedule") { 40 | {%- if options.server %} 41 | try { 42 | server = user_string_n($mmc->host, 16); 43 | } catch { 44 | server = "???"; 45 | } 46 | {%- endif %} 47 | keylen = $request->key_len; 48 | if (keylen <= 0) next; 49 | key = user_string_n($request->key, keylen); 50 | {%- if options.server %} 51 | keys[server, key]++; 52 | {%- else %} 53 | keys[key]++; 54 | {%- endif %} 55 | total++; 56 | } 57 | 58 | probe timer.ms({{ options.interval }}) { 59 | ansi_clear_screen(); 60 | ansi_cursor_hide(); 61 | ansi_set_color2(30, 42); 62 | {% if options.server %} 63 | printf("%6s │ %-16s │ %-20s ", "Count", "Server", "Key"); 64 | {% else %} 65 | printf("%6s │ %20s ", "Count", "Key"); 66 | {% endif %} 67 | ansi_reset_color(); 68 | ansi_new_line(); 69 | {% if options.server %} 70 | foreach ([server, key] in keys- limit {{ options.limit }}) { 71 | printf("%6d │ %-16s │ %-20s", keys[server, key], server, key); 72 | ansi_new_line(); 73 | } 74 | {% else %} 75 | foreach (key in keys- limit {{ options.limit }}) { 76 | printf("%6d │ %20s", keys[key], key); 77 | ansi_new_line(); 78 | } 79 | {% endif %} 80 | ansi_new_line(); 81 | ansi_set_color2(30, 46); 82 | printf(" Total: %6d ", total); 83 | ansi_reset_color(); 84 | ansi_new_line(); 85 | delete keys; 86 | total = 0; 87 | } 88 | """) 89 | probe = probe.render(memcache_so=os.path.join(stap.h.php.extension_dir(), 90 | "memcache.so"), 91 | options=options).encode("utf-8") 92 | stap.execute(probe, options) 93 | 94 | 95 | @stap.d.enable 96 | @stap.d.arg("--php", type=str, default="/usr/lib/apache2/modules/libphp5.so", 97 | help="path to PHP process or module") 98 | @stap.d.arg("--no-php-backtrace", dest="backtrace", 99 | action="store_false", 100 | help="do not display a PHP backtrace") 101 | def failures(options): 102 | """Display traceback for failures as well as key if available. 103 | 104 | Displaying the PHP traceback can be resource intensive. You can 105 | disable this with :option:`--no-php-backtrace`. 106 | 107 | """ 108 | probe = jinja2.Template(ur""" 109 | {% if options.backtrace %} 110 | {{ backtrace.init() }} 111 | {% endif %} 112 | 113 | function display_request(req:long, what:string) { 114 | if (req == 0 || @cast(req, "mmc_request_t", "{{ memcache_so }}")->key_len == 0) { 115 | printf(" %s: none\n", what); 116 | return 0; 117 | } 118 | key = user_string_n(@cast(req, "mmc_request_t", "{{ memcache_so }}")->key, 119 | @cast(req, "mmc_request_t", "{{ memcache_so }}")->key_len); 120 | printf(" %s: %s\n", what, key); 121 | } 122 | 123 | probe process("{{ memcache_so }}").function("mmc_server_deactivate") { 124 | try { 125 | server = user_string_n($mmc->host, 16); 126 | } catch { 127 | server = "???"; 128 | } 129 | printf("\n\n✄-------------------------------\n"); 130 | ansi_set_color2(30, 41); 131 | printf("⚠ Failure from %s: %d (%s) \n", 132 | server, $mmc->errnum, user_string($mmc->error)); 133 | ansi_reset_color(); 134 | display_request($mmc->sendreq, "sendreq"); 135 | display_request($mmc->readreq, "readreq"); 136 | display_request($mmc->buildreq, "buildreq"); 137 | print_ubacktrace(); 138 | 139 | {% if options.backtrace %} 140 | printf("PHP backtrace:\n"); 141 | print(phpstack()); 142 | {% endif %} 143 | } 144 | """) 145 | probe = probe.render(memcache_so=os.path.join(stap.h.php.extension_dir(), 146 | "memcache.so"), 147 | backtrace=stap.h.php.Backtrace(options.php), 148 | options=options).encode("utf-8") 149 | stap.execute(probe, options) 150 | 151 | 152 | stap.run(sys.modules[__name__]) 153 | -------------------------------------------------------------------------------- /profile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Generic profiling tools 5 | ============================ 6 | 7 | This script will handle profiling in a generic way using kernel and 8 | userspace backtraces. 9 | 10 | """ 11 | 12 | import sys 13 | import stap 14 | import jinja2 15 | 16 | 17 | @stap.d.enable 18 | @stap.d.linux("3.11") 19 | @stap.d.arg_pid 20 | @stap.d.arg("--time", "-t", default=10, metavar="S", type=int, 21 | help="sample during S seconds") 22 | @stap.d.arg("--limit", "-l", default=100, metavar="N", type=int, 23 | help="only display the N most frequent backtraces") 24 | @stap.d.arg("--kernel", action="store_true", 25 | help="Sample in kernel-space") 26 | @stap.d.arg("--user", action="store_true", 27 | help="Sample in user-space") 28 | def backtrace(options): 29 | """Backtrace profiling. 30 | 31 | Sample backtraces and display the most frequent ones. By default, 32 | only kernel is sampled. When requesting user backtraces, a PID 33 | should be specified, otherwise, the backtraces will be mangled. 34 | 35 | It is possible to generate a flamegraph using Brendan Gregg's 36 | scripts available here: 37 | 38 | https://github.com/brendangregg/FlameGraph 39 | """ 40 | if not options.kernel and not options.user: 41 | options.kernel = True 42 | probe = jinja2.Template(ur""" 43 | global backtraces%; 44 | global quit; 45 | 46 | probe timer.profile { 47 | if (!quit) { 48 | backtraces[{{ backtraces }}] <<< 1; 49 | } else { 50 | foreach ([sys, usr] in backtraces- limit {{ options.limit }}) { 51 | {%- if options.kernel %} 52 | print_stack(sys); 53 | {%- endif %} 54 | {%- if options.user %} 55 | print_ustack(usr); 56 | {%- endif %} 57 | ansi_set_color2(30, 46); 58 | printf(" ♦ Number of occurrences: %-6d \n", @count(backtraces[sys, usr])); 59 | ansi_reset_color(); 60 | } 61 | exit(); 62 | } 63 | } 64 | 65 | probe timer.s({{ options.time }}) { 66 | printf("Quitting...\n"); 67 | quit = 1; 68 | } 69 | """) 70 | if options.kernel and options.user: 71 | backtraces = "backtrace(), ubacktrace()" 72 | elif options.kernel: 73 | backtraces = "backtrace(), 1" 74 | else: 75 | backtraces = "1, ubacktrace()" 76 | probe = probe.render(options=options, 77 | backtraces=backtraces).encode("utf-8") 78 | args = options.kernel and ["--all-modules"] or [] 79 | stap.execute(probe, options, *args) 80 | 81 | 82 | @stap.d.enable 83 | @stap.d.linux("4.3") 84 | @stap.d.arg_pid 85 | @stap.d.arg("--time", "-t", default=10, metavar="S", type=int, 86 | help="sample during S seconds") 87 | @stap.d.arg("--limit", "-l", default=100, metavar="N", type=int, 88 | help="only display the N most frequent backtraces") 89 | @stap.d.arg("--kernel", action="store_true", 90 | help="Sample in kernel-space") 91 | @stap.d.arg("--user", action="store_true", 92 | help="Sample in user-space") 93 | def offcpu(options): 94 | """Off-CPU backtrace profiling. 95 | 96 | Sample backtraces when going off CPU and display the most frequent 97 | ones. By default, only kernel is sampled. When requesting user 98 | backtraces, a PID should be specified, otherwise, the backtraces 99 | will be mangled. 100 | 101 | It is possible to generate a flamegraph using Brendan Gregg's 102 | scripts available here: 103 | 104 | https://github.com/brendangregg/FlameGraph 105 | 106 | """ 107 | if not options.kernel and not options.user: 108 | options.kernel = True 109 | probe = jinja2.Template(ur""" 110 | global backtraces%; 111 | global start_time%; 112 | global quit; 113 | 114 | probe scheduler.cpu_off { 115 | if (!quit) { 116 | if ({{options.condition}}) { 117 | start_time[tid()] = gettimeofday_us(); 118 | } 119 | } else { 120 | foreach ([sys, usr] in backtraces- limit {{ options.limit }}) { 121 | {%- if options.kernel %} 122 | print_stack(sys); 123 | {%- endif %} 124 | {%- if options.user %} 125 | print_ustack(usr); 126 | {%- endif %} 127 | ansi_set_color2(30, 46); 128 | printf(" ♦ Occurrences: %-6d \n", @count(backtraces[sys, usr])); 129 | printf(" ♦ Elapsed time: %-6d \n", @sum(backtraces[sys, usr])); 130 | ansi_reset_color(); 131 | } 132 | exit(); 133 | } 134 | } 135 | 136 | probe scheduler.cpu_on { 137 | if (({{options.condition}}) && !quit) { 138 | t = tid(); 139 | begin = start_time[t]; 140 | if (begin > 0) { 141 | elapsed = gettimeofday_us() - begin; 142 | backtraces[{{ backtraces }}] <<< elapsed; 143 | } 144 | delete start_time[t]; 145 | } 146 | } 147 | 148 | probe timer.s({{ options.time }}) { 149 | printf("Quitting...\n"); 150 | quit = 1; 151 | } 152 | """) 153 | if options.kernel and options.user: 154 | backtraces = "backtrace(), ubacktrace()" 155 | elif options.kernel: 156 | backtraces = "backtrace(), 1" 157 | else: 158 | backtraces = "1, ubacktrace()" 159 | probe = probe.render(options=options, 160 | backtraces=backtraces).encode("utf-8") 161 | args = options.kernel and ["--all-modules"] or [] 162 | stap.execute(probe, options, *args) 163 | 164 | 165 | @stap.d.enable 166 | @stap.d.linux("4.2") 167 | @stap.d.arg("--time", "-t", default=10, metavar="S", type=int, 168 | help="sample during S seconds") 169 | @stap.d.arg("--max-cpus", default=64, metavar="CPU", type=int, 170 | help="maximum number of CPU") 171 | @stap.d.arg("probe", metavar="PROBE", 172 | help="probe point to sample") 173 | def histogram(options): 174 | """Display an histogram of execution times for the provided probe point. 175 | 176 | A probe point can be anything understood by Systemtap. For example: 177 | 178 | - kernel.function("net_rx_action") 179 | - process("haproxy").function("frontend_accept") 180 | """ 181 | # Stolen from: 182 | # https://github.com/majek/dump/blob/master/system-tap/histogram-kernel.stp 183 | probe = jinja2.Template(ur""" 184 | global trace[{{ options.max_cpus }}]; 185 | global etime[{{ options.max_cpus }}]; 186 | global intervals; 187 | 188 | probe {{ options.probe }}.call { 189 | trace[cpu()]++; 190 | if (trace[cpu()] == 1) { 191 | etime[cpu()] = gettimeofday_ns(); 192 | } 193 | } 194 | 195 | probe {{ options.probe }}.return { 196 | trace[cpu()]--; 197 | if (trace[cpu()] <= 0) { 198 | t1_ns = etime[cpu()]; 199 | trace[cpu()] = 0; 200 | etime[cpu()] = 0; 201 | if (t1_ns == 0) { 202 | printf("Cpu %d was already in that function?\n", cpu()); 203 | } else { 204 | intervals <<< (gettimeofday_ns() - t1_ns)/1000; 205 | } 206 | } 207 | } 208 | 209 | probe end { 210 | printf("Duration min:%dus avg:%dus max:%dus count:%d\n", 211 | @min(intervals), @avg(intervals), @max(intervals), 212 | @count(intervals)) 213 | printf("Duration (us):\n"); 214 | print(@hist_log(intervals)); 215 | printf("\n"); 216 | } 217 | probe timer.sec( {{options.time }}) { 218 | exit(); 219 | } 220 | """) 221 | probe = probe.render(options=options).encode("utf-8") 222 | args = "--all-modules" 223 | stap.execute(probe, options, *args) 224 | 225 | 226 | stap.run(sys.modules[__name__]) 227 | -------------------------------------------------------------------------------- /stap/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import argparse 4 | import inspect 5 | import subprocess 6 | import re 7 | 8 | from stap import log 9 | from stap import d 10 | from stap import h 11 | 12 | __all__ = [ "run", "execute", "d", "h" ] 13 | 14 | 15 | def normalize_fn(name): 16 | return name.replace("_", "-") 17 | 18 | def get_subcommands(module): 19 | """Extract list of subcommands in the given module.""" 20 | return [obj 21 | for name, obj in inspect.getmembers(module) 22 | if inspect.isfunction(obj) 23 | and hasattr(obj, "stap_enabled") 24 | and obj.stap_enabled] 25 | 26 | def get_options(module): 27 | """Return the command-line options. 28 | 29 | The provided module will be inspected for functions enabled with 30 | `stap.enable` and provide subcommands for each of them. 31 | """ 32 | raw = argparse.RawDescriptionHelpFormatter 33 | parser = argparse.ArgumentParser(description=module.__doc__, 34 | formatter_class=raw) 35 | 36 | g = parser.add_mutually_exclusive_group() 37 | g.add_argument("--debug", "-d", action="store_true", 38 | default=False, 39 | help="enable debugging") 40 | g.add_argument("--silent", "-s", action="store_true", 41 | default=False, 42 | help="silent output") 43 | 44 | parser.add_argument("--dump", "-D", action="store_true", 45 | help="dump the systemtap script source") 46 | parser.add_argument("--output", "-o", metavar="FILE", 47 | dest="stapoutput", 48 | help="redirect output to FILE") 49 | 50 | stapg = parser.add_argument_group("systemtap") 51 | stapg.add_argument("--stap-module", "-m", metavar="MOD", type=str, 52 | action="append", dest="modules", 53 | help="load unwind data for the specified modules as well") 54 | stapg.add_argument("--stap-arg", "-a", metavar="ARG", type=str, 55 | action="append", 56 | dest="stapargs", 57 | help="pass an extra argument to the stap utility") 58 | stapg.add_argument("--stap-no-overload", action="store_true", 59 | dest="stapnooverload", 60 | help="don't check for overload (dangerous)") 61 | stapg.add_argument("--stap-cmd", metavar="CMD", type=str, 62 | dest="stapcmd", 63 | help="command to run while running the probe") 64 | 65 | subparsers = parser.add_subparsers(help="subcommands", dest="command") 66 | for fn in get_subcommands(module): 67 | subparser = subparsers.add_parser(normalize_fn(fn.__name__), 68 | help=fn.__doc__.split("\n")[0], 69 | description=fn.__doc__, 70 | formatter_class=raw) 71 | if hasattr(fn, "stap_args"): 72 | for args, kwargs in fn.stap_args: 73 | subparser.add_argument(*args, **kwargs) 74 | 75 | options = parser.parse_args() 76 | conditions = [ "(1 == 1)" ] 77 | if "condition" in options and options.condition: 78 | conditions.append("({})".format(options.condition)) 79 | if "pid" in options and options.pid: 80 | conditions.append("(pid() == target())") 81 | if "process" in options and options.process: 82 | p = os.path.basename(options.process) 83 | conditions.append('(execname() == "{}")'.format(p)) 84 | options.condition = "({})".format(" && ".join(conditions)) 85 | return options 86 | 87 | 88 | def run(module): 89 | """Process options and execute subcommand""" 90 | global logger 91 | options = get_options(module) 92 | logger = log.get_logger("stap", options) 93 | try: 94 | for fn in get_subcommands(module): 95 | if normalize_fn(fn.__name__) != options.command: 96 | continue 97 | logger.debug("execute %s subcommand" % options.command) 98 | fn(options) 99 | except Exception as e: 100 | logger.exception(e) 101 | sys.exit(1) 102 | 103 | 104 | def sofiles(pid): 105 | """Retrieve libraries loaded by the process specified by the PID""" 106 | args = [] 107 | with open("/proc/{}/maps".format(pid)) as f: 108 | for line in f: 109 | mo = re.match(r".*\s+(/\S+\.so)$", line.strip()) 110 | if mo and mo.group(1) not in args: 111 | logger.debug("{} is using {}".format(pid, mo.group(1))) 112 | args.append("-d") 113 | args.append(mo.group(1)) 114 | return args 115 | 116 | 117 | def execute(probe, options, *args): 118 | """Execute the given probe with :command:`stap`.""" 119 | cmd = ["stap"] 120 | if not options.silent: 121 | cmd += ["-v"] 122 | if options.stapnooverload: 123 | cmd += ["-DSTP_NO_OVERLOAD"] 124 | if options.stapoutput: 125 | cmd += ["-o", str(options.stapoutput)] 126 | if options.stapargs: 127 | cmd += options.stapargs 128 | if options.stapcmd: 129 | cmd += ["-c", str(options.stapcmd)] 130 | if options.modules: 131 | cmd += ["--ldd"] 132 | for m in options.modules: 133 | cmd += ["-d", m] 134 | if "pid" in options and options.pid: 135 | cmd += ["-x", str(options.pid)] 136 | cmd += sofiles(options.pid) 137 | if "process" in options and options.process: 138 | if "/" in options.process: 139 | cmd += ["-d", options.process, "--ldd"] 140 | else: 141 | logger.warn("process is not fully qualified, " 142 | "additional symbols may be missing") 143 | cmd += args 144 | cmd += ["-"] 145 | 146 | if options.dump: 147 | logger.info("would run the following probe with `{}`".format( 148 | " ".join(cmd))) 149 | print probe 150 | return 151 | 152 | logger.info("execute probe") 153 | logger.debug("using the following command line: %s" % " ".join(cmd)) 154 | st = subprocess.Popen(cmd, 155 | stdin=subprocess.PIPE) 156 | try: 157 | st.communicate(input=probe) 158 | except KeyboardInterrupt: 159 | st.terminate() 160 | st.wait() 161 | sys.exit(0) 162 | 163 | -------------------------------------------------------------------------------- /stap/d.py: -------------------------------------------------------------------------------- 1 | """Useful decorators for stap functions.""" 2 | 3 | import platform 4 | import re 5 | import functools 6 | import logging 7 | logger = logging.getLogger(__name__) 8 | 9 | def warn(what): 10 | """Emit a warning about a specific function.""" 11 | def w(fn): 12 | @functools.wraps(fn) 13 | def wrapper(*args, **kwargs): 14 | logger.warn("command `%s` has been marked as %s" % ( 15 | fn.__name__, what)) 16 | return fn(*args, **kwargs) 17 | d = wrapper.__doc__.split("\n") 18 | d[0] = "[%s] %s" % (what, d[0]) 19 | wrapper.__doc__ = "\n".join(d) 20 | return wrapper 21 | return w 22 | 23 | def arg(*args, **kwargs): 24 | """Add the provided argument to the parser.""" 25 | def w(fn): 26 | if not hasattr(fn, "stap_args"): 27 | fn.stap_args = [] 28 | fn.stap_args.append((args, kwargs)) 29 | return fn 30 | return w 31 | 32 | def arg_pid(fn): 33 | """Accept an argument to specify a PID. 34 | 35 | It is expected that the probes will make use of `options.condition` 36 | to filter out probes. Moreover, :command:`stap` will be invoked 37 | with :option:`-x`. 38 | 39 | """ 40 | if not hasattr(fn, "stap_args"): 41 | fn.stap_args = [] 42 | fn.stap_args.append((("--pid", "-p"), 43 | dict(default=None, metavar="PID", type=int, 44 | help="limit profiling to process with pid PID"))) 45 | return fn 46 | 47 | def arg_process(fn): 48 | """Accept an argument to specify a specific process to watch. 49 | 50 | It is expected that the probes will make use of 51 | `options.condition` to filter out probes. 52 | 53 | """ 54 | if not hasattr(fn, "stap_args"): 55 | fn.stap_args = [] 56 | fn.stap_args.append((("--process", "-P"), 57 | dict(default=None, metavar="NAME", 58 | type=str, 59 | help="limit profiling to the process NAME"))) 60 | return fn 61 | 62 | def enable(fn): 63 | """Enable the function as a valid subcommand.""" 64 | fn.stap_enabled = True 65 | return fn 66 | 67 | def linux(*versions): 68 | """Emit a warning if running on an untested Linux version.""" 69 | def w(fn): 70 | @functools.wraps(fn) 71 | def wrapper(*args, **kwargs): 72 | current = platform.release() 73 | mo = re.match(r"((?:[0-9]+\.?)+)", current) 74 | if mo: 75 | current = mo.group(1).split(".") 76 | for version in versions: 77 | vv = version.split(".") 78 | if vv == current[:len(vv)]: 79 | break 80 | else: 81 | logger.warn("command `%s` has not been tested with Linux %s" % ( 82 | fn.__name__, ".".join(current))) 83 | return fn(*args, **kwargs) 84 | return wrapper 85 | return w 86 | -------------------------------------------------------------------------------- /stap/h/__init__.py: -------------------------------------------------------------------------------- 1 | """Various helpers""" 2 | 3 | import pkgutil 4 | 5 | __all__ = [] 6 | for loader, name, is_pkg in pkgutil.walk_packages(__path__): 7 | __all__.append(name) 8 | module = loader.find_module(name).load_module(name) 9 | exec("{} = module".format(name)) 10 | -------------------------------------------------------------------------------- /stap/h/php.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """PHP helpers""" 4 | 5 | import ctypes 6 | import subprocess 7 | import jinja2 8 | 9 | def extension_dir(): 10 | """Get PHP extension directory.""" 11 | extension_dir = subprocess.check_output(["php", "-r", 12 | 'echo ini_get("extension_dir");']) 13 | return extension_dir 14 | 15 | 16 | class Backtrace(object): 17 | """Helper functions to get PHP backtraces. 18 | 19 | When a backtrace should be displayed, call :method:`display`. The 20 | backtrace will be displayed by reacting to function 21 | return. Therefore, displaying the backtrace can take some time 22 | (until the request is shutdown). To avoid to interlace several 23 | backtraces, only one backtrace can be displayed at a given time. 24 | 25 | """ 26 | 27 | def __init__(self, interpreter): 28 | self.interpreter = interpreter 29 | 30 | def init(self): 31 | code = jinja2.Template(ur""" 32 | function phpstack:string() { 33 | __max_depth = 16; 34 | return phpstack_n(__max_depth); 35 | } 36 | 37 | function phpstack_full:string() { 38 | __max_depth = 16; 39 | return phpstack_full_n(__max_depth); 40 | } 41 | 42 | function __php_functionname:string(t:long) { 43 | if (@cast(t, "zend_execute_data", 44 | "{{ php }}")->function_state->function) { 45 | __name = user_string2(@cast(t, "zend_execute_data", 46 | "{{ php }}")->function_state->function->common->function_name, "(anonymous)"); 47 | if (@cast(t, "zend_execute_data", 48 | "{{ php }}")->function_state->function->common->scope) { 49 | return sprintf("%s::%s", 50 | user_string2(@cast(t, "zend_execute_data", 51 | "{{ php }}")->function_state->function->common->scope->name, "(unknown)"), 52 | __name); 53 | } 54 | return __name; 55 | } 56 | return "???"; 57 | } 58 | 59 | function __php_decode:string(z:long) { 60 | __type = @cast(z, "zval", "{{ php }}")->type; 61 | if (__type == 0) { 62 | __arg = "NULL"; 63 | } else if (__type == 1) { 64 | __arg = sprintf("%d", @cast(z, "zval", "{{ php }}")->value->lval); 65 | } else if (__type == 2) { 66 | __arg = ""; 67 | } else if (__type == 3) { 68 | if (@cast(z, "zval", "{{ php }}")->value->lval) { 69 | __arg = "true"; 70 | } else { 71 | __arg = "false"; 72 | } 73 | } else if (__type == 4) { 74 | __arg = sprintf("", 75 | @cast(z, "zval", "{{ php }}")->value->ht->nNumOfElements); 76 | } else if (__type == 5) { 77 | __arg = sprintf("", z); 78 | } else if (__type == 6) { 79 | __arg = user_string_n_quoted(@cast(z, "zval", "{{ php }}")->value->str->val, 80 | @cast(z, "zval", "{{ php }}")->value->str->len); 81 | } else { 82 | __arg = sprintf("", __type); 83 | } 84 | return __arg; 85 | } 86 | function __php_decode_safe:string(z:long) { 87 | try { 88 | return __php_decode(z); 89 | } catch { 90 | return ""; 91 | } 92 | } 93 | 94 | function __php_functionargs:string(t:long) { 95 | __void = {{ pointer }}; 96 | __nb = user_int(@cast(t, "zend_execute_data", 97 | "{{ php }}")->function_state->arguments); 98 | while (__nb > 0) { 99 | __zvalue = @cast(t, "zend_execute_data", 100 | "{{ php }}")->function_state->arguments; 101 | __arg = __php_decode_safe(&@cast(user_long(__zvalue-__nb*__void), "zval", "{{ php }}")); 102 | __result = sprintf("%s%s%s", __result, (__result != "")?",":"", __arg); 103 | __nb--; 104 | } 105 | return __result; 106 | } 107 | 108 | function __php_location:string(t:long) { 109 | if (!@cast(t, "zend_execute_data", "{{ php }}")->op_array) 110 | return "(???)"; 111 | return sprintf("%s:%d", 112 | user_string2(@cast(t, "zend_execute_data", "{{ php }}")->op_array->filename, "???"), 113 | @cast(t, "zend_execute_data", "{{ php }}")->opline->lineno); 114 | } 115 | 116 | function __php_function:string(t:long, full:long) { 117 | __name = __php_functionname(t); 118 | if (full) __args = __php_functionargs(t); 119 | __location = __php_location(t); 120 | return sprintf("%s(%s) %s", __name, __args, __location); 121 | } 122 | 123 | function __phpstack_n:string(max_depth:long, full:long) { 124 | try { 125 | __t = @var("executor_globals", "{{ php }}")->current_execute_data; 126 | while (__t && __depth < max_depth) { 127 | __result = sprintf("%s\n%s", __result, __php_function(__t, full)); 128 | __depth++; 129 | __t = @cast(__t, "zend_execute_data", "{{ php }}")->prev_execute_data; 130 | } 131 | if (__result == "") return "(empty)"; 132 | return __result; 133 | } catch { 134 | return "(unavailable)"; 135 | } 136 | } 137 | 138 | function phpstack_full_n:string(max_depth:long) { 139 | return __phpstack_n(max_depth, 1); 140 | } 141 | function phpstack_n:string(max_depth:long) { 142 | return __phpstack_n(max_depth, 0); 143 | } 144 | """) 145 | return code.render(php=self.interpreter, 146 | pointer=ctypes.sizeof(ctypes.c_void_p)) 147 | 148 | def display(self, depth=16): 149 | """Display PHP backtrace at th current point.""" 150 | return "print(phpstack_n({}))".format(depth); 151 | -------------------------------------------------------------------------------- /stap/log.py: -------------------------------------------------------------------------------- 1 | """Logging related stuff""" 2 | 3 | import logging 4 | 5 | class ColorizingStreamHandler(logging.StreamHandler): 6 | """Provide a nicer logging output to error output with colors.""" 7 | colors = 'black red green yellow blue magenta cyan white'.split(" ") 8 | color_map = dict([(x, colors.index(x)) for x in colors]) 9 | level_map = { 10 | logging.DEBUG: (None, 'blue', " DBG"), 11 | logging.INFO: (None, 'green', "INFO"), 12 | logging.WARNING: (None, 'yellow', "WARN"), 13 | logging.ERROR: (None, 'red', " ERR"), 14 | logging.CRITICAL: ('red', 'white', "CRIT") 15 | } 16 | csi = '\x1b[' 17 | reset = '\x1b[0m' 18 | 19 | @property 20 | def is_tty(self): 21 | isatty = getattr(self.stream, 'isatty', None) 22 | return isatty and isatty() 23 | 24 | def format(self, record): 25 | message = logging.StreamHandler.format(self, record) 26 | # Build the prefix 27 | params = [] 28 | levelno = record.levelno 29 | if levelno not in self.level_map: 30 | levelno = logging.WARNING 31 | bg, fg, level = self.level_map[levelno] 32 | if bg in self.color_map: 33 | params.append(str(self.color_map[bg] + 40)) 34 | if fg in self.color_map: 35 | params.append(str(self.color_map[fg] + 30)) 36 | params.append("1m") 37 | level = "[{}]".format(level) 38 | 39 | return "\n".join(["{}: {}".format( 40 | self.is_tty and params and ''.join((self.csi, ';'.join(params), 41 | level, self.reset)) or level, 42 | line) for line in message.split('\n')]) 43 | 44 | def get_logger(name, options): 45 | """Get a colorized stream logger""" 46 | logger = logging.getLogger(name) 47 | logger.addHandler(ColorizingStreamHandler()) 48 | logger.setLevel(options.debug and logging.DEBUG or 49 | options.silent and logging.WARNING or logging.INFO) 50 | return logger 51 | -------------------------------------------------------------------------------- /tcp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """TCP related tools 5 | ============================ 6 | 7 | This script will handle various instrumentation related to Linux TCP 8 | stack. It is heavily inspired from agentzh script: 9 | https://github.com/agentzh/nginx-systemtap-toolkit/blob/master/tcp-accept-queue 10 | 11 | """ 12 | 13 | import sys 14 | import os 15 | import stap 16 | import jinja2 17 | 18 | 19 | @stap.d.enable 20 | @stap.d.linux("3.11") 21 | @stap.d.arg("port", metavar="PORT", type=int, 22 | help="listening port to be analyzed") 23 | @stap.d.arg("--interval", default=1000, type=int, 24 | help="delay between screen updates in milliseconds") 25 | def accept_queue_length(options): 26 | """Distribution of SYN and accept queue length.""" 27 | probe = jinja2.Template(ur""" 28 | global syn_qlen_stats 29 | global acc_qlen_stats 30 | global max_syn_qlen 31 | global max_acc_qlen 32 | 33 | probe kernel.function("tcp_v4_conn_request") { 34 | tcphdr = __get_skb_tcphdr($skb); 35 | dport = __tcp_skb_dport(tcphdr); 36 | if (dport != {{ options.port }}) next; 37 | 38 | // First time: compute maximum queue lengths 39 | if (max_syn_qlen == 0) { 40 | max_qlen_log = @cast($sk, 41 | "struct inet_connection_sock")->icsk_accept_queue->listen_opt->max_qlen_log; 42 | max_syn_qlen = (1 << max_qlen_log); 43 | } 44 | if (max_acc_qlen == 0) { 45 | max_acc_qlen = $sk->sk_max_ack_backlog; 46 | } 47 | 48 | syn_qlen = @cast($sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->qlen; 49 | syn_qlen_stats <<< syn_qlen; 50 | 51 | acc_qlen_stats <<< $sk->sk_ack_backlog; 52 | } 53 | 54 | probe timer.ms({{ options.interval }}) { 55 | if (max_syn_qlen == 0) { 56 | printf("No new connection on port {{ options.port }}, yet.\n"); 57 | next; 58 | } 59 | ansi_clear_screen(); 60 | ansi_set_color2(30, 46); 61 | printf(" ♦ Syn queue \n"); 62 | ansi_reset_color(); 63 | print(@hist_log(syn_qlen_stats)) 64 | printf(" — min:%d avg:%d max:%d count:%d\n", 65 | @min(syn_qlen_stats), 66 | @avg(syn_qlen_stats), 67 | @max(syn_qlen_stats), 68 | @count(syn_qlen_stats)); 69 | printf(" — allowed maximum: %d\n\n", max_syn_qlen); 70 | 71 | ansi_set_color2(30, 46); 72 | printf(" ♦ Accept queue \n"); 73 | ansi_reset_color(); 74 | print(@hist_log(acc_qlen_stats)) 75 | printf(" — min:%d avg:%d max:%d count:%d\n", 76 | @min(acc_qlen_stats), 77 | @avg(acc_qlen_stats), 78 | @max(acc_qlen_stats), 79 | @count(acc_qlen_stats)); 80 | printf(" — allowed maximum: %d\n\n", max_acc_qlen); 81 | } 82 | """) 83 | probe = probe.render(options=options).encode("utf-8") 84 | stap.execute(probe, options) 85 | 86 | 87 | @stap.d.enable 88 | @stap.d.linux("3.11") 89 | @stap.d.arg("port", metavar="PORT", type=int, 90 | help="listening port to be analyzed") 91 | @stap.d.arg("--interval", default=1000, type=int, 92 | help="delay between screen updates in milliseconds") 93 | def accept_queue_latency(options): 94 | """Distribution of accept queue latencies.""" 95 | probe = jinja2.Template(ur""" 96 | global begin_times; 97 | global latency_stats; 98 | global found; 99 | 100 | probe kernel.function("tcp_openreq_init") { 101 | tcphdr = __get_skb_tcphdr($skb); 102 | dport = __tcp_skb_dport(tcphdr); 103 | if (dport != {{ options.port }}) next; 104 | 105 | begin_times[$req] = gettimeofday_us(); 106 | {%- if options.debug %} 107 | printf("%s: %p %d\n", ppfunc(), $req, dport); 108 | {%- endif %} 109 | } 110 | 111 | probe kernel.function("inet_csk_accept"), 112 | kernel.function("inet_csk_wait_for_connect").return { 113 | req = @cast($sk, "struct inet_connection_sock")->icsk_accept_queue->rskq_accept_head; 114 | begin = begin_times[req]; 115 | if (!begin) next; 116 | 117 | elapsed = gettimeofday_us() - begin; 118 | {%- if options.debug %} 119 | printf("%s: sk=%p, req=%p, latency=%d\n", ppfunc(), $sk, req, elapsed); 120 | {%- endif %} 121 | latency_stats <<< elapsed; 122 | delete begin_times[req]; 123 | found = 1; 124 | } 125 | 126 | probe timer.ms({{ options.interval }}) { 127 | if (found == 0) { 128 | printf("No new connection on port {{ options.port }}, yet.\n"); 129 | next; 130 | } 131 | ansi_clear_screen(); 132 | ansi_set_color2(30, 46); 133 | printf(" ♦ Accept queueing latency distribution \n"); 134 | ansi_reset_color(); 135 | print(@hist_log(latency_stats)) 136 | printf(" — min:%dus avg:%dus max:%dus count:%d\n", 137 | @min(latency_stats), 138 | @avg(latency_stats), 139 | @max(latency_stats), 140 | @count(latency_stats)); 141 | } 142 | """) 143 | probe = probe.render(options=options).encode("utf-8") 144 | stap.execute(probe, options) 145 | 146 | 147 | @stap.d.enable 148 | @stap.d.linux("3.11") 149 | @stap.d.arg("port", metavar="PORT", type=int, 150 | help="listening port to be analyzed") 151 | def accept_queue_overflow(options): 152 | """Trace SYN/ACK backlog queue overflows""" 153 | probe = jinja2.Template(ur""" 154 | probe kernel.function("tcp_v4_conn_request") { 155 | tcphdr = __get_skb_tcphdr($skb); 156 | dport = __tcp_skb_dport(tcphdr); 157 | if (dport != {{ options.port }}) next; 158 | 159 | syn_qlen = @cast($sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->qlen; 160 | max_syn_qlen_log = @cast($sk, "struct inet_connection_sock")->icsk_accept_queue->listen_opt->max_qlen_log; 161 | max_syn_qlen = (2 << max_syn_qlen_log); 162 | 163 | if (syn_qlen > max_syn_qlen) { 164 | now = tz_ctime(gettimeofday_s()); 165 | printf("[%s] SYN queue is overflown: %d > %d\n", now, syn_qlen, max_syn_qlen); 166 | } 167 | 168 | ack_backlog = $sk->sk_ack_backlog; 169 | max_ack_backlog = $sk->sk_max_ack_backlog; 170 | 171 | if (ack_backlog > max_ack_backlog) { 172 | now = tz_ctime(gettimeofday_s()); 173 | printf("[%s] ACK backlog queue is overflown: %d > %d\n", now, ack_backlog, max_ack_backlog); 174 | } 175 | } 176 | """) 177 | probe = probe.render(options=options).encode("utf-8") 178 | stap.execute(probe, options) 179 | 180 | 181 | @stap.d.enable 182 | @stap.d.linux("4.0.0") 183 | @stap.d.arg("--local", metavar="PORT", type=int, 184 | default=0, 185 | help="filter on local port") 186 | @stap.d.arg("--remote", metavar="PORT", type=int, 187 | default=0, 188 | help="filter on remote port") 189 | @stap.d.arg("--interval", metavar="MS", default=1000, type=int, 190 | help="delay between screen updates in milliseconds") 191 | @stap.d.arg("--step", metavar="SIZE", type=int, 192 | default=200, 193 | help="each bucket represent SIZE kbytes") 194 | def receive_window_size(options): 195 | """Display receive window size advertised for the specified connection""" 196 | # We could watch for tcp_select_window return value but this 197 | # function is inlined. We prefer tcp_option_write which is 198 | # happening a bit later. 199 | probe = jinja2.Template(ur""" 200 | global window_sizes; 201 | global found; 202 | 203 | probe kernel.function("tcp_options_write") { 204 | tcphdr = $ptr - &@cast(0, "tcphdr")[1]; 205 | sport = __tcp_skb_sport(tcphdr); 206 | {%- if options.local != 0 %} 207 | if ({{options.local}} != sport) next; 208 | {%- endif %} 209 | dport = __tcp_skb_dport(tcphdr); 210 | {%- if options.remote != 0 %} 211 | if ({{options.remote}} != dport) next; 212 | {%- endif %} 213 | scaled_window = ntohs(@cast(tcphdr, "tcphdr")->window); 214 | scale_factor = $tp->rx_opt->rcv_wscale 215 | window = scaled_window << scale_factor; 216 | window_sizes <<< window/1024; 217 | found = 1; 218 | {%- if options.debug %} 219 | printf("[%d -> %d]: window size=%d\n", sport, dport, window); 220 | {%- endif %} 221 | } 222 | 223 | probe timer.ms({{ options.interval }}) { 224 | if (found == 0) { 225 | printf("No window sizes captured, yet.\n"); 226 | next; 227 | } 228 | ansi_clear_screen(); 229 | ansi_set_color2(30, 46); 230 | printf(" ♦ Window size distribution \n"); 231 | ansi_reset_color(); 232 | print(@hist_linear(window_sizes, 0, {{ options.step * 20 }}, {{ options.step }})) 233 | printf(" — min:%dkb avg:%dkb max:%dkb count:%d\n", 234 | @min(window_sizes), 235 | @avg(window_sizes), 236 | @max(window_sizes), 237 | @count(window_sizes)); 238 | } 239 | """) 240 | probe = probe.render(options=options).encode("utf-8") 241 | stap.execute(probe, options) 242 | 243 | 244 | 245 | @stap.d.enable 246 | @stap.d.linux("3.16.0", "4.0.0", "4.1.0") 247 | @stap.d.arg("--local", metavar="PORT", type=int, 248 | default=0, 249 | help="filter on local port") 250 | @stap.d.arg("--remote", metavar="PORT", type=int, 251 | default=0, 252 | help="filter on remote port") 253 | @stap.d.arg("--interval", metavar="MS", default=100, type=int, 254 | help="sample rate for a given socket") 255 | @stap.d.arg("--bandwidth", default=False, 256 | action="store_true", 257 | help="display estimated bandwidth") 258 | @stap.d.arg("--extensive", default=False, 259 | action="store_true", 260 | help="display more extensive information") 261 | def sockstat(options): 262 | """Display various socket statistics, a bit like `ss'. 263 | 264 | The data can then be processed with the script available `here 265 | `__. 266 | """ 267 | probe = jinja2.Template(ur""" 268 | global last; 269 | {%- if options.bandwidth %} 270 | global snd_una; 271 | global rcv_nxt; 272 | {%- endif %} 273 | 274 | probe begin { 275 | # Print a header 276 | printf("ts,sk,func,state,sport,dport"); 277 | printf(",rq,wq,advwin,retransmits,probes,backoff"); 278 | printf(",snd_wscale,rcv_wscale,rto,ato,snd_mss,rcv_mss"); 279 | printf(",unacked,sacked,lost,retrans,fackets"); 280 | printf(",last_data_sent,last_data_rcv,last_ack_recv"); 281 | printf(",rcv_ssthresh,rtt,rtt_var,snd_ssthresh,snd_cwnd,advmss,reordering"); 282 | printf(",rcv_rtt,rcv_space,total_retrans"); 283 | printf(",skmem_r,skmem_rb,skmem_t,skmem_tb,skmem_f,skmem_w,skmem_o,skmem_bl"); 284 | {%- if options.extensive %} 285 | printf(",ack_bl,ack_max_bl"); 286 | printf(",gso_segs,rcv_nxt,copied_seq,rcv_wup,snd_nxt,snd_una,snd_sml"); 287 | printf(",window_clamp,snd_cwnd_cnt,snd_cwnd_clamp,prior_cwnd,rcv_wnd,write_seq"); 288 | printf(",pmtu_enabled,pmtu_low,pmtu_high,pmtu_size"); 289 | {%- endif %} 290 | {%- if options.bandwidth %} 291 | printf(",snd_bw,rcv_bw"); 292 | {%- endif %} 293 | printf("\n"); 294 | } 295 | 296 | # Plug to tcp_options_write just to be able to also get receive window. 297 | probe kernel.function("tcp_options_write") { 298 | # Check if we need to retrieve information 299 | state = tcp_ts_get_info_state($tp); 300 | now = gettimeofday_ms(); 301 | prev = last[$tp,state]; 302 | if (now - prev <= {{ options.interval }}) next; 303 | last[$tp,state] = now; 304 | 305 | # Retrieve source and destination port and do filtering 306 | tcphdr = $ptr - &@cast(0, "tcphdr")[1]; 307 | sport = __tcp_skb_sport(tcphdr); 308 | dport = __tcp_skb_dport(tcphdr); 309 | {%- if options.remote != 0 %} 310 | if ({{options.remote}} != dport) next; 311 | {%- endif %} 312 | {%- if options.local != 0 %} 313 | if ({{options.local}} != sport) next; 314 | {%- endif %} 315 | 316 | # Advertised receive window 317 | tcphdr = $ptr - &@cast(0, "tcphdr")[1]; 318 | scaled_window = ntohs(@cast(tcphdr, "tcphdr")->window); 319 | scale_factor = $tp->rx_opt->rcv_wscale 320 | window = scaled_window << scale_factor; 321 | 322 | # Print all available information 323 | printf("%lu,%lu,%s,%s,%d,%d", now, $tp, ppfunc(), 324 | tcp_sockstate_str(state), sport, dport); 325 | printf(",%lu,%lu,%lu,%lu,%lu,%lu", 326 | ($tp->rcv_nxt - $tp->copied_seq) & ((1<<32) - 1), 327 | ($tp->write_seq - $tp->snd_una) & ((1<<32) - 1), 328 | window, 329 | @cast($tp, "inet_connection_sock")->icsk_retransmits, 330 | @cast($tp, "inet_connection_sock")->icsk_probes_out, 331 | @cast($tp, "inet_connection_sock")->icsk_backoff); 332 | printf(",%d,%d,%lu,%lu,%lu,%lu", 333 | $tp->rx_opt->snd_wscale, 334 | $tp->rx_opt->rcv_wscale, 335 | tcp_get_info_rto($tp), 336 | cputime_to_usecs(@cast($tp, "inet_connection_sock")->icsk_ack->ato), 337 | $tp->mss_cache, 338 | @cast($tp, "inet_connection_sock")->icsk_ack->rcv_mss); 339 | printf(",%lu,%lu,%lu,%lu,%lu", 340 | $tp->packets_out, $tp->sacked_out, 341 | $tp->lost_out, $tp->retrans_out, $tp->fackets_out); 342 | printf(",%lu,%lu,%lu", 343 | now - cputime_to_msecs($tp->lsndtime), 344 | now - cputime_to_msecs(@cast($tp, "inet_connection_sock")->icsk_ack->lrcvtime), 345 | now - cputime_to_msecs($tp->rcv_tstamp)); 346 | printf(",%lu,%lu,%lu,%lu,%lu,%lu,%lu", 347 | $tp->rcv_ssthresh, 348 | $tp->srtt_us >> 3, 349 | $tp->mdev_us >> 2, 350 | $tp->snd_ssthresh, 351 | $tp->snd_cwnd, 352 | $tp->advmss, 353 | $tp->reordering); 354 | printf(",%lu,%lu,%lu", 355 | cputime_to_usecs($tp->rcv_rtt_est->rtt)>>3, 356 | $tp->rcvq_space->space, 357 | $tp->total_retrans); 358 | printf(",%lu,%lu,%lu,%lu,%lu,%lu,%lu,%lu", 359 | atomic_read(&@cast($tp, "sock")->sk_backlog->rmem_alloc), 360 | @cast($tp, "sock")->sk_rcvbuf, 361 | atomic_read(&@cast($tp, "sock")->sk_wmem_alloc), 362 | @cast($tp, "sock")->sk_sndbuf, 363 | @cast($tp, "sock")->sk_forward_alloc, @cast($tp, "sock")->sk_wmem_queued, 364 | atomic_read(&@cast($tp, "sock")->sk_omem_alloc), 365 | @cast($tp, "sock")->sk_backlog->len); 366 | 367 | {%- if options.extensive %} 368 | printf(",%u,%u", 369 | @cast($tp, "sock")->sk_ack_backlog, 370 | @cast($tp, "sock")->sk_max_ack_backlog); 371 | printf(",%u,%lu,%lu,%lu,%lu,%lu,%lu", 372 | @choose_defined($tp->gso_segs, 0), 373 | $tp->rcv_nxt, 374 | $tp->copied_seq, 375 | $tp->rcv_wup, 376 | $tp->snd_nxt, 377 | $tp->snd_una, 378 | $tp->snd_sml); 379 | printf(",%lu,%lu,%lu,%lu,%lu,%lu", 380 | $tp->window_clamp, 381 | $tp->snd_cwnd_cnt, 382 | $tp->snd_cwnd_clamp, 383 | $tp->prior_cwnd, 384 | $tp->rcv_wnd, 385 | $tp->write_seq); 386 | printf(",%d,%d,%d,%d", 387 | @cast($tp, "inet_connection_sock")->icsk_mtup->enabled, 388 | @cast($tp, "inet_connection_sock")->icsk_mtup->search_low, 389 | @cast($tp, "inet_connection_sock")->icsk_mtup->search_high, 390 | @cast($tp, "inet_connection_sock")->icsk_mtup->probe_size); 391 | {%- endif %} 392 | 393 | {%- if options.bandwidth %} 394 | last_snd_una = snd_una[$tp]; 395 | last_rcv_nxt = rcv_nxt[$tp]; 396 | snd_una[$tp] = $tp->snd_una; 397 | rcv_nxt[$tp] = $tp->rcv_nxt; 398 | if (last_snd_una != 0 && last_snd_una <= $tp->snd_una) 399 | printf(",%lu", ($tp->snd_una - last_snd_una)*1000/(now - prev)); 400 | else 401 | printf(","); 402 | if (last_rcv_nxt != 0 && last_rcv_nxt <= $tp->rcv_nxt) 403 | printf(",%lu", ($tp->rcv_nxt - last_rcv_nxt)*1000/(now - prev)); 404 | else 405 | printf(","); 406 | {%- endif %} 407 | 408 | printf("\n"); 409 | } 410 | """) 411 | probe = probe.render(options=options).encode("utf-8") 412 | stap.execute(probe, options) 413 | 414 | stap.run(sys.modules[__name__]) 415 | --------------------------------------------------------------------------------