├── .gitignore ├── Gatos ├── CatAvailabilityService.cs ├── Controllers │ ├── AuthController.cs │ ├── CatController.cs │ └── StatsController.cs ├── Dockerfile.analysis ├── Dockerfile.runtime ├── Gatos.csproj ├── Makefile ├── Program.cs ├── Startup.cs ├── appsettings.Development.json ├── appsettings.json ├── auth.json ├── cat.json ├── setup.sh └── stats.json ├── LICENSE ├── README.md ├── a.mat ├── blocky.c ├── bpf-contention.md ├── bpf-cpu.md ├── bpf-dbslower.md ├── bpf-files.md ├── bpf-io.md ├── bpf-issues.md ├── bpf-memleak.md ├── bpf-nodeblocked.md ├── bpf-nodegc.md ├── bpf-nodemysql.md ├── bpf-nodeopens.md ├── bpf-oneliners.md ├── bpf-opens.md ├── bpf-setuidsnoop.md ├── bpf-usdt.md ├── buggy ├── Allocy.java ├── Clienty.java ├── Collecty.java ├── Computey.java ├── Databasey.java ├── Makefile ├── Servery.java ├── Writey.java ├── mysql-connector-java-5.1.40-bin.jar ├── proxy.py └── setup-dns.sh ├── create-db.sql ├── create-user.sql ├── data_access.py ├── dbslower.py ├── dbstat.py ├── dotnet ├── Buggy.csproj ├── Makefile ├── NuGet.Config └── Program.cs ├── ftrace.md ├── heapsnapshot1.png ├── heapsnapshot2.png ├── lockstat-solution.py ├── lockstat.py ├── logger.c ├── matexp.cc ├── mysql-db.sh ├── mysqlsniff.py ├── netsend.py ├── nhttpslower.py ├── node-coredump.md ├── node-memleak.md ├── node-prof.md ├── node-slowdns.md ├── node-slowdown.md ├── node-slowhttp.md ├── nodey ├── app.js ├── bin │ └── www ├── inventory.lst ├── package.json ├── public │ ├── stylesheets │ │ └── style.css │ └── template.html ├── routes │ ├── index.js │ └── users.js ├── run.sh ├── utils │ ├── hashtable.js │ └── position.js └── views │ ├── about.jade │ ├── error.jade │ ├── index.jade │ └── layout.jade ├── pargrep.cc ├── parprimes.c ├── perf-io.md ├── perf-opens.md ├── perf.md ├── perros ├── .gitignore ├── Admin.java ├── App.java ├── Auth.java ├── BadRoute.java ├── Dockerfile ├── Makefile ├── Register.java ├── Stats.java ├── Users.java ├── agent.cc ├── allow-perf-event-open.json ├── contmonitor.py ├── jattach-container.sh ├── perf-std-user.sh ├── post-auth.json ├── post-register-bad.json ├── post-register.json ├── setup.sh └── usersbench.sh ├── pg-slow.sql ├── primes.c ├── server.c ├── server.js ├── setuidsnoop.py ├── setup-fedora.sh ├── slowy ├── App.class └── Slowy.java ├── whales └── Makefile └── wordcount.cc /.gitignore: -------------------------------------------------------------------------------- 1 | wordcount 2 | *.dSYM/ 3 | node_modules/ 4 | -------------------------------------------------------------------------------- /Gatos/CatAvailabilityService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | 6 | namespace Gatos 7 | { 8 | public class Cat 9 | { 10 | public string Name { get; set; } 11 | public int Age { get; set; } 12 | public int Weight { get; set; } 13 | } 14 | 15 | public interface ICatAvailabilityService 16 | { 17 | IEnumerable GetAvailableCats(int maxWeight, int maxAge); 18 | void UpdateCatRepository(); 19 | } 20 | 21 | public class CatAvailabilityService : ICatAvailabilityService 22 | { 23 | private readonly object _syncLock = new object(); 24 | 25 | public IEnumerable GetAvailableCats(int maxWeight, int maxAge) 26 | { 27 | lock (_syncLock) 28 | { 29 | Thread.Sleep(100); // Simulate external service call 30 | return new List 31 | { 32 | new Cat { Name = "Jefferson", Age = maxAge, Weight = maxWeight }, 33 | new Cat { Name = "Diego", Age = 1, Weight = 3 }, 34 | new Cat { Name = "Bertha", Age = maxAge, Weight = 10 } 35 | }; 36 | } 37 | } 38 | 39 | public void UpdateCatRepository() 40 | { 41 | throw new ApplicationException("Cat repository is not available"); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /Gatos/Controllers/AuthController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Microsoft.AspNetCore.Cryptography.KeyDerivation; 6 | using Microsoft.AspNetCore.Mvc; 7 | using Microsoft.AspNetCore.Mvc.ModelBinding; 8 | 9 | namespace Gatos.Controllers 10 | { 11 | public class AuthParameters 12 | { 13 | [BindRequired] public string Username { get; set; } 14 | [BindRequired] public string Password { get; set; } 15 | } 16 | 17 | [Route("api/[controller]")] 18 | public class AuthController : Controller 19 | { 20 | [HttpPost] 21 | public void Post([FromBody] AuthParameters auth) 22 | { 23 | string hashed = Convert.ToBase64String( 24 | KeyDerivation.Pbkdf2(auth.Password, new byte[] {1,2,3,4}, 25 | KeyDerivationPrf.HMACSHA1, 10000, 64)); 26 | // Actually doing something with the hash now 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Gatos/Controllers/CatController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using Microsoft.AspNetCore.Mvc; 7 | using Microsoft.AspNetCore.Mvc.ModelBinding; 8 | 9 | namespace Gatos.Controllers 10 | { 11 | public class CatRequest 12 | { 13 | [BindRequired] public int MaxWeight { get; set; } 14 | [BindRequired] public int MaxAge { get; set; } 15 | } 16 | 17 | public class CatResponse 18 | { 19 | public int NumberOfCatsAvailable { get; set; } 20 | public string BestCatName { get; set; } 21 | } 22 | 23 | [Route("api/[controller]")] 24 | public class CatController : Controller 25 | { 26 | private readonly ICatAvailabilityService _cas; 27 | 28 | public CatController(ICatAvailabilityService cas) 29 | { 30 | _cas = cas; 31 | } 32 | 33 | [HttpPost] 34 | public IActionResult CatsAvailable([FromBody] CatRequest request) 35 | { 36 | if (!ModelState.IsValid) 37 | return BadRequest(); 38 | 39 | var cats = _cas.GetAvailableCats(request.MaxWeight, request.MaxAge) 40 | .ToList(); 41 | 42 | return Json(new CatResponse 43 | { 44 | NumberOfCatsAvailable = cats.Count, 45 | BestCatName = cats.FirstOrDefault()?.Name 46 | }); 47 | } 48 | 49 | [HttpPost("update")] 50 | public void UpdateRepository() 51 | { 52 | Task.Run(() => _cas.UpdateCatRepository()); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Gatos/Controllers/StatsController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using System.Xml; 7 | using Microsoft.AspNetCore.Mvc; 8 | 9 | namespace Gatos.Controllers 10 | { 11 | [Route("api/[controller]")] 12 | public class StatsController : Controller 13 | { 14 | private static readonly Random s_rand = new Random(); 15 | 16 | [HttpPut] 17 | public void Put([FromBody] string stats) 18 | { 19 | if (s_rand.Next() % 17 == 0) 20 | { 21 | System.IO.File.WriteAllBytes("stats.bin", new byte[1000000]); 22 | } 23 | else 24 | { 25 | System.IO.File.WriteAllBytes("stats.bin", new byte[1000]); 26 | } 27 | } 28 | 29 | [HttpGet] 30 | public int Get() 31 | { 32 | XmlDocument doc = new XmlDocument(); 33 | XmlElement root = doc.CreateElement("root"); 34 | doc.AppendChild(root); 35 | for (int i = 0; i < 10000; ++i) 36 | { 37 | XmlElement childStat = doc.CreateElement($"stat{i}"); 38 | root.AppendChild(childStat); 39 | } 40 | return doc.OuterXml.Length; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Gatos/Dockerfile.analysis: -------------------------------------------------------------------------------- 1 | FROM goldshtn/dotnet-runtime:latest 2 | LABEL maintainer="Sasha Goldshtein " 3 | 4 | RUN apt update -y && apt install -y lldb-3.6 && rm -rf /var/lib/apt/lists/* 5 | 6 | VOLUME app 7 | 8 | ENTRYPOINT ["lldb-3.6", "-c", "/app/core", "/app/Gatos"] 9 | -------------------------------------------------------------------------------- /Gatos/Dockerfile.runtime: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | LABEL maintainer="Sasha Goldshtein " 3 | 4 | RUN apt update && apt install -y \ 5 | ca-certificates libc6 libcurl3 libgcc1 libgssapi-krb5-2 libicu52 \ 6 | liblttng-ust0 libssl1.0.0 libstdc++6 libunwind8 libuuid1 zlib1g \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | VOLUME app 10 | 11 | ENTRYPOINT ["/app/Gatos"] 12 | -------------------------------------------------------------------------------- /Gatos/Gatos.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Gatos/Makefile: -------------------------------------------------------------------------------- 1 | SRCROOT = $(CURDIR) 2 | LABROOT = /home/fedora/tracing-workshop 3 | FLAMEGRAPH = $(LABROOT)/FlameGraph 4 | PERFTOOLS = $(LABROOT)/perf-tools 5 | CORECLR = $(SRCROOT)/out/libcoreclr.so 6 | PORT = 8888 7 | 8 | dockersvc: 9 | sudo service docker start 10 | 11 | build: 12 | dotnet publish -c Release -o $(SRCROOT)/out --self-contained -r linux-x64 13 | 14 | run: 15 | ASPNETCORE_URLS="http://*:$(PORT)" COMPlus_PerfMapEnabled=1 \ 16 | COMPlus_EnableEventLog=1 \ 17 | $(SRCROOT)/out/Gatos 18 | 19 | dockerrun: 20 | sudo docker rm -f gatos || true 21 | sudo docker build -t goldshtn/dotnet-runtime:latest - < $(SRCROOT)/Dockerfile.runtime 22 | sudo docker run -d -p $(PORT):$(PORT) -v $(SRCROOT)/out/:/app --name gatos \ 23 | -e 'ASPNETCORE_URLS=http://*:8888' goldshtn/dotnet-runtime:latest 24 | 25 | authbench: 26 | ab -n 10000 -c 10 -T 'application/json' -p ./auth.json localhost:$(PORT)/api/auth 27 | 28 | authrecord: 29 | sudo perf record -a -g -F 97 -- sleep 10 30 | sudo chown fedora perf.data 31 | perf script | $(FLAMEGRAPH)/stackcollapse-perf.pl \ 32 | | $(FLAMEGRAPH)/flamegraph.pl > /tmp/dotnet.svg 33 | curl --upload-file /tmp/dotnet.svg https://transfer.sh 2>/dev/null 34 | 35 | putstatsbench: 36 | ab -n 100000 -T 'application/json' -u ./stats.json localhost:$(PORT)/api/stats 37 | 38 | putstatsrecord: 39 | sudo $(PERFTOOLS)/bin/iolatency 1 10 40 | sudo perf record -a -e block:block_rq_issue -- sleep 10 41 | sudo chown fedora perf.data 42 | perf report -n --sort comm,bytes --stdio 43 | 44 | getstatsbench: 45 | ab -n 10000 localhost:$(PORT)/api/stats 46 | 47 | getstatsrecord: 48 | sudo $(PERFTOOLS)/bin/uprobe \ 49 | "p:$(CORECLR):_ZN3SVR6GCHeap24GarbageCollectGenerationEj9gc_reason" 50 | 51 | catssingle: 52 | curl -v -H 'Content-Type: application/json' \ 53 | --data '{"maxweight":42, "maxage":3}' localhost:$(PORT)/api/cat 54 | 55 | catsrecord: 56 | sudo $(PERFTOOLS)/bin/kprobe -p `pidof Gatos` 'p:tcp_sendmsg size=%dx' 57 | 58 | gcrecord: 59 | sudo lttng create gc-trace --output /tmp 60 | sudo lttng add-context --userspace --type vpid 61 | sudo lttng add-context --userspace --type vtid 62 | sudo lttng add-context --userspace --type procname 63 | sudo lttng enable-event --userspace --tracepoint DotNETRuntime:GCStart* 64 | sudo lttng enable-event --userspace --tracepoint DotNETRuntime:GCEnd* 65 | sudo lttng enable-event --userspace --tracepoint DotNETRuntime:GCHeapStats* 66 | sudo lttng enable-event --userspace --tracepoint DotNETRuntime:GCAllocationTick* 67 | sudo lttng enable-event --userspace --tracepoint DotNETRuntime:GCTriggered 68 | sudo lttng start 69 | sleep 10 70 | sudo lttng stop 71 | sudo lttng destroy 72 | 73 | gcview: 74 | sudo babeltrace /tmp 75 | 76 | gcallocstats: 77 | sudo babeltrace /tmp | grep GCAllocationTick | grep 'TypeName = "[^"]*"' -o \ 78 | | sort | uniq -c | sort -n 79 | 80 | catsbench: 81 | ab -n 1000 -c 10 -T 'application/json' -p cat.json localhost:$(PORT)/api/cat 82 | 83 | catsrecord2: 84 | pidstat -p `pidof Gatos` 1 10 85 | sudo /usr/share/bcc/tools/offcputime -p `pidof Gatos` -f 10 > /tmp/offcpu.stacks 86 | $(FLAMEGRAPH)/flamegraph.pl < /tmp/offcpu.stacks > /tmp/offcpu.svg 87 | curl --upload-file /tmp/offcpu.svg https://transfer.sh 2>/dev/null 88 | 89 | alloccount: 90 | sudo /usr/share/bcc/tools/funccount -d 10 "$(CORECLR):*New*" 91 | 92 | allocstacks: 93 | sudo /usr/share/bcc/tools/stackcount -D 10 -f "$(CORECLR):*New*" > /tmp/alloc.stacks 94 | $(FLAMEGRAPH)/flamegraph.pl < /tmp/alloc.stacks > /tmp/alloc.svg 95 | curl --upload-file /tmp/alloc.svg https://transfer.sh 2>/dev/null 96 | 97 | update: 98 | echo core | sudo tee /proc/sys/kernel/core_pattern 99 | curl -v -H 'Content-Type: application/json' --data "" localhost:$(PORT)/api/cat/update 100 | @echo "Nothing bad seems to have happened. But now let's run some more requests:" 101 | sleep 10 102 | make getstatsbench 103 | 104 | updatelogs: 105 | sudo docker logs gatos 106 | 107 | updateanalyze: 108 | sudo docker cp gatos:/core.1 $(SRCROOT)/out/core 109 | sudo docker build -t analyzer - < $(SRCROOT)/Dockerfile.analysis 110 | sudo docker run --rm -it -v $(SRCROOT)/out:/app analyzer \ 111 | -o "plugin load /app/libsosplugin.so" 112 | sudo docker rm gatos 113 | sudo docker image prune -f 114 | -------------------------------------------------------------------------------- /Gatos/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using Microsoft.AspNetCore; 7 | using Microsoft.AspNetCore.Hosting; 8 | using Microsoft.Extensions.Configuration; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace Gatos 12 | { 13 | public class Program 14 | { 15 | public static void Main(string[] args) 16 | { 17 | BuildWebHost(args).Run(); 18 | } 19 | 20 | public static IWebHost BuildWebHost(string[] args) => 21 | WebHost.CreateDefaultBuilder(args) 22 | .UseStartup() 23 | .Build(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Gatos/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Microsoft.AspNetCore.Builder; 6 | using Microsoft.AspNetCore.Hosting; 7 | using Microsoft.Extensions.Configuration; 8 | using Microsoft.Extensions.DependencyInjection; 9 | using Microsoft.Extensions.Logging; 10 | using Microsoft.Extensions.Options; 11 | 12 | namespace Gatos 13 | { 14 | public class ShutdownException : Exception 15 | { 16 | public UnobservedTaskExceptionEventArgs Event { get; private set; } 17 | 18 | public ShutdownException(UnobservedTaskExceptionEventArgs e) : base() 19 | { 20 | Event = e; 21 | } 22 | } 23 | 24 | public class Startup 25 | { 26 | public Startup(IConfiguration configuration) 27 | { 28 | Configuration = configuration; 29 | } 30 | 31 | public IConfiguration Configuration { get; } 32 | 33 | // This method gets called by the runtime. Use this method to add services to the container. 34 | public void ConfigureServices(IServiceCollection services) 35 | { 36 | services.AddMvc(); 37 | services.AddSingleton(); 38 | 39 | TaskScheduler.UnobservedTaskException += HandleUnhandledExceptions; 40 | } 41 | 42 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 43 | public void Configure(IApplicationBuilder app, IHostingEnvironment env) 44 | { 45 | if (env.IsDevelopment()) 46 | { 47 | app.UseDeveloperExceptionPage(); 48 | } 49 | 50 | app.UseMvc(); 51 | } 52 | 53 | private static void HandleUnhandledExceptions(object o, UnobservedTaskExceptionEventArgs e) 54 | { 55 | throw new ShutdownException(e); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Gatos/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "IncludeScopes": false, 4 | "LogLevel": { 5 | "Default": "Debug", 6 | "System": "Information", 7 | "Microsoft": "Information" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Gatos/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "IncludeScopes": false, 4 | "Debug": { 5 | "LogLevel": { 6 | "Default": "Warning" 7 | } 8 | }, 9 | "Console": { 10 | "LogLevel": { 11 | "Default": "Warning" 12 | } 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Gatos/auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "username": "foo", 3 | "password": "goo" 4 | } 5 | -------------------------------------------------------------------------------- /Gatos/cat.json: -------------------------------------------------------------------------------- 1 | { 2 | "maxage": 6, 3 | "maxweight": 42 4 | } 5 | -------------------------------------------------------------------------------- /Gatos/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # 3 | # Assumes most dependencies have already been installed on this image 4 | # by the setup-fedora.sh script. Specifically, these labs depend on: 5 | # bcc 6 | # perf 7 | # perf-tools 8 | # FlameGraph 9 | # And some miscellaneous utils: pidstat, curl, etc. 10 | 11 | # 12 | # Taken from https://www.microsoft.com/net/core#linuxfedora 13 | # 14 | sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc 15 | 16 | sudo sh -c 'echo -e "[packages-microsoft-com-prod]\nname=packages-microsoft-com-prod \nbaseurl=https://packages.microsoft.com/yumrepos/microsoft-rhel7.3-prod\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" > /etc/yum.repos.d/dotnetdev.repo' 17 | 18 | sudo dnf update -y 19 | sudo dnf install -y libunwind libicu 20 | sudo dnf install -y dotnet-sdk-2.0.0 21 | 22 | # 23 | # LTTng 24 | # 25 | sudo dnf install -y lttng-tools 26 | sudo dnf install -y babeltrace 27 | 28 | # 29 | # Taken from https://docs.docker.com/engine/installation/linux/docker-ce/fedora/#install-docker-ce 30 | # 31 | sudo dnf install -y dnf-plugins-core 32 | sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo 33 | sudo dnf makecache -y fast 34 | sudo dnf install -y docker-ce 35 | sudo systemctl start docker 36 | -------------------------------------------------------------------------------- /Gatos/stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "memory": "ok", 3 | "disk": "ok", 4 | "cpu": "ok" 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Sasha Goldshtein 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Linux Tracing Workshops Materials 2 | 3 | This repository contains examples and hands-on labs for various Linux tracing workshops, focusing on modern tracing tools. To perform these labs, you will need a Linux box that meets several prerequisites. You can also use an EC2 instance supplied by the instructor. 4 | 5 | - - - 6 | 7 | #### Prerequisites 8 | 9 | 1. Linux 4.6+ (the distribution doesn't matter; tested on Ubuntu and Fedora Core) 10 | 1. [perf](https://perf.wiki.kernel.org/index.php/Main_Page) 11 | 1. [perf-map-agent](https://github.com/jrudolph/perf-map-agent) 12 | 1. [FlameGraph](https://github.com/brendangregg/FlameGraph) 13 | 1. [perf-tools](https://github.com/brendangregg/perf-tools) 14 | 1. [bcc](https://github.com/iovisor/bcc/blob/master/INSTALL.md) 15 | 1. [OpenJDK](http://openjdk.java.net) 16 | 1. The **systemtap-sdt-dev** package on Ubuntu or the **systemtap-sdt-devel** package on Fedora/RHEL 17 | 1. [Node.js](https://github.com/nodejs/node/wiki/Installation) built from source with `configure --with-dtrace` 18 | 1. MySQL or [MariaDB](https://mariadb.org) built from source with `-DENABLE_DTRACE=1` 19 | 1. PostgreSQL built from [source](https://github.com/postgres/postgres) with `configure --enable-dtrace` 20 | 1. [MySQL Python Connector](https://dev.mysql.com/doc/connector-python/en/connector-python-installation.html) 21 | 22 | If you're using Fedora 24+, try the experimental [setup script](setup-fedora.sh), which can fetch you a recent kernel from mainline and install all the packages and tools you need for this workshop. Use at your own risk! 23 | 24 | - - - 25 | 26 | #### Strigo Virtual Environment 27 | 28 | When this workshop is delivered as instructor-led training, the instructor will provision a Strigo virtual classroom (EC2 instances) for each student. To use the Strigo virtual environment: 29 | 30 | 1. Log in to Strigo using the link provided by the instructor (you can log in with Google or create a new account, no verification required) 31 | 1. Enter the classroom token (four characters) provided by the instructor to join the classroom 32 | 1. Navigate to the Lab tab (fourth from the top, the icon that looks like a test tube) to get your EC2 instance started 33 | 34 | - - - 35 | 36 | #### Labs 37 | 38 | 1. [Probing Tracepoints with ftrace](ftrace.md) 39 | 1. [CPU Sampling with `perf` and Flame Graphs](perf.md) 40 | 1. [Using BPF Tools: Broken File Opens](bpf-opens.md) 41 | 1. [Using BPF Tools: Slow File I/O](bpf-files.md) 42 | 1. [Using BPF Tools: Chasing a Memory Leak](bpf-memleak.md) 43 | 1. [Using BPF Tools: Database and Disk Stats and Stacks](bpf-io.md) 44 | 1. [Using BPF Tools: Node and JVM USDT Probes](bpf-usdt.md) 45 | 1. [Writing BPF Tools: `setuidsnoop`](bpf-setuidsnoop.md) 46 | 1. [Writing BPF Tools: `dbslower`](bpf-dbslower.md) 47 | 1. [Writing BPF Tools: Contention Stats and Stacks](bpf-contention.md) 48 | 1. [Writing BPF Tools: From BCC GitHub Issues](bpf-issues.md) 49 | 1. [Using `perf` Tools: Broken File Opens](perf-opens.md) 50 | 1. [Using BPF Tools: `trace` and `argdist` One-Liners](bpf-oneliners.md) 51 | 1. [Using BPF Tools: CPU and Off-CPU Investigation](bpf-cpu.md) 52 | 1. [Using `perf` Tools: Slow File I/O](perf-io.md) 53 | 1. [Using BPF Tools: Node Blocked Time Analysis](bpf-nodeblocked.md) 54 | 1. [Using BPF Tools: Node Garbage Collections](bpf-nodegc.md) 55 | 1. [Using BPF Tools: Node File Opens](bpf-nodeopens.md) 56 | 1. [Using BPF Tools: Node Slow MySQL Queries](bpf-nodemysql.md) 57 | 1. [Node Profiling with V8](node-prof.md) 58 | 1. [Node Core Dump Analysis with `llnode`](node-coredump.md) 59 | 1. [Node Memory Leak Analysis](node-memleak.md) 60 | 1. [Node Slow HTTP Requests](node-slowhttp.md) 61 | 1. [Node Slow DNS](node-slowdns.md) 62 | 1. [Node Leaky Slowdown](node-slowdown.md) 63 | 64 | - - - 65 | 66 | (C) Sasha Goldshtein, 2015-2017. All rights reserved. 67 | 68 | -------------------------------------------------------------------------------- /blocky.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | volatile long work_counter; 6 | int req_count; 7 | pthread_mutex_t q_lock; 8 | 9 | void grab_lock() { 10 | pthread_mutex_lock(&q_lock); 11 | } 12 | 13 | void ungrab_lock() { 14 | pthread_mutex_unlock(&q_lock); 15 | } 16 | 17 | void do_work() { 18 | for (int i = 0; i < 50000; ++i) 19 | ++work_counter; // Burn CPU 20 | } 21 | 22 | void * request_processor(void *dummy) { 23 | printf("[*] Request processor initialized.\n"); 24 | while (1) { 25 | grab_lock(); 26 | do_work(); 27 | ungrab_lock(); 28 | } 29 | return NULL; 30 | } 31 | 32 | void flush_work() { 33 | usleep(10000); // 10ms 34 | } 35 | 36 | void * backend_handler(void *dummy) { 37 | printf("[*] Backend handler initialized.\n"); 38 | while (1) { 39 | grab_lock(); 40 | ++req_count; 41 | if (req_count % 1000 == 0) { 42 | printf("[-] Handled %d requests.\n", req_count); 43 | } 44 | if (req_count % 37 == 0) { 45 | flush_work(); 46 | } 47 | ungrab_lock(); 48 | } 49 | return NULL; 50 | } 51 | 52 | #define NTHREADS 2 53 | 54 | int main() { 55 | pthread_t req_processors[NTHREADS]; 56 | pthread_t backend; 57 | 58 | pthread_mutex_init(&q_lock, NULL); 59 | for (int i = 0; i < NTHREADS; ++i) { 60 | pthread_create(&req_processors[i], NULL, request_processor, NULL); 61 | } 62 | pthread_create(&backend, NULL, backend_handler, NULL); 63 | 64 | printf("[*] Ready to process requests.\n"); 65 | pthread_join(backend, NULL); 66 | printf("[*] Exiting.\n"); 67 | 68 | return 0; 69 | } 70 | -------------------------------------------------------------------------------- /bpf-contention.md: -------------------------------------------------------------------------------- 1 | ### Writing BPF Tools: Contention Stats and Stacks 2 | 3 | In this lab, you will experiment with what it takes to write your own BPF-based tools using BCC. You will write C and Python code that analyzes lock acquire and release operations and produces statistics on locks and threads. Although the resulting tool is not going to be ready for production use, it should demonstrate the basic requirements for custom BPF tools. 4 | 5 | - - - 6 | 7 | #### Task 1: Build and Run the Concurrent Application 8 | 9 | First, you'll build a multi-threaded application ([parprimes.c](parprimes.c)) that calculates -- guess what -- prime numbers. It does so in multiple threads and uses a mutex, which makes it a good candidate for our purposes. Run the following command to build it: 10 | 11 | ``` 12 | $ gcc -g -fno-omit-frame-pointer -lpthread parprimes.c -o parprimes # on Fedora 13 | $ gcc -g -fno-omit-frame-pointer -pthread parprimes.c -o parprimes # on Ubuntu 14 | ``` 15 | 16 | Try to run it with the following parameters, just to see that it works: 17 | 18 | ``` 19 | $ ./parprimes 4 10000 20 | ``` 21 | 22 | It should spawn 4 threads, and each thread should print how many primes it was able to find. 23 | 24 | - - - 25 | 26 | #### Task 2: Implement `lockstat`, a Contention Monitoring Tool 27 | 28 | Next, you are going to fill in bits and pieces of [lockstat.py](lockstat.py), a contention monitoring tool based on BCC and uprobes. There's quite a bit of code to take in, so if you'd rather inspect and run a completed solution, it's also available as [lockstat-solution.py](lockstat-solution.py). 29 | 30 | The general idea is to probe multiple functions that have to do with mutexes (and potentially other synchronization mechanisms, in the future). For pthread mutexes, `pthread_mutex_lock` and `pthread_mutex_unlock` are the functions for acquiring and releasing the lock, and `pthread_mutex_init` is the function that initializes a mutex. By probing these functions, you will collect the following information: 31 | 32 | * The initialization stack trace for each mutex, which can be used to identify it later 33 | * The time each thread spent waiting for each mutex in the program, and the call stack where it waited 34 | * The time each thread spent inside each mutex in the program 35 | 36 | By aggregating this information, you will be able to pinpoint the contention points in the program: mutexes that become a bottleneck because many threads spend time waiting for them. You will also be able to determine which threads spend a lot of time holding these mutexes, thereby preventing other threads from making progress. 37 | 38 | Follow the `TODO` comments in the [lockstat.py](lockstat.py) file and fill in pieces of code as necessary. Specifically, you will fill in code that updates BPF hashes and histograms (on the C/BPF, kernel-mode side) and code that attaches probes and reads these data structures (on the Python, user-mode side). 39 | 40 | - - - 41 | 42 | #### Task 3: Monitor Lock Contention with `lockstat` 43 | 44 | Run the following command to keep a bunch of threads busy for a while: 45 | 46 | ``` 47 | $ ./parprimes 4 10000000 48 | ``` 49 | 50 | In a root shell, run your `lockstat` script and observe the results. Note that the lock wait and hold times are not distributed evenly across the threads. Do you understand why? What's the distribution of wait and hold times like? Are there are any long wait times, or long hold times? A long wait time could be caused by contention on the lock, but what could cause a long hold time, considering that the only operation under the lock is incrementing an integer? 51 | 52 | > In this particular example, the lock acquire and release operations are entirely artificial because the `count` variable they protect is actually private to a single thread. 53 | 54 | - - - 55 | 56 | #### Bonus: Add Extra Features to `lockstat` 57 | 58 | Here are some extra features `lockstat` needs to become closer to production use. You can pick any feature and work on it later. Pull requests are welcome! :-) 59 | 60 | * Print filtered information, e.g. only mutexes for which the aggregated wait time was more than N ms 61 | * Support additional synchronization mechanisms, other than a pthread mutex 62 | * Collect average, min, max, and possibly even a histogram of wait times and hold times per mutex and per thread 63 | * Perform sampling to reduce overhead -- collect only N% of the events 64 | 65 | - - - 66 | 67 | -------------------------------------------------------------------------------- /bpf-cpu.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: CPU and Off-CPU Investigation 2 | 3 | In this lab, you will work with several tools from BCC to investigate an application that is supposed to be CPU-bound and active, but in fact spends up a lot of time off-CPU for some reason. 4 | 5 | - - - 6 | 7 | #### Task 1: Basic Workload Characterization 8 | 9 | Run the following command to compile the target application: 10 | 11 | ``` 12 | $ gcc -g -std=gnu99 -fno-omit-frame-pointer -fno-inline -lpthread blocky.c -o blocky # on Fedora 13 | $ gcc -g -std=gnu99 -fno-omit-frame-pointer -fno-inline -pthread blocky.c -o blocky # on Ubuntu 14 | ``` 15 | 16 | Now, run `./blocky`. It prints out the number of requests processed, going at a nice pace. But is it really using all processors effectively? Run `top` (or `htop`) to find out -- it looks like the application is barely making a dent in CPU utilization. Most of the time, the system is idle. 17 | 18 | - - - 19 | 20 | #### Task 2: Profiling The CPU-Bound Part 21 | 22 | > As with all BPF-based tools, you will need a root shell to run the following commands. 23 | 24 | To understand where the application is spending CPU time (which, as we saw, is not most of the wall-clock time), run the following command: 25 | 26 | ``` 27 | # profile -F 997 -p $(pidof blocky) 28 | ``` 29 | 30 | > Note: The `profile` tool requires BCC support for `perf_events`, which was introduced in Linux 4.9. If you are running an older kernel, this tool will not work. You can use `perf record` instead, as discussed in [a previous lab](perf.md). 31 | 32 | After a few seconds, hit Ctrl+C to stop the `profile` tool. The call stacks you get should point to the `request_processor` and `do_work` functions, which are supposed to burn a lot of CPU. But, as we saw above, there's something blocking the application from making progress. 33 | 34 | > Bonus: You can also use `profile` to get flame graphs by using the `-f` switch, which will output folded stacks in the exact format that the flamegraph.pl script expects. If you have time, run `profile -F 997 -f -p $(pidof blocky) > folded-stacks`, hit Ctrl+C after a few seconds, and then run `flamegraph.pl folded-stacks > profile.svg` to generate the flame graph. It's important to mention that this method of generating flame graphs is a lot more efficient than using [`perf`](perf.md), because the stack aggregation (folding) is performed in-kernel rather than in user-space. 35 | 36 | - - - 37 | 38 | #### Task 3: Identifying On- and Off-CPU Time 39 | 40 | To determine how much time the application is spending on-CPU and off-CPU, the `cpudist` tool can be very useful. It produces a histogram of time spent in each of the two CPU states, and helps understand the context-switching characteristics of your application. Let's run it: 41 | 42 | ``` 43 | # cpudist -p $(pidof blocky) 44 | ``` 45 | 46 | After a few seconds, hit Ctrl+C to get a report. You will probably see a bimodal distribution, with a mode around the quantum length on your system (which is typically in the 4-8ms bin), and another, bigger mode around very short intervals -- tens or hundreds of microseconds each. It is these smaller intervals that we should be worried about. These bursts of activity indicate that the application threads want to do some work, and are successful at doing it for a very short time, but then they get switched out. 47 | 48 | Now, let's try to identify how long the application threads spend off-CPU, using the same tool: 49 | 50 | ``` 51 | # cpudist -O -p $(pidof blocky) 52 | ``` 53 | 54 | Again, hit Ctrl+C after a few seconds to get the histogram. You might get a bimodal distribution again, but the biggest mode this time is in the much longer bins -- 8-16ms and 16-32ms. Again, these are time intervals when some application thread was off the CPU waiting for something. So we have a great number of multi-millisecond waits. Where are these waits coming from? 55 | 56 | - - - 57 | 58 | #### Task 4: Getting Off-CPU Stacks 59 | 60 | The `offcputime` tool from BCC can be used to answer this final question. It collects stack traces whenever an application thread is blocked (switched off the CPU), and identifies the duration of time that blockage lasted. The output can be interpreted as a flame graph, much like on-CPU data. Let's run it in folded mode to generate data suitable for producing a flame graph: 61 | 62 | ``` 63 | # offcputime -f -p $(pidof blocky) > folded-stacks 64 | ``` 65 | 66 | After a few seconds, hit Ctrl+C. At this point, you can also stop the blocky process. To generate a flame graph, run: 67 | 68 | ``` 69 | # FlameGraph/flamegraph.pl folded-stacks > offcpu.svg 70 | ``` 71 | 72 | Inspecting the flame graph should show two sources of contention: one in the `backend_handler` function, which calls `nanosleep`, and another in the `request_processor` function, which calls `__lll_lock_wait`, which calls the `futex` syscall. These are specific, identifiable places in the source code that can now be inspected to understand why they're causing blockages. If you review [blocky.c](blocky.c), you'll find that the `backend_handler` function is the main culprit: it acquires a lock which the `request_processor` threads also require to make progress, and then occasionally sleeps while holding the lock! 73 | 74 | - - - 75 | -------------------------------------------------------------------------------- /bpf-files.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Slow File I/O 2 | 3 | In this lab, you will experiment with tracing an application that exhibits latency due to slow I/O operations and will figure out why some of its I/O operations are slower than others. 4 | 5 | - - - 6 | 7 | #### Task 1: Compile and Run the Application 8 | 9 | Run the following command to compile the `logger` application: 10 | 11 | ``` 12 | $ gcc -g -fno-omit-frame-pointer -O0 -pthread logger.c -o logger # on Ubuntu 13 | $ gcc -g -fno-omit-frame-pointer -O0 -lpthread logger.c -o logger # on FC 14 | ``` 15 | 16 | Next, run `./logger` -- it sits quietly in the background and churns some log files. 17 | 18 | > Note: Although specifying `-fno-omit-frame-pointer` is redundant when optimizations are turned off, it's a good habit to compile with this flag whenever you're going to use tracing tools. 19 | 20 | - - - 21 | 22 | #### Task 2: Collect I/O Latency Information 23 | 24 | For the sake of simplicity, assume you were already told that the `logger` application occasionally exhibits latency. You suspect that this is a result of slow I/O operations. Run the following command to get a distribution of block I/O operation latency across the system: 25 | 26 | ``` 27 | # biolatency 1 28 | ``` 29 | 30 | The result is likely a bimodal distribution. A lot of the operations complete very quickly, but there are some outliers that take a bit longer. Next, run the following command to see the actual I/O operations and their latencies: 31 | 32 | ``` 33 | # biosnoop 34 | ``` 35 | 36 | You should be able to quickly see that there are some fairly large I/Os performed by the `logger` application that take longer than other smaller I/Os. 37 | 38 | > Note: If you use an encrypted filesystem, you may see the I/Os requests as occuring on behalf 39 | of dmcrypt. This is a current limitation of `biosnoop`. 40 | 41 | - - - 42 | 43 | #### Task 3: Observe Slow File I/O Operations 44 | 45 | At the time of writing, `biosnoop` does not have filters that let you browse through only slower I/Os. Instead, because we know that `logger` is writing files, we can use the `fileslower` tool from BCC to see which files it is writing, and which operations are taking a bit longer than others: 46 | 47 | ``` 48 | # fileslower 1 49 | ``` 50 | 51 | This should show that `logger` is making occasional 1MB writes to the flush.data file, which are taking longer than the typical 1KB writes to the log.data file. 52 | 53 | - - - 54 | 55 | -------------------------------------------------------------------------------- /bpf-io.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Database and Disk Stats and Stacks 2 | 3 | In this lab, you will experiment with some of the task-focused tools from BCC to monitor disk performance, and, as a bonus, trace the queries performed by a database application. 4 | 5 | - - - 6 | 7 | #### Task 1: Monitor MySQL Insert Operations 8 | 9 | You will need to generate a bunch of data so we can monitor the database while inserting items and while querying them. To do so, use the provided [data_access.py](data_access.py) script. It takes a single command-line argument, which can be either `insert`, `insert_once`, or `select`. But first, we need to make sure MySQL is running: 10 | 11 | ``` 12 | # systemctl start mariadb # on Fedora 13 | # systemctl start mysql # on Ubuntu 14 | ``` 15 | 16 | Now run the insert script using the following command (it will run in an infinite loop): 17 | 18 | ``` 19 | $ ./data_access.py insert 20 | ``` 21 | 22 | In another root shell window, run the following command to get a quick reading at the block I/O operations performed by MySQL while you are inserting rows: 23 | 24 | ``` 25 | # biotop 26 | ``` 27 | 28 | You should see a display refreshing every second with the top disk I/O consumers. What's the I/O rate of the mysqld processes while the insert operations are running? It doesn't look like the disk is getting very busy. Run the following command to see the actual details of individual I/Os: 29 | 30 | ``` 31 | # biosnoop 32 | ``` 33 | 34 | How about the latency distribution? Are there any particularly slow I/Os? 35 | 36 | ``` 37 | # biolatency 5 38 | ``` 39 | 40 | OK, so we are seeing some block I/O being submitted. Let's take a look at the call stacks submitting these I/Os in the kernel, at 10 second intervals: 41 | 42 | ``` 43 | # stackcount -i 10 submit_bio 44 | ``` 45 | 46 | It seems that there are quite a few I/O operations. Let's take a look at some of the slower ones (slower than 1ms): 47 | 48 | ``` 49 | # fileslower 1 50 | ``` 51 | 52 | OK, so which files are being touched by mysqld while you're inserting rows? Run the following command to find out: 53 | 54 | ``` 55 | # filetop 56 | ``` 57 | 58 | Finally, what kind of interrupt load are these disk operations putting on the system? Run the following command to get a per-second summary of time spent servicing various interrupts: 59 | 60 | ``` 61 | # hardirqs 1 62 | ``` 63 | 64 | - - - 65 | 66 | #### Task 2: Monitor MySQL Queries 67 | 68 | You are going to experiment with queries now, so start by running the following command to get 10000 records in the database: 69 | 70 | ``` 71 | $ ./data_access.py insert_once 72 | ``` 73 | 74 | Next, run the following command to perform an infinite stream of queries on the data you just put in the database: 75 | 76 | ``` 77 | $ ./data_access.py select 78 | ``` 79 | 80 | In another root shell window, run a couple of the block I/O statistics commands from the previous task. Do you see any block I/Os being performed? Why do you think that is? 81 | 82 | One possible cause is that the data we need is being served from the file system cache. Stop the select script and run the following command: 83 | 84 | ``` 85 | # cachestat 1 86 | ``` 87 | 88 | Now run the select script again. You'll notice that there are some file accesses (all of which should go through cache successfully), but then they stop and the select script keeps running with no file accesses at all, even though cache. So it looks like MySQL is caching the results *internally*, and not using the system's file system cache. To verify this, while the select script is still running, clear the file system cache: 89 | 90 | ``` 91 | # echo 1 > /proc/sys/vm/drop_caches 92 | ``` 93 | 94 | - - - 95 | 96 | #### Bonus: Monitor MySQL Queries with `trace` and `argdist` 97 | 98 | In addition to the generic scripts from the previous sections, we can instrument MySQL more accurately because it uses USDT for static tracing. (Probing USDT is covered in [the next lab](bpf-usdt.md).) 99 | 100 | While the insert script is running, execute the following command to trace the insert statements as they occur: 101 | 102 | ``` 103 | # trace -p $(pidof mysqld) 'u:/usr/libexec/mysqld:query__start "%s", arg1' 104 | ``` 105 | 106 | You can see that there is a commit statement after every insert statement. Now let's get some statistics over the select script. Run it first, and then the following command to determine how many rows we're getting out of each select: 107 | 108 | ``` 109 | # argdist -p $(pidof mysqld) -C 'u:/usr/libexec/mysqld:select__done():u64:arg2#rows returned' 110 | ``` 111 | 112 | You can experiment with additional USDT probe points embedded in MySQL now, or wait for [the USDT lab](bpf-usdt.md). You can find a list of them [here](https://github.com/MariaDB/server/blob/10.1/include/probes_mysql.d.base). 113 | 114 | - - - 115 | -------------------------------------------------------------------------------- /bpf-issues.md: -------------------------------------------------------------------------------- 1 | ### Writing BPF Tools: From BCC GitHub Issues 2 | 3 | In this lab, you will have the opportunity to contribute to [BCC](https://github.com/iovisor/bcc) development by building tools or taking care of other issues brought up on the project's GitHub repository. The list of issues below is occasionally updated, but some issues might already be closed or reassigned. 4 | 5 | Each task has a rough time estimate attached to it, where: 6 | 7 | * Short means it's probably possible to finish the task in under an hour. 8 | * Medium means the task requires some design and planning, and will probably take a couple of days to complete. 9 | * Open-Ended means the task requires design, planning, collaboration, and a lot of development work, and will probably take more than a few days to complete. 10 | 11 | > NOTE: This list was last updated in October 2016. Some of the tasks below could be closed or addressed already. Please make sure to ask before starting to work on any of them, especially if you are reading this more than a month or two after the last update. 12 | 13 | - - - 14 | 15 | #### Tool Development Tasks 16 | 17 | * [Differentiate pids and tids](https://github.com/iovisor/bcc/issues/547) *Medium* 18 | 19 | Update all tools that take a command-line pid or tid argument and that print a pid or tid to make it clear what is being printed. The `bpf_get_current_pid_tgid()` helper returns 64 bits, the lower 32 bits being the tid (`task->pid`). A lot of tools that should really match, filter, or print the pid (`task->tgid`) use this value instead. 20 | 21 | * [Use `BPF_PERF_EVENT` Exclusively](https://github.com/iovisor/bcc/issues/540) *Medium* 22 | 23 | There are still a couple of tools remaining that print to the trace pipe, which is shared with other Linux tools. Replace any remaining usages of the trace pipe with the `BPF_PERF_OUTPUT` mechanism and move the old versions of the tools to **old/**. 24 | 25 | * [`lockstat`](https://github.com/iovisor/bcc/issues/378) *Open-Ended* 26 | 27 | This expands on the [`lockstat` lab](bpf-contention.md) to build a full contention monitoring tool with wait graph support. 28 | 29 | * [Allow wildcards or regexes in `trace` probe specifiers](https://github.com/iovisor/bcc/issues/746) *Medium* 30 | 31 | There are some tools like `stackcount` and `funclatency` that support multiple functions at once by having a wildcard or regex specifier. This can be useful for `trace` as well, e.g. `trace t:block:*` or `trace u:pthread:*mutex*`. 32 | 33 | * [Clean up linter issues](https://github.com/iovisor/bcc/issues/745) *Medium* 34 | 35 | The tool contribution guidelines require running pep8 and fixing style issues. There are some tools that got a bit neglected over time and have some issues that pep8 detects. 36 | 37 | * [Replace `stacksnoop` with `trace -KU`](https://github.com/iovisor/bcc/issues/737) *Short* 38 | 39 | Essentially, once stack tracing was introduced in `trace`, there's no longer need for `stacksnoop`. To retain compatibility and discoverability, we can turn `stacksnoop` into a wrapper that invokes `trace` with the appropriate flags. 40 | 41 | * [More options in `opensnoop`](https://github.com/iovisor/bcc/issues/616) *Medium* 42 | 43 | Bring `opensnoop` up to par with an older version that uses ftrace. 44 | 45 | - - - 46 | 47 | #### Documentation Tasks 48 | 49 | * [man Page Version Update](https://github.com/iovisor/bcc/issues/569) *Short* 50 | 51 | Update the tools' man pages to reflect which minimum kernel version is required to run the tool. Notably, tools that rely on perf output buffers require kernel 4.4, tools that rely on stack tracing require kernel 4.6, and tools that rely on BPF support for tracepoints require kernel 4.7. 52 | 53 | * [LINKS.md](https://github.com/iovisor/bcc/issues/466) *Short* 54 | 55 | Add a collection of links, presentations, blog posts and so on referencing the core BCC project. 56 | 57 | * Testing on a New Distribution *Medium* 58 | 59 | Find a distribution that isn't covered by the installation requirements, and see if you can get BCC to compile from source on that platform. Document your findings as a PR to the [INSTALL.md](https://github.com/iovisor/bcc/blob/master/INSTALL.md) file or as a separate file linked from INSTALL.md. 60 | 61 | - - - 62 | 63 | #### Core Development Tasks 64 | 65 | * [Higher-Level Language Support](https://github.com/iovisor/bcc/issues/425) *Open-Ended* 66 | 67 | Explore building a higher-level language (such as DTrace's language) for expressing BPF probes. 68 | 69 | - - - 70 | 71 | -------------------------------------------------------------------------------- /bpf-memleak.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Chasing a Memory Leak 2 | 3 | In this lab, you will experiment with a C++ application that leaks memory over time, and use the BPF `memleak` tool to determine which allocations are being performed and not being freed. You will do all this without recompiling or even relaunching the leaking application. 4 | 5 | - - - 6 | 7 | #### Task 1: Ascertain That There Is a Leak 8 | 9 | First, build the [wordcount](wordcount.cc) application using the following command: 10 | 11 | ``` 12 | $ g++ -fno-omit-frame-pointer -std=c++11 -g wordcount.cc -o wordcount 13 | ``` 14 | 15 | Run the application. It prompts you for a file name and will display a word count if you provide a valid file name. For example, try the application source file, wordcount.cc. But that's not very interesting. Download one or two books from [Project Gutenberg](http://www.gutenberg.org) -- we like Jane Austen's "Pride and Prejudice", but that's totally up to you. Try giving these as input to the application. 16 | 17 | In another shell, run `top` or `htop` and monitor the application's memory usage. When you provide a large file, memory usage goes up and doesn't seem to go down at all. This looks like a memory leak -- some of the data is not being reclaimed between subsequent runs of the tool. 18 | 19 | - - - 20 | 21 | #### Task 2: Finding the Leak Source with `memleak` 22 | 23 | The `memleak` tool from BCC attaches to memory allocation and deallocation functions (using kprobes or uprobes) and collects data on any allocation that is not being freed. Specifically, for user-mode C++ applications, `memleak` can attach to the `malloc` and `free` functions from libc. For kernel memory leaks, `memleak` can attach to the `kmalloc` and `kfree` functions. 24 | 25 | Use the following command to run `memleak` and attach to the word count application: 26 | 27 | ``` 28 | # memleak -p $(pidof wordcount) 29 | ``` 30 | 31 | By default, `memleak` will print the top 10 stacks sorted by oustanding allocations -- that is, allocations performed and not freed since the tool has attached to your process. Try to analyze these call stacks yourself. 32 | 33 | > Note that the C++ compiler does not make it very easy to understand what's going on because of name mangling -- you can pipe `memleak`'s output through the `c++filt` utility, which should help. For example: `memleak -p $(pidof wordcount) | stdbuf -oL c++filt` 34 | 35 | The most obvious source of allocations is in the `word_counter::word_count` method, which calls `std::copy` to read from the input file. It pushes a bunch of strings into a vector using a `std::back_insert_iterator`. However, it's not obvious why these strings aren't being reclaimed when we move to the next file. What's surprising is that the `std::shared_ptr` to the `word_counter` class, allocated in the `main` function, isn't being freed either. For each file processed, we allocate a 36 | `word_counter` that is not freed. At this point, you could inspect the [wordcount.cc](wordcount.cc) source file carefully and try to determine where the memory leak is coming from. 37 | 38 | > Hint: `std::shared_ptr`s do not automatically break cyclic references. Two objects referring to each other through a `std::shared_ptr` will not be automatically reclaimed. 39 | 40 | - - - 41 | 42 | #### Bonus: Extending `memleak` 43 | 44 | Currently, `memleak` is designed to trace *memory* allocations and deallocations. However, there is nothing special about memory -- it is a resource like any other that is allocated and freed in a pair of specific functions. We could use the same approach to trace an arbitrary pair of functions -- `open` and `close` for file descriptions, `sqlite3_open` and `sqlite3_close` for SQLite database handles, and so on. 45 | 46 | Take a look at [memleak.py](https://github.com/iovisor/bcc/blob/master/tools/memleak.py) and try to figure out what changes will be required to get `memleak` (or a more general tool) to work with arbitrary pairs of resource-allocating and -freeing functions. Pull requests are always welcome! 47 | 48 | - - - 49 | 50 | -------------------------------------------------------------------------------- /bpf-nodeblocked.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Node Blocked Time Analysis 2 | 3 | In this lab, you will experiment with identifying blocked time in a Node application. Because Node apps have only one thread for running JavaScript code, any blocking can be disastrous, and figuring out why the thread was blocked can be hard. However, with a tool designed for monitoring context switches and aggregating them in a lightweight way, blocking bottlenecks can be easily found and fixed. 4 | 5 | - - - 6 | 7 | #### Task 1: Run the Slow App 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start our simple Node application (the `perf` subcommand makes the script invoke the Node runtime with `--perf_basic_prof`, which generates a map file needed for translating addresses to method names): 10 | 11 | ``` 12 | $ ./run.sh perf 13 | ``` 14 | 15 | Now, run `top` in one window, and the following command in another window, to monitor the CPU utilization of the app while we hit the `/stats` endpoint: 16 | 17 | ``` 18 | $ ab -n 10 http://localhost:3000/stats 19 | ``` 20 | 21 | There is no visible CPU activity in the Node process, but the benchmark results are not great: the response time is not nearly instantaneous. To figure out what's happening, we need to look beyond just CPU time, and inspect off-CPU time as well, also known as blocked time. 22 | 23 | - - - 24 | 25 | #### Task 2: Collect Off-CPU Stacks with `offcputime` 26 | 27 | The `offcputime` tool from [BCC](https://github.com/iovisor/bcc) can collect off-CPU stacks for your application's threads, and aggregate them in a way suitable for flame graph generation or direct browsing. Run the following command in a root shell: 28 | 29 | ``` 30 | # offcputime -p $(pgrep -n node) -f > folded.stacks 31 | ``` 32 | 33 | ...and in the original shell, run the benchmark command again: 34 | 35 | ``` 36 | $ ab -n 10 http://localhost:3000/stats 37 | ``` 38 | 39 | Go back to the root shell and hit Ctrl+C so that the tool finalizes the recording. The folded.stacks file is now ready; you can take a look at it directly -- it contains stacks from the Node application when the thread was blocked, i.e. transitioning _off_ the CPU (as opposed to traditional CPU profiling, which looks only at _on_-CPU time). 40 | 41 | It is probably easier to generate a flame graph from the recorded stacks to see what the event thread is doing: 42 | 43 | ``` 44 | $ cat folded.stacks | FlameGraph/flamegraph.pl > offcputime.svg 45 | ``` 46 | 47 | It should become evident that the application is spending lots of time writing to files, and doing so _synchronously_ from the event loop thread. This blocks further request processing, effectively serializing the handling of these requests. One option would be to rewrite this section of the code to use asynchronous file operations, but first, we need to understand which files are being accessed, and why. Perhaps the file writes are really small, and we simply have a defective 48 | disk, which is making these writes execute so slowly? 49 | 50 | > NOTE: Detecting synchronous operations on the event thread is also possible using the `--trace-sync-io` command-line switch. Try launching Node directly with this switch, and seeing the exact call stacks responsible. This doesn't help, though, when background threads issue the slow I/O operations, or when you're investigating a production issue. 51 | 52 | - - - 53 | 54 | #### Task 3: Identify Files Being Accessed with `fileslower` 55 | 56 | Now that we know the application is spending a lot of time synchronously writing to files, we want to identify which files are involved, the write sizes, and the individual operation latencies. Even though this application is issuing writes, the same technique can be used for reads; e.g., to identify which static files being served by the application can be cached at the application level or by an upstream proxy. 57 | 58 | Run the following command, which uses the `fileslower` tool from BCC, to get a capture of all the file accesses performed by the Node process taking longer than 1ms: 59 | 60 | ``` 61 | # fileslower -p $(pgrep -n node) 1 62 | ``` 63 | 64 | While this is executing in the root shell, run the benchmark one more time in the original shell: 65 | 66 | ``` 67 | $ ab -n 10 http://localhost:3000/stats 68 | ``` 69 | 70 | In the root shell, `fileslower` now shows the exact files being written, the write sizes, and the latency of each individual operation. It also shows clearly that there are _large_ file writes being performed synchronously, so the disk is probably not to blame for the latency. 71 | 72 | - - - 73 | -------------------------------------------------------------------------------- /bpf-nodegc.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Node Garbage Collections 2 | 3 | In this lab, you will experiment with a Node application that allocates a lot of memory, causing significant garbage collection delays. You will identify the number of GCs, their latencies, and the code locations making large memory allocations that cause the GC to kick in. 4 | 5 | - - - 6 | 7 | #### Task 1: Monitor Garbage Collections with `nodegc` 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start the Node application: 10 | 11 | ``` 12 | $ ./run.sh 13 | ``` 14 | 15 | In a root shell, run the following command, which uses the `nodegc` tool from [BCC](https://github.com/iovisor/bcc) for monitoring Node garbage collections (it relies on a version of Node built with USDT probes): 16 | 17 | ``` 18 | # nodegc $(pgrep -n node) 19 | ``` 20 | 21 | Back in the original shell, run the following command to generate some load, whcih causes many garbage collections: 22 | 23 | ``` 24 | $ ab -n 10 http://localhost:3000/users 25 | ``` 26 | 27 | In the `nodegc` shell, note that each individual GC event is printed, along with a timestamp and a duration. Even though there's no single super-long GC, the multi-millisecond pauses add up and can cause significant problems if your application is placed under heavy load. 28 | 29 | > NOTE: The Node runtime has a built-in switch for tracing garbage collection events, which you can turn on when your launch the application, using the `--trace_gc` and/or `--trace_gc_verbose` command-line arguments. You need to do this ahead-of-time, though, and the information is only available through the standard output. You can experiment with this option, too, to understand the trade-off. 30 | 31 | - - - 32 | 33 | #### Task 2: Get a Histogram of Garbage Collection Durations 34 | 35 | The `nodegc` tool is nice, but it prints a lot of messages if your application is causing many garbage collections. What we might be after in many cases is rather a simple histogram showing garbage collection frequency and latency. There's a generic tool in BCC, called `funclatency`, which can attach to an arbitrary function and produce a histogram of its running times. We only need to discover the right function, and we're set. 36 | 37 | There are several interesting functions related to garbage collection in [heap.cc](https://github.com/nodejs/node/blob/2db2857c72c219e5ba1642a345e52cfdd8c44a66/deps/v8/src/heap/heap.cc) on the Node.js source repo. Take a look at some of these: `PerformGarbageCollection`, `CollectGarbage`, `MarkCompact`, `Scavenge`, and others sound promising. The nice thing about tracing arbitrary functions is that we don't need any special probes embedded in the application, or any tracing 38 | flags enabled ahead of time. 39 | 40 | Run the following command from a root shell to get a latency histogram of any function in the Node binary that has `PerformGarbageCollection` in its name, using the `funclatency` tool: 41 | 42 | ``` 43 | # funclatency -m "$(which node):*PerformGarbageCollection*" 44 | ``` 45 | 46 | In the original shell, run the following command to generate some load from 20 concurrent clients: 47 | 48 | ``` 49 | $ ab -c 20 -n 1000 http://localhost:3000/users 50 | ``` 51 | 52 | Hit Ctrl+C in the root shell to get the histogram. As you can see, there are _many_ garbage collections, even though they're not taking very long. On one system I tested, the total benchmark time was 22 seconds, and there were over 1800 garbage collections taking from 4 to 7 milliseconds (which accounts for at least 7.2 seconds of execution time, assuming all these collections took 4ms). 53 | 54 | - - - 55 | 56 | #### Task 3: Identify Call Stacks Causing Heavy Garbage Collection 57 | 58 | Finally, in some cases it might not be enough to know that you have a lot of GC; you need to know where the garbage is coming from! Because most GCs are triggered by allocations, it often makes sense to simply sample call stacks for GC start events, and see where in the application they are coming from. To resolve JavaScript symbols, we will need to run the Node process with `--perf_basic_prof`, as before: 59 | 60 | ``` 61 | $ ./run.sh perf 62 | ``` 63 | 64 | In a root shell, run the `stackcount` tool from BCC, attaching to the `gc__start` probe, which is also used by `nodegc`: 65 | 66 | ``` 67 | # stackcount -p $(pgrep -n node) "u:$(which node):gc__start" 68 | ``` 69 | 70 | Run the benchmark again in the original shell to generate some GCs: 71 | 72 | ``` 73 | $ ab -n 100 http://localhost:3000/users 74 | ``` 75 | 76 | And finally, hit Ctrl+C in the root shell to get a stack summary. The heaviest stacks are shown on the bottom, and you can clearly see the paths in the application source that triggered collections. You can repeat the experiment, adding `-f` to the `stackcount` invocation, to get a folded stacks file ready for flame graph generation, if you'd like a better visualization. 77 | 78 | - - - 79 | 80 | #### Bonus 81 | 82 | The [`node-gc-profiler`](https://github.com/bretcope/node-gc-profiler) module can be embedded in your Node application and emit events after each GC cycle, detailing the GC duration and type. You can then use this information to programmatically detect long or excessive GCs. If you have time, try this module out with the `nodey` application and see if you can easily get GC statistics reported. 83 | 84 | - - - 85 | -------------------------------------------------------------------------------- /bpf-nodemysql.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Node MySQL Slow Queries 2 | 3 | In this lab, you will experiment with a Node application that performs MySQL database queries, and some of these queries are taking an unreasonably long time. You will trace slow queries on the database side, and also learn how to trace specific query stacks on the Node side. 4 | 5 | - - - 6 | 7 | #### Task 1: Running the Slow Application 8 | 9 | Prepare the MySQL database process by running it first. If you're using the instructor-provided environment, you can launch MySQL using the following command: 10 | 11 | ``` 12 | $ sudo -u mysql /usr/local/mysql/bin/mysqld_safe --user=mysql & 13 | ``` 14 | 15 | Then, prepare the database using the [`mysql-db.sh`](mysql-db.sh) script: 16 | 17 | ``` 18 | $ ./mysql-db.sh 19 | ``` 20 | 21 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start the Node application: 22 | 23 | ``` 24 | $ ./run.sh 25 | ``` 26 | 27 | Now that the application is started, run the following command to query the set of products from the database: 28 | 29 | ``` 30 | $ curl http://localhost:3000/products 31 | ``` 32 | 33 | You should see a JSON document dumping all the products and descriptions. This is great, but feels a bit sluggish; run the following command to benchmark this operation as it is executed by multiple concurrent clients: 34 | 35 | ``` 36 | $ ab -c 5 -n 20 http://localhost:3000/products 37 | ``` 38 | 39 | It takes about 3 seconds on average on one of my test machines to return each product listing. And that sounds extraneous, considering the small size of the output set. 40 | 41 | - - - 42 | 43 | #### Task 2: Tracing Queries on the Database Side 44 | 45 | The `dbslower` tool from [BCC](https://github.com/iovisor/bcc) recognizes MySQL and PostgreSQL, and can trace query executions in general, and specifically queries slower than a given threshold. 46 | 47 | > NOTE: For MySQL, the `dbslower` tool was recently updated with support for tracing the database even if it wasn't built with USDT support, by using uprobes directly. For PostgreSQL, only USDT support is available at the time of writing. 48 | 49 | Run the following command to trace all database queries to the MySQL database (from a root shell): 50 | 51 | ``` 52 | # dbslower -p $(pidof mysqld) mysql -m 0 53 | ``` 54 | 55 | In the original shell, make another trip to the Products page. The root shell should then show the queries executed by the database. It might be a bit hard to see the slow query right away, so you can re-run `dbslower` with a larger threshold, e.g. `dbslower -p $(pidof mysqld) mysql -m 500` to trace only queries slower than 500ms. 56 | 57 | This shows that the `CALL getproduct(97)` query is very slow, taking more than 2 seconds, and much slower than any other query. And it's a bit odd: what's so special about number 97, compared to, say, 96, or 98? Turns out, we can trace database queries on a slightly lower level, by using the USDT probes directly with the `trace` tool from BCC: 58 | 59 | ``` 60 | # trace -p $(pidof mysqld) 'u::query__exec__start "%s", arg1' 61 | ``` 62 | 63 | If you make another trip to the Products page now, and then look at the executed queries, you'll notice that there are additional commands the database performs, which were not in the `dbslower` output. Notably, the `SLEEP` call suddenly shows up, and explains the 2 second delays in executing the `getproduct` call! 64 | 65 | - - - 66 | 67 | #### Task 3: Tracing Queries on the Application Side 68 | 69 | At this point, we have reasonably good clarity on the database side as to the queries being executed. In some cases, though, you might not have access to the database server, and need to perform this kind of tracing on the application side. Although there are APM solutions that can trace database queries from Node.js, there is nothing built-in. We will use a demo tool called [`mysqlsniff`](mysqlsniff.py), which sniffs the MySQL protocol on the socket layer and prints queries matching a given 70 | prefix; it can even include call stacks! 71 | 72 | In a root shell, run the following command from the main labs directory: 73 | 74 | ``` 75 | ./mysqlsniff.py -p $(pgrep -n node) -a __write -f 'CALL getproduct(97)' -S 76 | ``` 77 | 78 | If you visit the Products page again, you'll see that we get the desired query printed to the console, including the Node.js call stack from which this call was invoked! It's an asynchronous stack, so it doesn't show the original application code (only the MySQL module), but this can still occasionally be useful. And in any case, this is the foundation you can use to trace database queries on the application level without any code changes or special agents injected. 79 | 80 | - - - 81 | -------------------------------------------------------------------------------- /bpf-nodeopens.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Node File Opens 2 | 3 | In this lab, you will experiment with a Node application that returns a cryptic error to the client because something goes wrong. You'll investigate the different operations performed by the application prior to issuing the error, and identify the call stack responsible for the faulty operation. 4 | 5 | - - - 6 | 7 | #### Task 1: Identify Failing Syscalls 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start the Node application: 10 | 11 | ``` 12 | $ ./run.sh perf 13 | ``` 14 | 15 | Now, run the following command to request the application's About page: 16 | 17 | ``` 18 | $ curl http://localhost:3000/about 19 | ``` 20 | 21 | Instead of an about page, the application returns an error page with no meaningful error details: "An error occurred". In cases like these, you can start chasing through the source code looking for exceptions, or attaching a debugger, but another approach that works in many cases is workload characterization: trying to identify an interaction the application has made with the OS that failed. 22 | 23 | In a root shell, run the following command, which uses the `syscount` tool from [BCC](https://github.com/iovisor/bcc), to display any failed syscalls performed by the Node process: 24 | 25 | ``` 26 | # syscount -x -p $(pgrep -n node) 27 | ``` 28 | 29 | Repeat the request to the About page a few times, and then hit Ctrl+C in the root shell. You should see a summary of the failing syscalls; notably, `stat` and `open` are featured multiple times, which leads us to the suspicion that the application might be failing to open a file required for its normal operation, but the error is getting swallowed and a generic message is returned to the client. 30 | 31 | - - - 32 | 33 | #### Task 2: Snoop Failed File Opens 34 | 35 | Now that we know we have failures in `stat` and `open`, let's trace all the failed file opens performed by the Node process when this error occurs. We can use the `opensnoop` tool from BCC for this purpose (run from a root shell): 36 | 37 | ``` 38 | # opensnoop -x -p $(pgrep -n node) 39 | ``` 40 | 41 | In the original shell, repeat the command to request the application's About page. 42 | 43 | You should see messages printed by `opensnoop` showing which file the application is trying to open and failing. The error number is also displayed. 44 | 45 | - - - 46 | 47 | #### Task 3: Identify Stacks Opening Files 48 | 49 | The only thing that's left is to try and figure out where the `/etc/nodey.conf` file is being opened. Perhaps you don't recognize this file name, or the application is big enough so that you don't really know which piece of the code might be accessing it. It's time to pull one of the Swiss Army knives BCC has in stock: the `trace` multi-tool, which can attach to arbitrary probe locations and print custom log messages. 50 | 51 | Run the following command from a root shell to attach `trace` to the `open` syscall, filtering only to the Node process, and requiring that the first argument of the `open` syscall is the specific string `/etc/nodey.conf` (the `-U` flag requests the userspace call stack when the event occurs): 52 | 53 | ``` 54 | # trace 'SyS_open (STRCMP("/etc/nodey.conf", arg1))' -U -p $(pgrep -n node) 55 | ``` 56 | 57 | Repeat the command to request the application's About page one final time; in the root shell, you should see an output message with a complete userspace stack from the Node process, pointing exactly to the source location which tried to open the requested file. 58 | 59 | - - - 60 | -------------------------------------------------------------------------------- /bpf-oneliners.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: `trace` and `argdist` One-Liners 2 | 3 | In this lab, you will experiment with the multi-purpose `trace` and `argdist` tools, which place a lot of tracing power just a one-liner shell command away. This kind of ad-hoc analysis is very powerful, and provides a lot of observability into the system even if you don't have a dedicated, well-polished tool. 4 | 5 | - - - 6 | 7 | #### Task 1: Display All System Login Attempts with `trace` 8 | 9 | Whenever a user logs into the system -- and in fact, even when you use `sudo` to run a command as a different user -- one of the `set*uid` syscall family is being invoked. By tracing these syscalls, we can obtain a complete trace of system login and `sudo` activity over time. 10 | 11 | For example, let's trace `setuid` -- the corresponding syscall name is `sys_setuid`: 12 | 13 | ``` 14 | trace '::sys_setuid "uid = %d", arg1' 15 | ``` 16 | 17 | In another shell, run `sudo su` or establish a new SSH connection to the machine. In both cases you should see a trace message in the previous shell. 18 | 19 | Now, try adding additional probe definitions to also trace `setreuid` and `setresuid`. The latter one has three parameters: the real user id, the effective user id, and the saved set-user-id. 20 | 21 | - - - 22 | 23 | #### Task 2: Identify Hot Files with `argdist` 24 | 25 | Next, we're going to identify hot files (written or read frequently) by using `argdist` and probing specific kernel functions: `__vfs_write` and `__vfs_read`. This is a slightly brittle approach, but it's also the one used by a variety of BCC tools, including `fileslower` and `filetop`. (In fact, we are reproducing a tiny part of `filetop`'s capabilities: in production, you should probably use that tool instead.) 26 | 27 | Run the following command to attach to the aforementioned `__vfs_write` and `__vfs_read` functions and collect the file name being accessed (with 5-second summaries): 28 | 29 | ``` 30 | argdist -T 5 -i 5 \ 31 | -C 'p::__vfs_write(struct file *f):char*:f->f_path.dentry->d_name.name#writes' \ 32 | -C 'p::__vfs_read(struct file *f):char*:f->f_path.dentry->d_name.name#reads' 33 | ``` 34 | 35 | In another shell window, run the following command to generate artificial I/O: 36 | 37 | ``` 38 | dd if=/dev/zero of=/dev/null bs=1K count=1M 39 | ``` 40 | 41 | Observe the null and zero files showing up in `argdist`'s output as the hottest files. Also note that you could have filtered the output to a specific process by using the `-p` switch. 42 | 43 | - - - 44 | 45 | #### Task 3: Display PostgreSQL Queries with `trace` 46 | 47 | > Before starting this task, you will need the PostgreSQL database running. If you followed the installation instructions, and compiled PostgreSQL from source, you should be able to run it from /usr/local/pgsql with a command similar to `sudo -u postgres /usr/local/pgsql/bin/postgres -D /usr/local/pgsql/data &`. 48 | 49 | In this task, we are going to trace PostgreSQL queries by using the USDT probes embedded into the database engine. Run the `psql` client (change the path if necessary according to your system): 50 | 51 | ``` 52 | sudo -u postgres /usr/local/pgsql/bin/psql 53 | ``` 54 | 55 | In another shell, run the following command to discover all the probes embedded into the PostgreSQL process: 56 | 57 | ``` 58 | tplist -p $(pgrep -n postgres) | grep postgres 59 | ``` 60 | 61 | As you see, there are a number of probes for query execution, such as `query__start`, `query__execute__start`, `query__done`, `query__execute__done`, and some others. We are going to use the first one of these, which takes the executed query as its first parameter. The problem is finding the process id to attach to -- you can see all the postgres processes by running `ps -ef | grep postgres`, and pick the backend process for your client connection (experimenting until you find the right one): 62 | 63 | ``` 64 | trace -p $POSTGRES_PID 'u:/usr/local/pgsql/bin/postgres:query__start "%s", arg1' 65 | ``` 66 | 67 | Now, go back to the psql shell and run some SQL statements. For example: 68 | 69 | ``` 70 | create table info (id integer primary key, name varchar(200)); 71 | insert into info values (1, 'Chair'); 72 | select * from info; 73 | ``` 74 | 75 | If you picked the right process id, you should see trace statements after you run each query. If not, try again with another id. 76 | 77 | - - - 78 | 79 | #### Task 4: Display a PostgreSQL Query Latency Histogram with `argdist` 80 | 81 | Once you have the ability to trace queries, it becomes interesting to trace their latency as well. This is not very easy with the USDT probes, because there are two distinct probes we need to trace: `query__start` and `query__done`. Then, we'd need to subtract the end time from the start time to determine the latency -- all of which sounds like too much work for a one-liner. Indeed, in the [`dbslower`](bpf-dbslower.md) lab you will write a custom tool for tracing PostgreSQL/MySQL queries. 82 | 83 | For now, if we want to keep using `argdist`, we need to find a single function to probe. Then, `argdist` will attach to its entry and exit points, and aggregate latency automatically for us. 84 | 85 | Although it doesn't cover the entire time of query execution, we are going to probe the `PortalRun` function (it doesn't include query parsing and planning costs). To trace the latency of that function as a histogram, run the following command: 86 | 87 | ``` 88 | argdist -c -i 5 -H 'r:/usr/local/pgsql/bin/postgres:PortalRun():u64:$latency/1000000#latency (ms)' 89 | ``` 90 | 91 | Copy the [pg-slow.sql](pg-slow.sql) file to /tmp and run `sudo chown postgres /tmp/pg-slow.sql` to let the postgres user access it. Then, hop over to the psql shell (run psql again if you don't have it open) and run the script, while at the same time monitoring the latency output from `argdist`: 92 | 93 | ``` 94 | \i /tmp/pg-slow.sql 95 | ``` 96 | 97 | - - - 98 | -------------------------------------------------------------------------------- /bpf-opens.md: -------------------------------------------------------------------------------- 1 | ### Using BPF Tools: Broken File Opens 2 | 3 | In this lab, you will experiment with diagnosing an application that fails to start correctly by using some of the BCC tools. 4 | 5 | - - - 6 | 7 | #### Task 1: Compile and Run the Application 8 | 9 | First, run the following command to compile `server.c` into the server application you'll be diagnosing: 10 | 11 | ``` 12 | $ gcc -g -fno-omit-frame-pointer -O0 server.c -o server 13 | ``` 14 | 15 | Now, run `./server`. It should print a message saying that it's starting up, but it never completes initialization. 16 | 17 | - - - 18 | 19 | #### Task 2: Perform Basic Diagnostics 20 | 21 | Because the application process seems to be stuck, let's try to see what it's doing. Run `top` -- you should see the app near the top of the output, consuming some CPU. Next, run the following command to see a report of the process' user and system CPU utilization every second: 22 | 23 | ``` 24 | $ pidstat -u -p $(pidof server) 1 25 | ``` 26 | 27 | It looks like the process is spending a bit of time in kernel mode. 28 | 29 | - - - 30 | 31 | #### Task 3: Snoop Syscalls 32 | 33 | If the process is running frequently in kernel mode, it must be making quite a bunch of syscalls. To characterize its workload, we can use the BCC `syscount` tool: 34 | 35 | ``` 36 | # syscount -c -p $(pidof server) 37 | ``` 38 | 39 | This collects all syscall events. Press Ctrl+C after a few seconds to stop collection. It looks like the application is calling `nanosleep()` and `open()` quite frequently. 40 | 41 | - - - 42 | 43 | #### Task 4: Snooping Opens 44 | 45 | Fortunately, BCC has a tool for snooping all `open` calls performed by a certain process, including the path being opened and the result of the call. Run the following command to trace opens: 46 | 47 | ``` 48 | # opensnoop -p $(pidof server) 49 | ``` 50 | 51 | The problem becomes apparent -- the application is trying to open the `/etc/tracing-server-example.conf` file, and is getting a -1 result with an errno of 2, which stands for ENOENT (no such file or directory). Indeed, you can confirm that the file does not exist. You can even try creating the file and making sure the server now starts successfully. 52 | 53 | - - - 54 | 55 | #### Bonus: Use `argdist` for Argument Analysis 56 | 57 | The `argdist` tool from BCC can be used for quick argument analysis when you're interested in the values an application passes to a certain function or syscall. In our case, let's get a histogram of sleep durations by tracing the `nanosleep()` syscall in the application process: 58 | 59 | ``` 60 | # argdist -p $(pidof server) -H 'p::do_nanosleep(struct timespec *time):u64:time->tv_nsec' 61 | ``` 62 | 63 | This prints a histogram (using -H) of the sleep durations, which seem to be concentrated in one specific bin: 512-1023 ns. Indeed, inspecting the application source code you can verify that it calls `usleep(1)`, which corresponds to 1000 nanoseconds. 64 | 65 | Similarly, we could use `argdist` to get a frequency count (using -C) of which files the application is trying to open: 66 | 67 | ``` 68 | # argdist -p $(pidof server) -C 'p:c:open(char *filename):char*:filename' 69 | ``` 70 | 71 | And similarly, we could get a frequency count of return values from `open()`: 72 | 73 | ``` 74 | # argdist -p $(pidof server) -C 'r:c:open():int:$retval' 75 | ``` 76 | 77 | - - - 78 | -------------------------------------------------------------------------------- /bpf-setuidsnoop.md: -------------------------------------------------------------------------------- 1 | ### Writing BPF Tools: `setuidsnoop` 2 | 3 | In this lab, you will modify the existing [`killsnoop`](https://github.com/iovisor/bcc/tree/master/tools/killsnoop.py) tool to trace `setuid` calls, which include SSH logins, sudo, su, and other potentially interesting activity. 4 | 5 | - - - 6 | 7 | #### Task 1: Inspect `killsnoop.py` 8 | 9 | Navigate to the [`killsnoop.py`](https://github.com/iovisor/bcc/tree/master/tools/killsnoop.py) file and inspect the source code. This is a simple kprobe-based tool that attaches a kprobe and a kretprobe to the `sys_kill` function. In the enter probe, data about the kill signal and target process is recorded, and in the return probe, the result of the operation is recorded and a custom structure is submitted to user space for display. 10 | 11 | The Python script mostly deals with parsing arguments and taking care of the BPF program (e.g., by embedding a filter that makes sure only signals to a specific process are traced), and then polls the kernel buffer for results and prints them as they arrive. 12 | 13 | Try running `killsnoop` in one shell, and killing some process (e.g. with `kill -9`) in another shell to see what the tool's output looks like. 14 | 15 | - - - 16 | 17 | #### Task 2: Approximating `setuidsnoop` with `trace` 18 | 19 | For lack of a dedicated tool, we can use the general-purpose `trace` multitool to trace `setuid` calls by attaching two probes: 20 | 21 | ``` 22 | # trace 'sys_setuid "uid=0x%x", arg1' 'r::sys_setuid "rc=%d", retval' 23 | ``` 24 | 25 | In a separate shell, run `sudo su` or a similar command, and note the trace printouts when `setuid` is called.` 26 | 27 | - - - 28 | 29 | #### Task 3: Developing `setuidsnoop` 30 | 31 | A dedicated tool can be better than the trace printouts. For one thing, it would consolidate the `setuid` argument and return code, much like `killsnoop` does. Copy `killsnoop.py` to `setuidsnoop.py` and make the following changes: 32 | 33 | * Attach the kprobe and kretprobe to `sys_setuid` instead of `sys_kill` 34 | * Modify the signature of the kprobe function to match `sys_setuid` 35 | * Modify the data structure to include the uid instead of the kill arguments (both the C and Python structures need to be modified) 36 | * Populate the data structure accordingly in the kprobe and kretprobe functions 37 | * Modify the printing code on the Python side to output the uid and result 38 | 39 | Finally, test your tool by running `sudo su` or a similar command. 40 | 41 | - - - 42 | 43 | -------------------------------------------------------------------------------- /buggy/Allocy.java: -------------------------------------------------------------------------------- 1 | class ResponseBuilder { 2 | private String response; 3 | 4 | public ResponseBuilder(String template) { 5 | response = template; 6 | } 7 | 8 | public void addLine(String line) { 9 | response += line; 10 | } 11 | 12 | public String getResponse() { 13 | return response; 14 | } 15 | } 16 | 17 | class Allocy { 18 | public static void main(String[] args) { 19 | while (true) { 20 | ResponseBuilder rb = new ResponseBuilder("Result:"); 21 | for (int i = 0; i < 100000; ++i) { 22 | rb.addLine("#" + i + ": OK"); 23 | } 24 | rb.addLine("End of response."); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /buggy/Clienty.java: -------------------------------------------------------------------------------- 1 | import java.security.Security; 2 | import java.net.URL; 3 | import java.io.InputStream; 4 | import java.util.Scanner; 5 | import java.util.List; 6 | import java.util.ArrayList; 7 | 8 | class Fetchy { 9 | static { 10 | Security.setProperty( 11 | "networkaddress.cache.negative.ttl", "0"); 12 | } 13 | 14 | private String url; 15 | 16 | public Fetchy(String url) { 17 | this.url = url; 18 | } 19 | 20 | public String fetch() throws Exception { 21 | InputStream res = new URL(url).openStream(); 22 | Scanner scn = new Scanner(res); 23 | return scn.useDelimiter("\\A").next(); 24 | } 25 | } 26 | 27 | class Crawley { 28 | private List urls; 29 | 30 | public Crawley(List urls) { 31 | this.urls = urls; 32 | } 33 | 34 | public void crawl() { 35 | try { 36 | for (String url : urls) { 37 | Fetchy fetchy = new Fetchy(url); 38 | fetchy.fetch(); 39 | } 40 | } catch (Exception e) { 41 | } 42 | } 43 | } 44 | 45 | class Clienty { 46 | public static void main(String[] args) { 47 | ArrayList urls = new ArrayList(); 48 | if (args[0].equals("bad")) { 49 | urls.add("https://i-dont-exist-at-all-20170126.com"); 50 | } else { 51 | urls.add("https://facebook.com"); 52 | } 53 | urls.add("https://google.com"); 54 | urls.add("https://example.org"); 55 | Crawley crawley = new Crawley(urls); 56 | while (true) { 57 | long start = System.currentTimeMillis(); 58 | crawley.crawl(); 59 | long end = System.currentTimeMillis(); 60 | System.out.println("Crawl complete, elapsed: " + 61 | (end-start) + " milliseconds."); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /buggy/Collecty.java: -------------------------------------------------------------------------------- 1 | class DataFetcher { 2 | public void fetchData(String datum, int index) { 3 | if (index == 0 || index % 37 != 0) { 4 | processIt(); 5 | return; 6 | } 7 | System.out.println("Error fetching data, cleaning up."); 8 | System.gc(); 9 | } 10 | 11 | private void processIt() { 12 | } 13 | } 14 | 15 | class RequestProcessor { 16 | private DataFetcher fetcher; 17 | 18 | public RequestProcessor() { 19 | fetcher = new DataFetcher(); 20 | } 21 | 22 | public void processRequest(String url, int index) throws Exception { 23 | if (index != 0 && index % 43 == 0) 24 | fetcher.fetchData(url, index); 25 | else 26 | Thread.sleep(1); 27 | } 28 | } 29 | 30 | class Collecty { 31 | public static void main(String[] args) throws Exception { 32 | RequestProcessor rp = new RequestProcessor(); 33 | for (int i = 0; ; ++i) { 34 | rp.processRequest("/api/all", i); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /buggy/Computey.java: -------------------------------------------------------------------------------- 1 | import java.util.ArrayList; 2 | 3 | class Primes { 4 | private final int from; 5 | private final int to; 6 | private final int numThreads; 7 | private ArrayList primes = new ArrayList<>(); 8 | private Object primesLock = new Object(); 9 | 10 | public Primes(int from, int to, int numThreads) { 11 | this.from = from; 12 | this.to = to; 13 | this.numThreads = numThreads; 14 | } 15 | 16 | public int count() { 17 | return primes.size(); 18 | } 19 | 20 | public void execute() throws Exception { 21 | Thread[] threads = new Thread[numThreads]; 22 | int chunkSize = (to-from)/numThreads; 23 | for (int i = 0; i < numThreads; ++i) { 24 | final int myStart = i*chunkSize; 25 | final int myEnd = (i+1)*chunkSize - 1; 26 | threads[i] = new Thread(new Runnable() { 27 | public void run() { 28 | primesThread(myStart, myEnd); 29 | } 30 | }); 31 | threads[i].start(); 32 | } 33 | for (Thread thread : threads) { 34 | thread.join(); 35 | } 36 | } 37 | 38 | private void primesThread(int from, int to) { 39 | for (int i = from; i <= to; ++i) { 40 | if (isPrime(i)) { 41 | synchronized (primesLock) { 42 | primes.add(i); 43 | } 44 | } 45 | } 46 | } 47 | 48 | private static boolean isPrime(int number) { 49 | // Obviously very inefficient 50 | if (number == 2) return true; 51 | if (number % 2 == 0) return false; 52 | for (int i = 3; i < number; i += 2) 53 | if (number % i == 0) 54 | return false; 55 | return true; 56 | } 57 | } 58 | 59 | class Computey { 60 | public static void main(String[] args) throws Exception { 61 | int from = Integer.parseInt(args[0]); 62 | int to = Integer.parseInt(args[1]); 63 | int numThreads = Integer.parseInt(args[2]); 64 | Primes primes = new Primes(from, to, numThreads); 65 | System.out.println("Press RETURN to start."); 66 | System.in.read(); 67 | primes.execute(); 68 | System.out.println("Found " + primes.count() + " primes"); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /buggy/Databasey.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.util.ArrayList; 3 | 4 | class User { 5 | private int id; 6 | private String name; 7 | 8 | public static User load(int id) throws Exception { 9 | User user = null; 10 | Statement stmt = Databasey.getConnection().createStatement(); 11 | ResultSet res = stmt.executeQuery( 12 | "select * from users where id = " + id); 13 | if (res.next()) { 14 | user = new User(); 15 | user.id = res.getInt("id"); 16 | user.name = res.getString("name"); 17 | } 18 | res.close(); 19 | stmt.close(); 20 | return user; 21 | } 22 | 23 | public Iterable loadProducts() throws Exception { 24 | ArrayList products = new ArrayList<>(); 25 | Statement stmt = Databasey.getConnection().createStatement(); 26 | ResultSet res = stmt.executeQuery( 27 | "select id from products where userid = " + id); 28 | while (res.next()) { 29 | Product prod = Product.load(res.getInt("id")); 30 | products.add(prod); 31 | } 32 | res.close(); 33 | stmt.close(); 34 | return products; 35 | } 36 | 37 | public int getId() { return id; } 38 | public String getName() { return name; } 39 | } 40 | 41 | class Product { 42 | private int id; 43 | private int userId; 44 | private String name; 45 | private String description; 46 | private double price; 47 | 48 | public static Product load(int id) throws Exception { 49 | Product product = null; 50 | Statement stmt = Databasey.getConnection().createStatement(); 51 | ResultSet res = stmt.executeQuery( 52 | "call getproduct(" + id + ")"); 53 | if (res.next()) { 54 | product = new Product(); 55 | product.id = res.getInt("id"); 56 | product.userId = res.getInt("userid"); 57 | product.name = res.getString("name"); 58 | product.description = res.getString("description"); 59 | product.price = res.getDouble("price"); 60 | } 61 | res.close(); 62 | stmt.close(); 63 | return product; 64 | } 65 | } 66 | 67 | class Databasey { 68 | private static Connection connection; 69 | private static final String CONN_STRING = 70 | "jdbc:mysql://localhost/acme?user=newuser&password=password"; 71 | 72 | public static Connection getConnection() { 73 | return connection; 74 | } 75 | 76 | public static void main(String[] args) throws Exception { 77 | Class.forName("com.mysql.jdbc.Driver"); 78 | connection = DriverManager.getConnection(CONN_STRING); 79 | while (true) { 80 | for (int i = 0; i < 100; ++i) { 81 | User user = User.load(i); 82 | user.loadProducts(); 83 | System.out.println("Loaded for user " + i); 84 | } 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /buggy/Makefile: -------------------------------------------------------------------------------- 1 | JAVAFLAGS=-XX:+PreserveFramePointer -Xcomp 2 | JAVAEXTFLAGS=-XX:+ExtendedDTraceProbes 3 | 4 | build: 5 | mkdir -p bin 6 | cp mysql-connector-java-5.1.40-bin.jar bin/ 7 | javac *.java -d bin 8 | 9 | clean: 10 | rm -rf bin 11 | 12 | allocy: build 13 | java $(JAVAFLAGS) -cp bin Allocy 14 | 15 | allocyext: build 16 | java $(JAVAFLAGS) $(JAVAEXTFLAGS) -cp bin Allocy 17 | 18 | clientygood: build 19 | sudo pip install dnslib 20 | sudo ./proxy.py > /dev/null & 21 | java $(JAVAFLAGS) -cp bin Clienty good 22 | 23 | clientybad: build 24 | sudo pip install dnslib 25 | sudo ./proxy.py > /dev/null & 26 | java $(JAVAFLAGS) -cp bin Clienty bad 27 | 28 | collecty: build 29 | java $(JAVAFLAGS) $(JAVAEXTFLAGS) -cp bin Collecty 30 | 31 | computey: build 32 | java $(JAVAFLAGS) -cp bin Computey 2 4000000 4 33 | 34 | computeyext: build 35 | java $(JAVAFLAGS) $(JAVAEXTFLAGS) -cp bin Computey 2 4000000 4 36 | 37 | databasey: build 38 | java $(JAVAFLAGS) -cp "bin:bin/mysql-connector-java-5.1.40-bin.jar" \ 39 | Databasey 40 | databaseyext: build 41 | java $(JAVAFLAGS) $(JAVAEXTFLAGS) \ 42 | -cp "bin:bin/mysql-connector-java-5.1.40-bin.jar" Databasey 43 | 44 | servery: build 45 | java $(JAVAFLAGS) -cp bin Servery 46 | 47 | writey: build 48 | java $(JAVAFLAGS) -cp bin Writey 49 | 50 | all: build 51 | -------------------------------------------------------------------------------- /buggy/Servery.java: -------------------------------------------------------------------------------- 1 | import java.io.File; 2 | import java.util.Scanner; 3 | 4 | class Initializer { 5 | private static boolean initializationComplete = false; 6 | private static final Object initLock = new Object(); 7 | 8 | public static void initializeAsync(String configFile) { 9 | new Thread(new Runnable() { 10 | public void run() { 11 | synchronized (initLock) { 12 | while (!openConfigFile(configFile)) 13 | ; 14 | initializationComplete = true; 15 | initLock.notifyAll(); 16 | } 17 | } 18 | }).start(); 19 | } 20 | 21 | public static void waitForInitialization() { 22 | while (true) { 23 | synchronized (initLock) { 24 | if (initializationComplete) 25 | return; 26 | try { initLock.wait(); } 27 | catch (InterruptedException e) { } 28 | } 29 | } 30 | } 31 | 32 | private static boolean openConfigFile(String configFile) { 33 | System.out.println("[*] Opening config file."); 34 | try { 35 | String content = new Scanner(new File(configFile)) 36 | .useDelimiter("\\Z").next(); 37 | } catch (Exception e) { 38 | try { Thread.sleep(1000); } 39 | catch (InterruptedException e2) {} 40 | return false; 41 | } 42 | System.out.println("[*] Config file read successfully."); 43 | return true; 44 | } 45 | } 46 | 47 | class Servery { 48 | public static void main(String[] args) throws Exception { 49 | Initializer.initializeAsync("/etc/acme-svr.config"); 50 | System.out.println("[*] Server started, initializing."); 51 | Initializer.waitForInitialization(); 52 | System.out.println("[*] Initialization complete!"); 53 | System.in.read(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /buggy/Writey.java: -------------------------------------------------------------------------------- 1 | import java.io.FileOutputStream; 2 | 3 | class Writey { 4 | private static void doWrite(int size) throws Exception { 5 | FileOutputStream fos = new FileOutputStream("db-wal"); 6 | byte[] data = new byte[size]; 7 | fos.write(data); 8 | fos.close(); 9 | } 10 | 11 | private static void flushData() throws Exception { 12 | doWrite((int)(Math.random() * 4 * 1048576 + 1)); 13 | } 14 | 15 | private static void writer() throws Exception { 16 | flushData(); 17 | } 18 | 19 | public static void main(String[] args) throws Exception { 20 | Thread t = new Thread(new Runnable() { 21 | public void run() { 22 | while (true) { 23 | try { 24 | Thread.sleep(100); 25 | doWrite(1024); 26 | } catch (Exception e) { 27 | } 28 | } 29 | } 30 | }); 31 | t.start(); 32 | while (true) { 33 | Thread.sleep(1000); 34 | writer(); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /buggy/mysql-connector-java-5.1.40-bin.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goldshtn/linux-tracing-workshop/a3a34fe7242a16047f277f28c1e5c1edef29340d/buggy/mysql-connector-java-5.1.40-bin.jar -------------------------------------------------------------------------------- /buggy/proxy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Modified from dnslib: http://pydoc.net/Python/dnslib/0.9.3/dnslib.intercept/ 3 | 4 | from __future__ import print_function 5 | 6 | import binascii,socket,struct 7 | 8 | from dnslib import DNSRecord,RCODE 9 | from dnslib.server import DNSServer,DNSHandler,BaseResolver,DNSLogger 10 | 11 | class ProxyResolver(BaseResolver): 12 | """ 13 | Proxy resolver - passes all requests to upstream DNS server and 14 | returns response 15 | 16 | Note that the request/response will be each be decoded/re-encoded 17 | twice: 18 | 19 | a) Request packet received by DNSHandler and parsed into DNSRecord 20 | b) DNSRecord passed to ProxyResolver, serialised back into packet 21 | and sent to upstream DNS server 22 | c) Upstream DNS server returns response packet which is parsed into 23 | DNSRecord 24 | d) ProxyResolver returns DNSRecord to DNSHandler which re-serialises 25 | this into packet and returns to client 26 | 27 | In practice this is actually fairly useful for testing but for a 28 | 'real' transparent proxy option the DNSHandler logic needs to be 29 | modified (see PassthroughDNSHandler) 30 | 31 | """ 32 | 33 | def __init__(self,address,port,timeout=0): 34 | self.address = address 35 | self.port = port 36 | self.timeout = timeout 37 | 38 | def resolve(self,request,handler): 39 | time.sleep(1) # delay all responses by 1 second 40 | try: 41 | if handler.protocol == 'udp': 42 | proxy_r = request.send(self.address,self.port, 43 | timeout=self.timeout) 44 | else: 45 | proxy_r = request.send(self.address,self.port, 46 | tcp=True,timeout=self.timeout) 47 | reply = DNSRecord.parse(proxy_r) 48 | except socket.timeout: 49 | reply = request.reply() 50 | reply.header.rcode = getattr(RCODE,'NXDOMAIN') 51 | 52 | return reply 53 | 54 | class PassthroughDNSHandler(DNSHandler): 55 | """ 56 | Modify DNSHandler logic (get_reply method) to send directly to 57 | upstream DNS server rather then decoding/encoding packet and 58 | passing to Resolver (The request/response packets are still 59 | parsed and logged but this is not inline) 60 | """ 61 | def get_reply(self,data): 62 | host,port = self.server.resolver.address,self.server.resolver.port 63 | 64 | request = DNSRecord.parse(data) 65 | self.log_request(request) 66 | 67 | if self.protocol == 'tcp': 68 | data = struct.pack("!H",len(data)) + data 69 | response = send_tcp(data,host,port) 70 | response = response[2:] 71 | else: 72 | response = send_udp(data,host,port) 73 | 74 | reply = DNSRecord.parse(response) 75 | self.log_reply(reply) 76 | 77 | return response 78 | 79 | def send_tcp(data,host,port): 80 | """ 81 | Helper function to send/receive DNS TCP request 82 | (in/out packets will have prepended TCP length header) 83 | """ 84 | sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM) 85 | sock.connect((host,port)) 86 | sock.sendall(data) 87 | response = sock.recv(8192) 88 | length = struct.unpack("!H",bytes(response[:2]))[0] 89 | while len(response) - 2 < length: 90 | response += sock.recv(8192) 91 | sock.close() 92 | return response 93 | 94 | def send_udp(data,host,port): 95 | """ 96 | Helper function to send/receive DNS UDP request 97 | """ 98 | sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) 99 | sock.sendto(data,(host,port)) 100 | response,server = sock.recvfrom(8192) 101 | sock.close() 102 | return response 103 | 104 | if __name__ == '__main__': 105 | 106 | import argparse,sys,time 107 | 108 | p = argparse.ArgumentParser(description="DNS Proxy") 109 | p.add_argument("--port","-p",type=int,default=53, 110 | metavar="", 111 | help="Local proxy port (default:53)") 112 | p.add_argument("--address","-a",default="", 113 | metavar="
", 114 | help="Local proxy listen address (default:all)") 115 | p.add_argument("--upstream","-u",default="8.8.8.8:53", 116 | metavar="", 117 | help="Upstream DNS server:port (default:8.8.8.8:53)") 118 | p.add_argument("--tcp",action='store_true',default=False, 119 | help="TCP proxy (default: UDP only)") 120 | p.add_argument("--timeout","-o",type=float,default=5, 121 | metavar="", 122 | help="Upstream timeout (default: 5s)") 123 | p.add_argument("--passthrough",action='store_true',default=False, 124 | help="Dont decode/re-encode request/response (default: off)") 125 | p.add_argument("--log",default="request,reply,truncated,error", 126 | help="Log hooks to enable (default: +request,+reply,+truncated,+error,-recv,-send,-data)") 127 | p.add_argument("--log-prefix",action='store_true',default=False, 128 | help="Log prefix (timestamp/handler/resolver) (default: False)") 129 | args = p.parse_args() 130 | 131 | args.dns,_,args.dns_port = args.upstream.partition(':') 132 | args.dns_port = int(args.dns_port or 53) 133 | 134 | print("Starting Proxy Resolver (%s:%d -> %s:%d) [%s]" % ( 135 | args.address or "*",args.port, 136 | args.dns,args.dns_port, 137 | "UDP/TCP" if args.tcp else "UDP")) 138 | 139 | resolver = ProxyResolver(args.dns,args.dns_port,args.timeout) 140 | handler = PassthroughDNSHandler if args.passthrough else DNSHandler 141 | logger = DNSLogger(args.log,args.log_prefix) 142 | udp_server = DNSServer(resolver, 143 | port=args.port, 144 | address=args.address, 145 | logger=logger, 146 | handler=handler) 147 | udp_server.start_thread() 148 | 149 | if args.tcp: 150 | tcp_server = DNSServer(resolver, 151 | port=args.port, 152 | address=args.address, 153 | tcp=True, 154 | logger=logger, 155 | handler=handler) 156 | tcp_server.start_thread() 157 | 158 | while udp_server.isAlive(): 159 | time.sleep(1) 160 | 161 | -------------------------------------------------------------------------------- /buggy/setup-dns.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" | sudo tee /etc/resolv.conf 4 | -------------------------------------------------------------------------------- /create-db.sql: -------------------------------------------------------------------------------- 1 | drop database if exists acme; 2 | create database acme; 3 | use acme; 4 | 5 | create table users (id integer primary key, name varchar(200)); 6 | create table products (id integer primary key, userid integer, 7 | name varchar(200), description varchar(200), 8 | price double); 9 | 10 | delimiter // 11 | create procedure initialize() 12 | begin 13 | declare i int default 0; 14 | while i < 100 do 15 | insert into users values (i, "Dave"); 16 | insert into products values (2 * i, i, "Milk", "", 4.52); 17 | insert into products values (2 * i + 1, i, "Bread", "", 1.88); 18 | set i = i + 1; 19 | end while; 20 | end // 21 | delimiter ; 22 | 23 | call initialize(); 24 | drop procedure initialize; 25 | 26 | delimiter // 27 | create procedure getproduct(in pid integer) 28 | begin 29 | if pid = 97 then 30 | do sleep(2); 31 | end if; 32 | select * from products where id = pid; 33 | end // 34 | delimiter ; 35 | -------------------------------------------------------------------------------- /create-user.sql: -------------------------------------------------------------------------------- 1 | drop user if exists 'newuser'@'localhost'; 2 | create user 'newuser'@'localhost' identified by 'password'; 3 | grant all privileges on *.* to 'newuser'@'localhost' with grant option; 4 | -------------------------------------------------------------------------------- /data_access.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from random import randint 4 | import sys 5 | import mysql.connector 6 | 7 | NUM_EMPLOYEES = 10000 8 | 9 | def insert(): 10 | cursor = connection.cursor() 11 | try: 12 | cursor.execute("drop table employees") 13 | except: 14 | pass 15 | cursor.execute("create table employees (id integer primary key, name text)") 16 | cursor.close() 17 | 18 | print("Inserting employees...") 19 | for n in xrange(0, NUM_EMPLOYEES): 20 | cursor = connection.cursor() 21 | cursor.execute("insert into employees (id, name) values (%d, 'Employee_%d')" % 22 | (n, n)) 23 | connection.commit() 24 | cursor.close() 25 | 26 | def select(): 27 | print("Selecting employees...") 28 | while True: 29 | cursor = connection.cursor() 30 | cursor.execute("select * from employees where name like '%%%d'" % randint(0, NUM_EMPLOYEES)) 31 | for row in cursor: 32 | pass 33 | cursor.close() 34 | 35 | connection = mysql.connector.connect(host='localhost', database='test', user='root') 36 | 37 | if "insert" in sys.argv: 38 | while True: 39 | insert() 40 | elif "insert_once" in sys.argv: 41 | insert() 42 | elif "select" in sys.argv: 43 | select() 44 | else: 45 | print("USAGE: data_access.py ") 46 | 47 | connection.close() 48 | -------------------------------------------------------------------------------- /dbslower.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # dbslower Trace MySQL and PostgreSQL queries slower than a threshold. 4 | # 5 | # USAGE: dbslower {mysql,postgres} [threshold_ms] [-v] 6 | # 7 | # By default, a threshold of 1ms is used. Set the threshold to 0 to trace all 8 | # queries (verbose). 9 | # 10 | # This tool uses USDT probes, which means it needs MySQL and PostgreSQL built 11 | # with USDT (DTrace) support. 12 | # 13 | # Strongly inspired by Brendan Gregg's work on the mysqld_slower script. 14 | # 15 | # Licensed under the Apache License, Version 2.0 16 | # 17 | 18 | from bcc import BPF, USDT 19 | import argparse 20 | import ctypes as ct 21 | import subprocess 22 | 23 | examples = """ 24 | dbslower postgres # trace PostgreSQL queries slower than 1ms 25 | dbslower mysql 30 # trace MySQL queries slower than 30ms 26 | dbslower mysql -v # trace MySQL queries and print the BPF program 27 | """ 28 | parser = argparse.ArgumentParser( 29 | description="", 30 | formatter_class=argparse.RawDescriptionHelpFormatter, 31 | epilog=examples) 32 | parser.add_argument("-v", "--verbose", action="store_true", 33 | help="print the BPF program") 34 | parser.add_argument("db", choices=["mysql", "postgres"], 35 | help="the database engine to use") 36 | parser.add_argument("threshold", type=int, nargs="?", default=1, 37 | help="trace queries slower than this threshold (ms)") 38 | args = parser.parse_args() 39 | 40 | if args.db == "mysql": 41 | dbpid = int(subprocess.check_output("pidof mysqld".split())) 42 | elif args.db == "postgres": 43 | dbpid = int(subprocess.check_output("pgrep -n postgres".split())) 44 | 45 | threshold_ns = args.threshold * 1000000 46 | 47 | program = """ 48 | #include 49 | 50 | struct temp_t { 51 | u64 timestamp; 52 | char *query; 53 | }; 54 | 55 | struct data_t { 56 | u64 pid; 57 | u64 timestamp; 58 | u64 duration; 59 | char query[256]; 60 | }; 61 | 62 | BPF_HASH(temp, u64, struct temp_t); 63 | BPF_PERF_OUTPUT(events); 64 | 65 | int probe_start(struct pt_regs *ctx) { 66 | struct temp_t tmp = {}; 67 | tmp.timestamp = bpf_ktime_get_ns(); 68 | bpf_usdt_readarg(1, ctx, &tmp.query); 69 | u64 pid = bpf_get_current_pid_tgid(); 70 | temp.update(&pid, &tmp); 71 | return 0; 72 | } 73 | 74 | int probe_end(struct pt_regs *ctx) { 75 | struct temp_t *tempp; 76 | u64 pid = bpf_get_current_pid_tgid(); 77 | tempp = temp.lookup(&pid); 78 | if (!tempp) 79 | return 0; 80 | 81 | u64 delta = bpf_ktime_get_ns() - tempp->timestamp; 82 | if (delta >= """ + str(threshold_ns) + """) { 83 | struct data_t data = {}; 84 | data.pid = pid >> 32; // only process id 85 | data.timestamp = tempp->timestamp; 86 | data.duration = delta; 87 | bpf_probe_read(&data.query, sizeof(data.query), tempp->query); 88 | events.perf_submit(ctx, &data, sizeof(data)); 89 | } 90 | temp.delete(&pid); 91 | return 0; 92 | } 93 | """ 94 | 95 | usdt = USDT(pid=int(dbpid)) 96 | usdt.enable_probe("query__start", "probe_start") 97 | usdt.enable_probe("query__done", "probe_end") 98 | 99 | bpf = BPF(text=program, usdt_contexts=[usdt]) 100 | if args.verbose: 101 | print(usdt.get_text()) 102 | print(program) 103 | 104 | class Data(ct.Structure): 105 | _fields_ = [ 106 | ("pid", ct.c_ulonglong), 107 | ("timestamp", ct.c_ulonglong), 108 | ("delta", ct.c_ulonglong), 109 | ("query", ct.c_char * 256) 110 | ] 111 | 112 | start = 0 113 | def print_event(cpu, data, size): 114 | global start 115 | event = ct.cast(data, ct.POINTER(Data)).contents 116 | if start == 0: 117 | start = event.timestamp 118 | print("%-14.6f %-6d %8.3f %s" % (float(event.timestamp - start) / 1000000000, 119 | event.pid, float(event.delta) / 1000000, event.query)) 120 | 121 | print("Tracing database queries for PID %d slower than %d ms..." % 122 | (dbpid, args.threshold)) 123 | print("%-14s %-6s %8s %s" % ("TIME(s)", "PID", "MS", "QUERY")) 124 | 125 | bpf["events"].open_perf_buffer(print_event) 126 | while True: 127 | bpf.kprobe_poll() 128 | 129 | -------------------------------------------------------------------------------- /dbstat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # dbstat Display a histogram of MySQL and PostgreSQL query latencies. 4 | # 5 | # USAGE: dbstat {mysql,postgres} [-v] 6 | # queries (verbose). 7 | # 8 | # This tool uses USDT probes, which means it needs MySQL and PostgreSQL built 9 | # with USDT (DTrace) support. 10 | # 11 | # Licensed under the Apache License, Version 2.0 12 | # 13 | 14 | from bcc import BPF, USDT 15 | import argparse 16 | import subprocess 17 | from time import sleep 18 | 19 | examples = """ 20 | dbstat postgres # display a histogram of PostgreSQL query latencies 21 | dbstat mysql -v # display MySQL query latencies and print the BPF program 22 | dbstat mysql -m # display query latencies in milliseconds (default: us) 23 | dbstat mysql 5 # trace only queries slower than 5ms 24 | """ 25 | parser = argparse.ArgumentParser( 26 | description="", 27 | formatter_class=argparse.RawDescriptionHelpFormatter, 28 | epilog=examples) 29 | parser.add_argument("-v", "--verbose", action="store_true", 30 | help="print the BPF program") 31 | parser.add_argument("-m", "--milliseconds", action="store_true", 32 | help="display query latencies in milliseconds (default: microseconds)") 33 | parser.add_argument("db", choices=["mysql", "postgres"], 34 | help="the database engine to use") 35 | parser.add_argument("threshold", type=int, default=0, nargs='?', 36 | help="trace only queries slower than this threshold (default: 0)") 37 | args = parser.parse_args() 38 | 39 | if args.db == "mysql": 40 | dbpid = int(subprocess.check_output("pidof mysqld".split())) 41 | elif args.db == "postgres": 42 | dbpid = int(subprocess.check_output("pgrep -n postgres".split())) 43 | 44 | program = """ 45 | #include 46 | 47 | BPF_HASH(temp, u64, u64); 48 | BPF_HISTOGRAM(latency); 49 | 50 | int probe_start(struct pt_regs *ctx) { 51 | u64 timestamp = bpf_ktime_get_ns(); 52 | u64 pid = bpf_get_current_pid_tgid(); 53 | temp.update(&pid, ×tamp); 54 | return 0; 55 | } 56 | 57 | int probe_end(struct pt_regs *ctx) { 58 | u64 *timestampp; 59 | u64 pid = bpf_get_current_pid_tgid(); 60 | timestampp = temp.lookup(&pid); 61 | if (!timestampp) 62 | return 0; 63 | 64 | u64 delta = bpf_ktime_get_ns() - *timestampp; 65 | if (delta/1000000 < %d) 66 | return 0; 67 | 68 | delta /= %d; 69 | latency.increment(bpf_log2l(delta)); 70 | temp.delete(&pid); 71 | return 0; 72 | } 73 | """ % (args.threshold, 1000000 if args.milliseconds else 1000) 74 | 75 | usdt = USDT(pid=int(dbpid)) 76 | usdt.enable_probe("query__start", "probe_start") 77 | usdt.enable_probe("query__done", "probe_end") 78 | 79 | bpf = BPF(text=program, usdt_contexts=[usdt]) 80 | if args.verbose: 81 | print(usdt.get_text()) 82 | print(program) 83 | 84 | print("Tracing database queries slower than %dms for PID %d... Ctrl+C to quit." 85 | % (args.threshold, dbpid)) 86 | 87 | try: 88 | sleep(999999999999) 89 | except KeyboardInterrupt: 90 | pass 91 | 92 | bpf["latency"].print_log2_hist("query latency (%s)" % 93 | ("ms" if args.milliseconds else "us")) 94 | -------------------------------------------------------------------------------- /dotnet/Buggy.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.0 6 | ubuntu.16.10-x64 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /dotnet/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | ../dotnet publish 3 | chmod u+x bin/Debug/*/ubuntu*/publish/Buggy 4 | 5 | run: 6 | bin/Debug/*/ubuntu*/publish/Buggy $(command) 7 | -------------------------------------------------------------------------------- /dotnet/NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /dotnet/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Net.Http; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace Buggy 10 | { 11 | class Locker 12 | { 13 | public string Name; 14 | } 15 | 16 | class Program 17 | { 18 | static List _systemStateSendAttempts = new List(); 19 | 20 | static void ProcessResult(string result) 21 | { 22 | string s = ""; 23 | for (int i = 0; i < result.Length; ++i) 24 | s += i.ToString() + result[i]; 25 | } 26 | 27 | static void Fetch(string url) 28 | { 29 | var client = new HttpClient(); 30 | for (int i = 0; i < 1000; ++i) 31 | { 32 | if (i % 10 == 0) Console.Write("."); 33 | string result = client.GetStringAsync(url).Result; 34 | ProcessResult(result); 35 | } 36 | } 37 | 38 | static void LoadConfig(string configFile) 39 | { 40 | string config = File.ReadAllText(configFile); 41 | Console.WriteLine("Configuration loaded successfully."); 42 | } 43 | 44 | static void Initialize() 45 | { 46 | while (true) 47 | { 48 | Console.WriteLine("Opening configuration file..."); 49 | try { LoadConfig("/etc/buggy.conf"); return; } 50 | catch (Exception) { Thread.Sleep(1000); } 51 | } 52 | } 53 | 54 | static bool PushSystemState(string url) 55 | { 56 | var client = new HttpClient(); 57 | try 58 | { 59 | _systemStateSendAttempts.Add(Encoding.ASCII.GetString(new byte[1000000])); 60 | client.PostAsync(url, new FormUrlEncodedContent(new [] { 61 | new KeyValuePair("state", "OK") 62 | })).Wait(); 63 | } 64 | catch (Exception) { return false; } 65 | return true; 66 | } 67 | 68 | static void Push(string pushTargetUrl) 69 | { 70 | while (!PushSystemState(pushTargetUrl)) 71 | Thread.Sleep(500); 72 | } 73 | 74 | static void Reconfigure() 75 | { 76 | var txLocker = new Locker { Name = "Transaction" }; 77 | var configLocker = new Locker { Name = "Config" }; 78 | lock (txLocker) 79 | { 80 | Task.Run(() => { 81 | lock (configLocker) 82 | { 83 | lock (txLocker) 84 | { 85 | } 86 | } 87 | }); 88 | Thread.Sleep(1000); 89 | Console.WriteLine("Attempting configuration update..."); 90 | lock (configLocker) 91 | { 92 | } 93 | } 94 | } 95 | 96 | static void UpdateAsync() 97 | { 98 | if (new Random().Next() % 3 == 0) 99 | throw new InvalidOperationException("The update operation cannot be completed."); 100 | } 101 | 102 | static void HandleUnhandledExceptions(object sender, EventArgs e) 103 | { 104 | throw new ApplicationException("Task crashed!"); 105 | } 106 | 107 | static void Update() 108 | { 109 | TaskScheduler.UnobservedTaskException += HandleUnhandledExceptions; 110 | Task.Run(() => UpdateAsync()); 111 | } 112 | 113 | static void Main(string[] args) 114 | { 115 | switch (args[0]) 116 | { 117 | case "fetch": Fetch(args[1]); break; 118 | case "initialize": Initialize(); break; 119 | case "push": Push(args[1]); break; 120 | case "reconfigure": Reconfigure(); break; 121 | case "update": Update(); break; 122 | } 123 | Thread.Sleep(1000); 124 | GC.Collect(); 125 | GC.WaitForPendingFinalizers(); 126 | Thread.Sleep(1000); 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /ftrace.md: -------------------------------------------------------------------------------- 1 | ### Probing Tracepoints with ftrace 2 | 3 | In this lab, you will experiment with using ftrace directly (through debugfs) for enabling a kernel tracepoint and reviewing the trace results. You will also experiment with setting filters and with the general-purpose function tracer. 4 | 5 | - - - 6 | 7 | #### Task 1: Enable the `sched:sched_switch` Tracepoint 8 | 9 | If you suspect that some component on the system is burning CPU cycles by switching threads too frequently, you can figure out who is responsible by using the `sched:sched_switch` tracepoint. This tracepoint provides information about the process being switched out as well as the process being switched in. 10 | 11 | From a root shell, cd to the **/sys/kernel/debug/tracing** directory: 12 | 13 | ``` 14 | # cd /sys/kernel/debug/tracing 15 | ``` 16 | 17 | Enable tracing if it's not already enabled: 18 | 19 | ``` 20 | # echo 1 > tracing_on 21 | ``` 22 | 23 | Review the format of the `sched:sched_switch` tracepoint using the following command: 24 | 25 | ``` 26 | # cat events/sched/sched_switch/format 27 | ``` 28 | 29 | Now, enable the tracepoint: 30 | 31 | ``` 32 | # echo 1 > events/sched/sched_switch/enable 33 | ``` 34 | 35 | Wait a few seconds (optionally launching some processes that compete for CPU time), and then inspect the trace file: 36 | 37 | ``` 38 | # cat trace 39 | ``` 40 | 41 | Make sure you understand the format of the output. Specifically, which process is being switched out and which process is being switched in. It is fairly easy to build a tool that would post-process the trace output file and aggregate some statistics for you. Unfortunately, this kind of post-processing would have to happen in user-space, after the trace events have already been emitted. 42 | 43 | Finally, disable the tracepoint: 44 | 45 | ``` 46 | # echo 0 > events/sched/sched_switch/enable 47 | ``` 48 | 49 | #### Task 2: Enable the Function Tracer 50 | 51 | If you need to understand what's happening during the invocation of a particular kernel function, how often it's called, or how long it takes to run, you can use the `function` or `function_graph` tracers. These are restricted to kernel functions only, but can be very useful. 52 | 53 | Make sure you are still in the **/sys/kernel/debug/tracing** directory. Enable the function tracer: 54 | 55 | ``` 56 | # echo function > current_tracer 57 | ``` 58 | 59 | Make `vfs_write` the function you want traced: 60 | 61 | ``` 62 | # echo vfs_write > set_ftrace_filter 63 | ``` 64 | 65 | Inspect the trace file. There's not a lot of information for each invocation, so you might also care about what's going on inside the `vfs_write` function. That's what the `function_graph` tracer is used for: 66 | 67 | ``` 68 | # echo function_graph > current_tracer 69 | # echo > set_ftrace_filter 70 | # echo vfs_write > set_graph_function 71 | ``` 72 | 73 | Take a look at the trace file again. This time, each invocation of `vfs_write` is traced in a detailed format so you can understand exactly which other methods were invoked and how long they took. Again, some post-processing of this data might be in order to create a more accurate, visual illustration of the flow graph inside the function. If you want to restrict the depth of the graph, use: 74 | 75 | ``` 76 | # echo 2 > max_graph_depth 77 | ``` 78 | 79 | Finally, disable the tracer: 80 | 81 | ``` 82 | # echo nop > current_tracer 83 | # echo > set_graph_function 84 | ``` 85 | 86 | - - - 87 | 88 | #### Bonus: Setting User-Mode Probes with ftrace 89 | 90 | Figure out how to set user-mode probes (uprobes) with ftrace. Note that ftrace does not natively understand user-space function names; you will need to provide a binary and an offset. To discover these, you'll need `objdump` or a similar tool. 91 | 92 | Configure ftrace to trace all calls to the libc `write` function across all processes on the system. Try to add an additional filter that would keep only writes to file descriptor 1 (traditionally stdout). 93 | 94 | - - - 95 | -------------------------------------------------------------------------------- /heapsnapshot1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goldshtn/linux-tracing-workshop/a3a34fe7242a16047f277f28c1e5c1edef29340d/heapsnapshot1.png -------------------------------------------------------------------------------- /heapsnapshot2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goldshtn/linux-tracing-workshop/a3a34fe7242a16047f277f28c1e5c1edef29340d/heapsnapshot2.png -------------------------------------------------------------------------------- /lockstat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import itertools 5 | from time import sleep 6 | from bcc import BPF 7 | 8 | text = """ 9 | #include 10 | 11 | struct thread_mutex_key_t { 12 | u32 tid; 13 | u64 mtx; 14 | int lock_stack_id; 15 | }; 16 | 17 | struct thread_mutex_val_t { 18 | u64 wait_time_ns; 19 | u64 lock_time_ns; 20 | u64 enter_count; 21 | }; 22 | 23 | struct mutex_timestamp_t { 24 | u64 mtx; 25 | u64 timestamp; 26 | }; 27 | 28 | struct mutex_lock_time_key_t { 29 | u32 tid; 30 | u64 mtx; 31 | }; 32 | 33 | struct mutex_lock_time_val_t { 34 | u64 timestamp; 35 | int stack_id; 36 | }; 37 | 38 | // Mutex to the stack id which initialized that mutex 39 | BPF_HASH(init_stacks, u64, int); 40 | 41 | // Main info database about mutex and thread pairs 42 | BPF_HASH(locks, struct thread_mutex_key_t, struct thread_mutex_val_t); 43 | 44 | // Pid to the mutex address and timestamp of when the wait started 45 | BPF_HASH(lock_start, u32, struct mutex_timestamp_t); 46 | 47 | // Pid and mutex address to the timestamp of when the wait ended (mutex acquired) and the stack id 48 | BPF_HASH(lock_end, struct mutex_lock_time_key_t, struct mutex_lock_time_val_t); 49 | 50 | // Histogram of wait times 51 | BPF_HISTOGRAM(mutex_wait_hist, u64); 52 | 53 | // Histogram of hold times 54 | BPF_HISTOGRAM(mutex_lock_hist, u64); 55 | 56 | BPF_STACK_TRACE(stacks, 4096); 57 | 58 | int probe_mutex_lock(struct pt_regs *ctx) 59 | { 60 | u64 now = bpf_ktime_get_ns(); 61 | u32 pid = bpf_get_current_pid_tgid(); 62 | struct mutex_timestamp_t val = {}; 63 | val.mtx = PT_REGS_PARM1(ctx); 64 | val.timestamp = now; 65 | lock_start.update(&pid, &val); 66 | return 0; 67 | } 68 | 69 | int probe_mutex_lock_return(struct pt_regs *ctx) 70 | { 71 | u64 now = bpf_ktime_get_ns(); 72 | 73 | u32 pid = bpf_get_current_pid_tgid(); 74 | struct mutex_timestamp_t *entry = lock_start.lookup(&pid); 75 | if (entry == 0) 76 | return 0; // Missed the entry 77 | 78 | u64 wait_time = now - entry->timestamp; 79 | int stack_id = stacks.get_stackid(ctx, BPF_F_REUSE_STACKID|BPF_F_USER_STACK); 80 | 81 | // If pthread_mutex_lock() returned 0, we have the lock 82 | if (PT_REGS_RC(ctx) == 0) { 83 | // Record the lock acquisition timestamp so that we can read it when unlocking 84 | struct mutex_lock_time_key_t key = {}; 85 | key.mtx = entry->mtx; 86 | key.tid = pid; 87 | struct mutex_lock_time_val_t val = {}; 88 | val.timestamp = now; 89 | val.stack_id = stack_id; 90 | lock_end.update(&key, &val); 91 | } 92 | 93 | struct thread_mutex_key_t tm_key = {}; 94 | // TODO Update tm_key fields with the mutex, tid, and stack id 95 | // TODO Call locks.lookup_or_init(...) and update the wait time and the enter count 96 | // of the entry in the locks data structure 97 | 98 | u64 mtx_slot = bpf_log2l(wait_time / 1000); 99 | mutex_wait_hist.increment(mtx_slot); 100 | 101 | lock_start.delete(&pid); 102 | 103 | return 0; 104 | } 105 | 106 | int probe_mutex_unlock(struct pt_regs *ctx) 107 | { 108 | u64 now = bpf_ktime_get_ns(); 109 | u64 mtx = PT_REGS_PARM1(ctx); 110 | u32 pid = bpf_get_current_pid_tgid(); 111 | struct mutex_lock_time_key_t lock_key = {}; 112 | lock_key.mtx = mtx; 113 | lock_key.tid = pid; 114 | struct mutex_lock_time_val_t *lock_val = lock_end.lookup(&lock_key); 115 | if (lock_val == 0) 116 | return 0; // Missed the lock of this mutex 117 | 118 | u64 hold_time = now - lock_val->timestamp; 119 | 120 | struct thread_mutex_key_t tm_key = {}; 121 | tm_key.mtx = mtx; 122 | tm_key.tid = pid; 123 | tm_key.lock_stack_id = lock_val->stack_id; 124 | struct thread_mutex_val_t *existing_tm_val = locks.lookup(&tm_key); 125 | if (existing_tm_val == 0) 126 | return 0; // Couldn't find this record 127 | existing_tm_val->lock_time_ns += hold_time; 128 | 129 | // TODO Update the mutex_lock_hist histogram with the time we held the lock 130 | 131 | lock_end.delete(&lock_key); 132 | 133 | return 0; 134 | } 135 | 136 | int probe_mutex_init(struct pt_regs *ctx) 137 | { 138 | int stack_id = stacks.get_stackid(ctx, BPF_F_REUSE_STACKID|BPF_F_USER_STACK); 139 | u64 mutex_addr = PT_REGS_PARM1(ctx); 140 | init_stacks.update(&mutex_addr, &stack_id); 141 | return 0; 142 | } 143 | """ 144 | 145 | def attach(bpf, pid): 146 | bpf.attach_uprobe(name="pthread", sym="pthread_mutex_init", fn_name="probe_mutex_init", pid=pid) 147 | # TODO Similarly to the previous probe, attach the following probes: 148 | # uprobe in pthread_mutex_lock handled by probe_mutex_lock 149 | # uretprobe in pthread_mutex_lock handled by probe_mutex_lock_return 150 | # uprobe in pthread_mutex_unlock handled by probe_mutex_unlock 151 | 152 | def print_frame(bpf, pid, addr): 153 | print("\t\t%16s (%x)" % (bpf.sym(addr, pid, show_module=True, show_offset=True), addr)) 154 | 155 | def print_stack(bpf, pid, stacks, stack_id): 156 | for addr in stacks.walk(stack_id): 157 | print_frame(bpf, pid, addr) 158 | 159 | def run(pid): 160 | bpf = BPF(text=text) 161 | attach(bpf, pid) 162 | init_stacks = bpf["init_stacks"] 163 | stacks = bpf["stacks"] 164 | locks = bpf["locks"] 165 | mutex_lock_hist = bpf["mutex_lock_hist"] 166 | mutex_wait_hist = bpf["mutex_wait_hist"] 167 | while True: 168 | sleep(5) 169 | mutex_ids = {} 170 | next_mutex_id = 1 171 | for k, v in init_stacks.items(): 172 | mutex_id = "#%d" % next_mutex_id 173 | next_mutex_id += 1 174 | mutex_ids[k.value] = mutex_id 175 | print("init stack for mutex %x (%s)" % (k.value, mutex_id)) 176 | print_stack(bpf, pid, stacks, v.value) 177 | print("") 178 | grouper = lambda (k, v): k.tid 179 | sorted_by_thread = sorted(locks.items(), key=grouper) 180 | locks_by_thread = itertools.groupby(sorted_by_thread, grouper) 181 | for tid, items in locks_by_thread: 182 | print("thread %d" % tid) 183 | for k, v in sorted(items, key=lambda (k, v): -v.wait_time_ns): 184 | mutex_descr = mutex_ids[k.mtx] if k.mtx in mutex_ids else bpf.sy(k.mtx, pid) 185 | # TODO Print a nicely formatted line with the mutex description, wait time, 186 | # hold time, enter count, and stack (use print_stack) 187 | print("") 188 | mutex_wait_hist.print_log2_hist(val_type="wait time (us)") 189 | mutex_lock_hist.print_log2_hist(val_type="hold time (us)") 190 | 191 | if __name__ == "__main__": 192 | if len(sys.argv) < 2: 193 | print("USAGE: %s pid" % sys.argv[0]) 194 | else: 195 | run(int(sys.argv[1])) 196 | -------------------------------------------------------------------------------- /logger.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define LARGE_WRITE 1048576 11 | #define SMALL_WRITE 1024 12 | 13 | void do_write(int fd, char *buf, int count) { 14 | write(fd, buf, count); 15 | } 16 | 17 | void write_flushed_data() { 18 | int fd; 19 | char *buf; 20 | buf = malloc(LARGE_WRITE); 21 | strcpy(buf, "Large data buffer follows.\n"); 22 | fd = open("flush.data", O_CREAT | O_WRONLY | O_SYNC, 0644); 23 | do_write(fd, buf, LARGE_WRITE); 24 | close(fd); 25 | free(buf); 26 | } 27 | 28 | void *flusher(void *ignore) { 29 | while (1) { 30 | usleep(200000); 31 | write_flushed_data(); 32 | } 33 | return NULL; 34 | } 35 | 36 | void logger() { 37 | int fd; 38 | char *buf; 39 | buf = malloc(SMALL_WRITE); 40 | strcpy(buf, "[*] INFO Writing a log message.\n"); 41 | fd = open("log.data", O_CREAT | O_WRONLY | O_SYNC, 0644); 42 | while (1) { 43 | usleep(20000); 44 | do_write(fd, buf, SMALL_WRITE); 45 | } 46 | close(fd); 47 | free(buf); 48 | } 49 | 50 | int main() { 51 | pthread_t flusher_thr; 52 | pthread_create(&flusher_thr, NULL, flusher, NULL); 53 | logger(); 54 | pthread_join(flusher_thr, NULL); 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /matexp.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | template 8 | class matrix { 9 | private: 10 | std::vector data_; 11 | int rows_, cols_; 12 | public: 13 | matrix(int rows, int cols) : rows_(rows), cols_(cols) { 14 | data_.resize(rows * cols); 15 | } 16 | T &operator()(int i, int j) { 17 | return data_[i*cols_ + j]; 18 | } 19 | T const &operator()(int i, int j) const { 20 | return data_[i*cols_ + j]; 21 | } 22 | int height() const { return rows_; } 23 | int width() const { return cols_; } 24 | matrix operator+(matrix const &other) const { 25 | if (height() != other.height() || width() != other.width()) 26 | throw std::invalid_argument("matrix dimensions don't match"); 27 | auto result = *this; 28 | for (int i = 0; i < rows_; ++i) 29 | for (int j = 0; j < cols_; ++j) 30 | result(i, j) += other(i, j); 31 | return result; 32 | } 33 | matrix operator*(matrix const &other) const { 34 | if (height() != width()) 35 | throw std::invalid_argument("multiplication implemented only " 36 | "for square matrices"); 37 | if (height() != other.height() || width() != other.width()) 38 | throw std::invalid_argument("matrix dimensions don't match"); 39 | matrix result(rows_, cols_); 40 | for (int i = 0; i < rows_; ++i) 41 | for (int j = 0; j < cols_; ++j) 42 | for (int k = 0; k < cols_; ++k) 43 | result(i, j) += (*this)(i, k) * other(k, j); 44 | return result; 45 | } 46 | }; 47 | 48 | template 49 | class exponentiator { 50 | private: 51 | matrix mat_; 52 | int exponent_; 53 | public: 54 | exponentiator(matrix mat, int exp) : 55 | mat_(std::move(mat)), exponent_(exp) { 56 | } 57 | matrix operator()() { 58 | auto a = mat_; 59 | for (int i = 1; i < exponent_; ++i) { 60 | a = a * mat_; 61 | } 62 | return a; 63 | } 64 | }; 65 | 66 | class io { 67 | std::string infile_; 68 | 69 | matrix read() { 70 | std::ifstream in(infile_.c_str()); 71 | if (!in) 72 | throw std::invalid_argument("unable to open input file"); 73 | int rows, cols; 74 | in >> rows; 75 | in >> cols; 76 | 77 | matrix res{ rows, cols }; 78 | for (int i = 0; in; ++i) { 79 | in >> res(i/cols, i%cols); 80 | } 81 | return res; 82 | } 83 | void write(matrix const &mat, std::ofstream &out) { 84 | out << mat.height() << " " << mat.width(); 85 | for (int i = 0; i < mat.height(); ++i) { 86 | for (int j = 0; j < mat.width(); ++j) 87 | out << mat(i, j) << " "; 88 | out << '\n'; 89 | } 90 | } 91 | public: 92 | io(std::string filename) : infile_(filename.c_str()) { 93 | } 94 | void exp_to(std::string filename, int rounds) { 95 | std::ofstream out(filename.c_str()); 96 | auto mat = read(); 97 | exponentiator exp(mat, rounds); 98 | write(exp(), out); 99 | } 100 | }; 101 | 102 | int usage() { 103 | std::cerr << "usage: matexp \n"; 104 | return 1; 105 | } 106 | 107 | int main(int argc, char *argv[]) { 108 | if (argc != 4) 109 | return usage(); 110 | 111 | io prog(argv[1]); 112 | prog.exp_to(argv[3], std::atoi(argv[2])); 113 | 114 | return 0; 115 | } 116 | -------------------------------------------------------------------------------- /mysql-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ps aux | grep mysqld | grep -vq grep 4 | if [ $? -eq 1 ]; then 5 | sudo -u mysql /usr/local/mysql/bin/mysqld_safe --user=mysql & 6 | fi 7 | exit 8 | 9 | sudo /usr/local/mysql/bin/mysql --user=root < ./create-user.sql 10 | /usr/local/mysql/bin/mysql --user=newuser --password=password < ./create-db.sql 11 | -------------------------------------------------------------------------------- /mysqlsniff.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # mysqlsniff MySQL client command sniffer tool. 4 | # This tool is experimental and designed for teaching purposes 5 | # only; it is neither tested nor suitable for production work. 6 | # 7 | # Copyright (C) Sasha Goldshtein, 2017 8 | 9 | import argparse 10 | import ctypes as ct 11 | import sys 12 | from bcc import BPF 13 | 14 | parser = argparse.ArgumentParser(description="MySQL client command sniffer " + 15 | "tool. Probes the network send layer and attempts to detect MySQL " + 16 | "commands.") 17 | parser.add_argument("-p", "--pid", type=int, required=True, 18 | help="the client process id") 19 | parser.add_argument("-f", "--filter", type=str, default="", 20 | help="the query prefix to search for in commands") 21 | parser.add_argument("-S", "--stack", action="store_true", 22 | help="capture a stack trace for each command") 23 | parser.add_argument("-d", "--debug", action="store_true", 24 | help="show raw message contents for debugging purposes") 25 | parser.add_argument("-l", "--library", default="pthread", 26 | help="the library to probe (default: pthread)") 27 | parser.add_argument("-a", "--api", default="__libc_send", 28 | help="the API to probe (default: __libc_send)") 29 | args = parser.parse_args() 30 | 31 | text = """ 32 | #include 33 | #include 34 | 35 | struct data_t { 36 | char data[128]; 37 | int len; 38 | #ifdef NEED_STACK 39 | int stackid; 40 | #endif 41 | }; 42 | 43 | BPF_PERF_OUTPUT(events); 44 | BPF_STACK_TRACE(stacks, 1024); 45 | 46 | static inline bool starts_with(char *query) { 47 | char needle[] = "%s";""" % args.filter + """ 48 | #pragma unroll 49 | for (int i = 0; i < sizeof(needle)-1; ++i) { 50 | if (query[i] != needle[i]) 51 | return false; 52 | } 53 | return true; 54 | } 55 | 56 | int probe(struct pt_regs *ctx, int fd, const void *buf, size_t len) { 57 | struct data_t data = {0}; 58 | bpf_probe_read(&data.data[0], sizeof(data.data), (void *)buf); 59 | 60 | int cmd_len = *(int *)&data.data[0]; 61 | u8 cmd = data.data[4]; 62 | if (cmd != 3) 63 | return 0; // not a query 64 | 65 | if (!starts_with(data.data + 5)) 66 | return false; 67 | 68 | data.len = cmd_len + 5; 69 | #ifdef NEED_STACK 70 | data.stackid = stacks.get_stackid(ctx, 71 | BPF_F_REUSE_STACKID | BPF_F_USER_STACK); 72 | #endif 73 | events.perf_submit(ctx, &data, sizeof(data)); 74 | 75 | return 0; 76 | } 77 | """ 78 | 79 | if args.stack: 80 | text = "#define NEED_STACK\n" + text 81 | 82 | class Data(ct.Structure): 83 | _fields_ = [ 84 | ("data", ct.c_ubyte * 128), 85 | ("len", ct.c_int), 86 | ("stackid", ct.c_int) 87 | ] 88 | 89 | def fmt_char(c): 90 | return unichr(c) if c > 20 and c < 128 else '.' 91 | 92 | bpf = BPF(text=text) 93 | # This is by far the most brittle part of the script. We need to attach 94 | # The MySQL Java client uses __libc_send in libpthread, while the Node.js 95 | # MySQL module uses __write in libpthread. In other scenarios, this will 96 | # likely need to change. The user can control these values using cmdline args. 97 | bpf.attach_uprobe(name=args.library, sym=args.api, 98 | fn_name="probe", pid=args.pid) 99 | def print_event(cpu, data, size): 100 | event = ct.cast(data, ct.POINTER(Data)).contents 101 | data = event.data[:event.len] 102 | if args.debug: 103 | print(''.join(fmt_char(x) for x in data)) 104 | print(' '.join('{:02x}'.format(x) for x in data)) 105 | else: 106 | print(''.join(fmt_char(x) for x in data[5:-1])) 107 | if args.stack: 108 | for addr in stacks.walk(event.stackid): 109 | print("\t%s" % bpf.sym(addr, args.pid, show_offset=True)) 110 | print("") 111 | 112 | bpf["events"].open_perf_buffer(print_event) 113 | if args.stack: 114 | stacks = bpf["stacks"] 115 | print("Sniffing process %d, Ctrl+C to quit." % args.pid) 116 | while 1: 117 | bpf.kprobe_poll() 118 | -------------------------------------------------------------------------------- /netsend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Example BPF program for aggregating outgoing network interface traffic 4 | # using the net:net_dev_* kernel tracepoints and displaying a histogram. 5 | # 6 | # Copyright 2016 Sasha Goldshtein 7 | 8 | from time import sleep 9 | from bcc import BPF 10 | 11 | text = """ 12 | #include 13 | 14 | struct key_t { 15 | char devname[16]; // IFNAMSIZ 16 | u64 slot; 17 | }; 18 | 19 | BPF_HISTOGRAM(dist, struct key_t); 20 | 21 | TRACEPOINT_PROBE(net, net_dev_start_xmit) { 22 | struct key_t key = {0}; 23 | 24 | TP_DATA_LOC_READ_CONST(&key.devname, name, 16); 25 | key.slot = bpf_log2l(args->len); 26 | dist.increment(key); 27 | 28 | return 0; 29 | } 30 | """ 31 | 32 | bpf = BPF(text=text) 33 | dist = bpf["dist"] 34 | while True: 35 | sleep(1) 36 | dist.print_log2_hist("bytes", "device name") 37 | -------------------------------------------------------------------------------- /nhttpslower.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # nhttpslower Snoops and prints Node.js HTTP requests slower than a threshold. 4 | # This tool is experimental and designed for teaching purposes 5 | # only; it is neither tested nor suitable for production work. 6 | # 7 | # NOTE: Node http__client* probes are not accurate in that they do not report 8 | # the fd, remote, and port. This makes it impossible to correlate request 9 | # and response probes, and a result the tool does not work correctly if 10 | # there are multiple client requests in flight in parallel. 11 | # 12 | # Copyright (C) Sasha Goldshtein, 2017 13 | 14 | import argparse 15 | import ctypes as ct 16 | import sys 17 | from bcc import BPF, USDT, DEBUG_PREPROCESSOR 18 | 19 | # TODO This can also be done by using uprobes -- explore what it would take 20 | # TODO Filter by HTTP method, or by URL prefix/substring 21 | 22 | parser = argparse.ArgumentParser(description="Snoops and prints Node.js HTTP " + 23 | "requests and responses (client and server) slower than a threshold. " + 24 | "Requires Node.js built with --enable-dtrace (confirm by using tplist).") 25 | parser.add_argument("pid", type=int, help="the Node.js process id") 26 | parser.add_argument("threshold", type=int, nargs="?", default=0, 27 | help="print only requests slower than this threshold (ms)") 28 | parser.add_argument("-S", "--stack", action="store_true", 29 | help="capture a stack trace for each event") 30 | parser.add_argument("--client", action="store_true", 31 | help="client events only (outgoing requests)") 32 | parser.add_argument("--server", action="store_true", 33 | help="server events only (incoming requests)") 34 | parser.add_argument("-d", "--debug", action="store_true", 35 | help="print the BPF program (for debugging purposes)") 36 | args = parser.parse_args() 37 | 38 | text = """ 39 | #include 40 | 41 | struct val_t { 42 | u64 start_ns; 43 | u64 duration_ns; 44 | int type; // CLIENT, SERVER 45 | #ifdef NEED_STACK 46 | int stackid; 47 | #endif 48 | char method[16]; 49 | char url[128]; 50 | }; 51 | 52 | struct data_t { 53 | u64 start_ns; 54 | u64 duration_ns; 55 | int port; 56 | int type; 57 | int stackid; 58 | char method[16]; 59 | char url[128]; 60 | char remote[128]; 61 | }; 62 | 63 | struct key_t { 64 | // TODO nhttpsnoop correlates by remote and port, while we use fd and port 65 | int fd; 66 | int port; 67 | }; 68 | 69 | #define CLIENT 0 70 | #define SERVER 1 71 | 72 | BPF_PERF_OUTPUT(events); 73 | BPF_STACK_TRACE(stacks, 1024); 74 | BPF_HASH(starts, struct key_t, struct val_t); 75 | 76 | #define THRESHOLD """ + str(args.threshold*1000000) + "\n" 77 | 78 | probe = """ 79 | int PROBE_start(struct pt_regs *ctx) { 80 | struct val_t val = {0}; 81 | struct key_t key = {0}; 82 | char *str; 83 | 84 | bpf_usdt_readarg(7, ctx, &key.fd); 85 | bpf_usdt_readarg(4, ctx, &key.port); 86 | bpf_usdt_readarg(5, ctx, &str); 87 | bpf_probe_read(val.method, sizeof(val.method), str); 88 | bpf_usdt_readarg(6, ctx, &str); 89 | bpf_probe_read(val.url, sizeof(val.url), str); 90 | 91 | val.start_ns = bpf_ktime_get_ns(); 92 | val.type = TYPE; 93 | #ifdef NEED_STACK 94 | val.stackid = stacks.get_stackid(ctx, 95 | BPF_F_REUSE_STACKID | BPF_F_USER_STACK); 96 | #endif 97 | 98 | starts.update(&key, &val); 99 | return 0; 100 | } 101 | 102 | int PROBE_end(struct pt_regs *ctx) { 103 | struct key_t key = {0}; 104 | struct val_t *valp; 105 | struct data_t data = {0}; 106 | u64 duration; 107 | char *remote; 108 | 109 | bpf_usdt_readarg(4, ctx, &key.fd); 110 | bpf_usdt_readarg(3, ctx, &key.port); 111 | 112 | valp = starts.lookup(&key); 113 | if (!valp) 114 | return 0; // Missed the start event for this request 115 | 116 | duration = bpf_ktime_get_ns() - valp->start_ns; 117 | if (duration < THRESHOLD) 118 | goto EXIT; 119 | 120 | data.start_ns = valp->start_ns; 121 | data.duration_ns = duration; 122 | data.port = key.port; 123 | data.type = valp->type; 124 | #ifdef NEED_STACK 125 | data.stackid = valp->stackid; 126 | #endif 127 | __builtin_memcpy(data.method, valp->method, sizeof(data.method)); 128 | __builtin_memcpy(data.url, valp->url, sizeof(data.url)); 129 | 130 | bpf_usdt_readarg(2, ctx, &remote); 131 | bpf_probe_read(&data.remote, sizeof(data.remote), remote); 132 | 133 | events.perf_submit(ctx, &data, sizeof(data)); 134 | 135 | EXIT: 136 | starts.delete(&key); 137 | return 0; 138 | } 139 | """ 140 | 141 | if not args.server: 142 | text += probe.replace("PROBE", "client").replace("TYPE", "CLIENT") 143 | if not args.client: 144 | text += probe.replace("PROBE", "server").replace("TYPE", "SERVER") 145 | 146 | CLIENT = 0 147 | SERVER = 1 148 | 149 | if args.stack: 150 | text = "#define NEED_STACK\n" + text 151 | 152 | usdt = USDT(pid=args.pid) 153 | if not args.server: 154 | usdt.enable_probe("http__client__request", "client_start") 155 | usdt.enable_probe("http__client__response", "client_end") 156 | if not args.client: 157 | usdt.enable_probe("http__server__request", "server_start") 158 | usdt.enable_probe("http__server__response", "server_end") 159 | bpf = BPF(text=text, usdt_contexts=[usdt], 160 | debug=DEBUG_PREPROCESSOR if args.debug else 0) 161 | 162 | class Data(ct.Structure): 163 | _fields_ = [ 164 | ("start_ns", ct.c_ulong), 165 | ("duration_ns", ct.c_ulong), 166 | ("port", ct.c_int), 167 | ("type", ct.c_int), 168 | ("stackid", ct.c_int), 169 | ("method", ct.c_char * 16), 170 | ("url", ct.c_char * 128), 171 | ("remote", ct.c_char * 128) 172 | ] 173 | 174 | delta = 0 175 | prev_ts = 0 176 | 177 | def print_event(cpu, data, size): 178 | global delta, prev_ts 179 | event = ct.cast(data, ct.POINTER(Data)).contents 180 | typ = "CLI" if event.type == CLIENT else "SVR" 181 | if prev_ts != 0: 182 | delta += event.start_ns - prev_ts 183 | prev_ts = event.start_ns 184 | print("%-14.9f %3s %11.3f %-16s %-5d %-8s %s" % 185 | (delta/1e9, typ, event.duration_ns/1e6, event.remote, 186 | event.port, event.method, event.url)) 187 | if args.stack: 188 | for addr in stacks.walk(event.stackid): 189 | print("\t%s" % bpf.sym(addr, args.pid, show_offset=True)) 190 | print("") 191 | 192 | bpf["events"].open_perf_buffer(print_event) 193 | if args.stack: 194 | stacks = bpf["stacks"] 195 | print("Snooping HTTP requests in Node process %d, Ctrl+C to quit." % args.pid) 196 | print("%-14s %3s %-11s %-16s %-5s %-8s %s" % 197 | ("TIME_s", "TYP", "DURATION_ms", "REMOTE", "PORT", "METHOD", "URL")) 198 | while 1: 199 | bpf.kprobe_poll() 200 | -------------------------------------------------------------------------------- /node-coredump.md: -------------------------------------------------------------------------------- 1 | ### Node Core Dump Analysis with `llnode` 2 | 3 | In this lab, you will experiment with analyzing a core dump of a crashing Node application. Core dump analysis of Node processes used to be a tedious and time-consuming tasks, relying on mdb; this is no longer the case with the introduction of [`llnode`](https://github.com/nodejs/llnode), an LLDB plugin that supports Node core dump analysis. 4 | 5 | - - - 6 | 7 | #### Task 1: Run the Crashing Application 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start our simple Node application: 10 | 11 | ``` 12 | $ ./run.sh 13 | ``` 14 | 15 | The process exposes an HTTP PUT endpoint called `/stats`, which expects an authentication key. Try to access the service using the following command, which uses the valid (hard-coded) key: 16 | 17 | ``` 18 | $ curl -X PUT -H 'Content-Type: application/json' -d '{ "auth": "mykey" }' http://localhost:3000/stats 19 | ``` 20 | 21 | You should get a 201 (Created) response back. But what if you provide the wrong key? This is something that would probably not happen most of the time, but in a mis-configured system or when the service is being probed from the Internet, this might happen; run the following command to see what happens then: 22 | 23 | ``` 24 | $ curl -X PUT -H 'Content-Type: application/json' -d '{ "auth": "mykey123" }' http://localhost:3000/stats 25 | ``` 26 | 27 | It seems that the Node process crashes in flames! This isn't good, and the way things are configured right now, it simply disappears without a trace. 28 | 29 | - - - 30 | 31 | #### Task 2: Capture a Core Dump of the Crash 32 | 33 | To diagnose the problem, run the following commands, which set up core dump generation and remove the core file size limit: 34 | 35 | ``` 36 | $ ulimit -c unlimited 37 | $ sudo bash -c 'echo core > /proc/sys/kernel/core_pattern' 38 | ``` 39 | 40 | Additionally, the Node process needs to be started with the `--abort-on-uncaught-exception` flag, so that an unhandled exception ends up crashing the process and generating the core dump. Run the following command to enable it using the wrapper script: 41 | 42 | ``` 43 | $ ./run.sh core 44 | ``` 45 | 46 | And now reproduce the crash by accessing the `/stats` endpoint again with an invalid key: 47 | 48 | ``` 49 | $ curl -X PUT -H 'Content-Type: application/json' -d '{ "auth": "mykey123" }' http://localhost:3000/stats 50 | ``` 51 | 52 | This time, when the process crashes, a core file should be generated in the current directory. This is the file we will now analyze. 53 | 54 | - - - 55 | 56 | #### Task 3: Analyze the Crash in LLDB 57 | 58 | Now that we have the core dump, we can open it in LLDB and use the `llnode` plugin. The `llnode` plugin can be installed separately using `npm install -g llnode`; in the instructor-provided VM, it will be in `~/tracing-workshop/node_modules/.bin/llnode`. You can run it as follows, to provide the core file to load: 59 | 60 | ``` 61 | $ ~/tracing-workshop/node_modules/.bin/llnode -c core.1234 62 | ``` 63 | 64 | When LLDB initializes, it will print basic information about the crash. Try running the `bt` command, which displays a stack backtrace. You can clearly see the native Node functions, but not the JavaScript frames -- these are not resolved, because LLDB doesn't have built-in support for them. Run the following command to see the actual stack with JavaScript frames resolved: 65 | 66 | ``` 67 | (lldb) v8 bt 68 | ``` 69 | 70 | This points clearly to the [index.js](nodey/routes/index.js) file as the culprit, showing an anonymous function that is getting invoked as part of a timer. The two anonymous frames in index.js have `fn=0xnnnn...` tacked to the end; you can pass these addresses to the `v8 inspect` command to see more information about these anonymous functions. This should lead you to the source of the crash; make sure you can see the `key` parameter passed to the authentication function, which you'd be 71 | able to use to figure out the wrong key provided by the client (if you hadn't run the client yourself just a few minutes ago). 72 | 73 | - - - 74 | 75 | #### Bonus 76 | 77 | The [`core-dump`](https://github.com/davidmarkclements/core-dump) Node module can be used for somewhat more advanced core dump generation. For example, you can use it to generate core dumps programmatically from within the Node process (e.g. when an interesting condition is met), or you can use it to generate a core dump externally from a shell. (The latter can also be done with `gcore`; see the [Node Memory Leak Analysis exercise](node-memleak.md).) 78 | 79 | - - - 80 | -------------------------------------------------------------------------------- /node-prof.md: -------------------------------------------------------------------------------- 1 | ### Node Profiling with V8 2 | 3 | In this lab, you will experiment with profiling a CPU bottleneck in a Node.js application using the built-in V8 profiler. The key advantage of using the built-in profiler is that it is, well, built-in: you don't need any additional software, and it works across all platforms. On the other hand, it requires that you restart your Node application with a special profiling switch, which you can't leave on for a long duration of time. 4 | 5 | - - - 6 | 7 | #### Task 1: Run the Application with a Benchmark 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start our simple Node application: 10 | 11 | ``` 12 | $ ./run.sh 13 | ``` 14 | 15 | This starts the web server asynchronously. One of its endpoints is a simple authentication service, which takes a username and password and returns user information. You can try it with curl: 16 | 17 | ``` 18 | $ curl -X POST 'http://localhost:3000/users/auth?username=foo&password=wrong' 19 | ``` 20 | 21 | Let's ignore for a second the horrible practice of passing the username and password as part of the query string, and focus on the performance of this endpoint. Run the following command to get a simple local benchmark of its behavior: 22 | 23 | ``` 24 | $ ab -c 10 -n 100 -m POST 'http://localhost:3000/users/auth?username=foo&password=wrong' 25 | ``` 26 | 27 | In another shell, you can run `top` to get a quick idea of the system's behavior. You can clearly see the Node process consuming lots of CPU. The benchmark results are not very optimistic, either; the mean time to service an authentication request is quite high. 28 | 29 | - - - 30 | 31 | #### Task 2: Run the Application with the V8 Profiler 32 | 33 | To profile the application, we will use the built-in V8 profiler. The wrapper script, [run.sh](nodey/run.sh), can invoke the Node runtime with the `--prof` switch, which turns on the profiler. The profiler output goes to a file named `isolate-nnnnn.log`, which can then be parsed and analyzed. 34 | 35 | Run the application with the profiler enabled: 36 | 37 | ``` 38 | $ ./run.sh prof 39 | ``` 40 | 41 | Run the benchmark again: 42 | 43 | ``` 44 | $ ab -c 10 -n 100 -m POST 'http://localhost:3000/users/auth?username=foo&password=wrong' 45 | ``` 46 | 47 | Now, look in the current directory for the `isolate-nnnnn.log` file. You don't have to parse it by hand; you can ask the Node runtime to parse it and produce a text report: 48 | 49 | 50 | ``` 51 | $ node --prof-process isolate*.log | less 52 | ``` 53 | 54 | Inspect the report and try to answer the following questions: 55 | 56 | * Which JavaScript method was responsible for most of the CPU usage? 57 | * What's the proportion of time spent in JavaScript code, C++ code, or GC code? 58 | * Which call stack leads to the most CPU-consuming function? 59 | 60 | - - - 61 | 62 | -------------------------------------------------------------------------------- /node-slowdns.md: -------------------------------------------------------------------------------- 1 | ### Node Slow DNS 2 | 3 | In this lab, you will experiment with a Node service that talks to external HTTP endpoints (think additional microservices) and occasionally returns a slow result. This time, we will focus on investigating the DNS queries performed by the application, and see what it looks like when the DNS server is slow to respond, or fails to produce a result. 4 | 5 | - - - 6 | 7 | #### Task 1: Run the Application 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start our simple Node application: 10 | 11 | ``` 12 | $ ./run.sh dns 13 | ``` 14 | 15 | Run a couple of queries to the Inventory endpoint by using the following commands. Note the time it takes to return a response from the service. 16 | 17 | ``` 18 | $ time curl http://localhost:3000/inventory?product_id=a19224 19 | $ time curl http://localhost:3000/inventory?product_id=a19877 20 | $ time curl http://localhost:3000/inventory?product_id=a88711 21 | ``` 22 | 23 | The service seems to working quite slowly, and the same products that worked all right in the [HTTP exercise](node-slowhttp.md) are now fetching very slowly. Add a different request into the mix: a request for a "dynamic" product, where the inventory service endpoint is generated dynamically: 24 | 25 | ``` 26 | $ time curl http://localhost:3000/inventory?product_id=dyn88711 27 | ``` 28 | 29 | The result is even slower to arrive, and there's also a generic error that shows up in the response document. 30 | 31 | - - - 32 | 33 | #### Task 2: Inspect DNS Resolution Requests 34 | 35 | In some cases, a DNS mis-configuration can be responsible for very poor HTTP request latency. The `gethostlatency` tool from [BCC](https://github.com/iovisor/bcc) is designed to inspect DNS resolution requests and their latencies. Run it as follows, from a root shell: 36 | 37 | ``` 38 | # gethostlatency 39 | ``` 40 | 41 | If you repeat one of the accesses to the Inventory endpoint, you'll see the application makes multiple DNS requests to resolve its backend service hostnames. Node.js doesn't cache DNS responses (by design), which means we keep issuing these DNS resolution queries on each HTTP request. It still doesn't explain why the DNS resolution is slow, but we can confirm that the DNS resolution time is a considerable portion of the overall slowdown our HTTP queries are experiencing. 42 | 43 | To understand the delays in processing our DNS query, it helps to take a look at the DNS servers configured on the machine: 44 | 45 | ``` 46 | $ cat /etc/resolv.conf 47 | ``` 48 | 49 | Looks like there is a local DNS server, which takes precedence to everything else. Now we can use the `dig` tool to query that specific DNS server and see its behavior compared to a public Internet nameserver (`dig` is in the dnsutils package on Ubuntu, or the bind-utils package on Fedora): 50 | 51 | ``` 52 | $ dig google.com @127.0.0.1 53 | $ dig google.com @8.8.8.8 54 | ``` 55 | 56 | The local query is very slow! So it looks like we have a rogue local DNS server on our hands. Indeed, the [`slodns`](https://github.com/goldshtn/slodns) tool is running in the background and slowing down our DNS queries. Kill it with the following command: 57 | 58 | ``` 59 | $ ./run.sh killdns 60 | ``` 61 | 62 | Now retry the Inventory endpoint accesses. They should be much faster, and if you look at what `gethostlatency` has to say, you'll see the DNS queries are resolved more rapidly. 63 | 64 | - - - 65 | 66 | #### Bonus 67 | 68 | Do you think it's normal that the Node process is making a DNS query for each outgoing HTTP request? Try to look online to find a reasonable solution for this problem that would make sense in your environment. 69 | 70 | - - - 71 | -------------------------------------------------------------------------------- /node-slowdown.md: -------------------------------------------------------------------------------- 1 | ### Node Leaky Slowdown 2 | 3 | In this lab, you will experiment with a Node application that gradually slows down as more and more requests are made to the process. Over time, this becomes unbearable and administrators are used to forcing a restart; that's not how you roll, though, right? 4 | 5 | > NOTE: Unlike other exercises, this exercise is open-ended; we don't provide extraneous clues. It's up to you to use any of the tools from the previous exercises to diagnose this issue. If this is your first lab, perhaps you might want to consider looking at the [Node Profiling with V8](node-prof.md) or [CPU Sampling with `perf` and Flame Graphs](perf.md) labs first. 6 | 7 | - - - 8 | 9 | #### Task 1: Reproduce the Issue 10 | 11 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start our simple Node application: 12 | 13 | ``` 14 | $ ./run.sh 15 | ``` 16 | 17 | The application comes with a self-sufficient benchmark that reproduces the issue at hand by issuing 2500 POST requests to the `/position` endpoint, which is supposed to track accurate position information for certain real-world objects reported to the application. Run the following command to start the benchmark: 18 | 19 | ``` 20 | $ ./run.sh bench 21 | ``` 22 | 23 | You will see individual operations reporting their elapsed time. The time begins to degrade, and can reach 2x or worse in only a few seconds (after making a few hundred requests). 24 | 25 | - - - 26 | 27 | #### Task 2: Diagnose the Problem 28 | 29 | From here, you're on your own: profile the application using whatever tools you see fit, until you manage to pinpoint the problem. If you have time, try _fixing_ the problem and running the benchmark again to make sure the issue is really gone. 30 | 31 | - - - 32 | -------------------------------------------------------------------------------- /node-slowhttp.md: -------------------------------------------------------------------------------- 1 | ### Node Slow HTTP Requests 2 | 3 | In this lab, you will experiment with a Node service that talks to external HTTP endpoints (think additional microservices) and occasionally returns a slow result. Although there are third-party APM solutions that can help diagnose this kind of issue, we will use the USDT probes built into Node, which don't require any code changes or restarting the service. 4 | 5 | - - - 6 | 7 | #### Task 1: Run the Application 8 | 9 | Navigate to the `nodey` directory. If you haven't yet, you should make sure all required packages are installed by running `npm install`. Then, run the following command to start our simple Node application: 10 | 11 | ``` 12 | $ ./run.sh 13 | ``` 14 | 15 | Run a couple of queries to the Inventory endpoint by using the following commands. Note the time it takes to return a response from the service. 16 | 17 | ``` 18 | $ time curl http://localhost:3000/inventory?product_id=a19224 19 | $ time curl http://localhost:3000/inventory?product_id=a19877 20 | $ time curl http://localhost:3000/inventory?product_id=a88711 21 | ``` 22 | 23 | Depending on how far you are from the inventory service's backend, these queries may return in under a second or take a few full seconds. However, this is the baseline, and most requests seem to take the same time. 24 | 25 | Occasionally, though, we have reports of slow requests. After some spelunking, this is how to make a slow request: 26 | 27 | ``` 28 | $ time curl http://localhost:3000/inventory?product_id=g11899 29 | ``` 30 | 31 | What's wrong with this specific product? It seems to be taking much slower than the other calls, in a fairly reliable way. 32 | 33 | - - - 34 | 35 | #### Task 2: Inspect Outgoing HTTP Requests with `trace` and `nhttpslower` 36 | 37 | It might be interesting to explore the outgoing HTTP requests that our Node service makes to the inventory service on our behalf. Recall that our build of Node is instrumented with USDT probes, including probes for HTTP requests; this means we can use the `trace` tool from [BCC](https://github.com/iovisor/bcc) to trace outgoing HTTP requests. In a root shell, run the following command to do so: 38 | 39 | ``` 40 | # trace -p $(pgrep -n node) 'u::http__client__request "%s %s", arg5, arg6' 41 | ``` 42 | 43 | Now, in the original shell, access the Inventory endpoint again. You should see printouts from `trace` indicating which backend services are being invoked. It looks like there are five different backend services that our Node application is talking to. But how long are these requests taking? 44 | 45 | The [`nhttpslower`](nhttpslower.py) tool is a demo tool that traces the duration of Node HTTP requests (incoming and outgoing), using the USDT probes built into Node. It can be very useful in our case; run it like this, from a root shell: 46 | 47 | ``` 48 | # ./nhttpslower.py $(pgrep -n node) --client 49 | ``` 50 | 51 | > The `--client` switch instructs the tool to print only HTTP requests our Node application makes to the outside world (as a client), and not any incoming HTTP requests. This is just to avoid cluttering the output. 52 | 53 | Now, access the Inventory endpoint again. `nhttpslower` should report the exact endpoints accessed, the timestamps when the operation was issued, and its duration. Immediately, the slow backend service becomes apparent: it's the `g11` service. It also becomes apparent that the application is issuing the inventory requests serially: it waits for each request to complete before issuing the next one. 54 | 55 | ``` 56 | # ./nhttpslower.py $(pgrep -n node) --client 57 | Snooping HTTP requests in Node process 14248, Ctrl+C to quit. 58 | TIME_s TYP DURATION_ms REMOTE PORT METHOD URL 59 | 0.000000000 CLI 870.875 undefined 0 GET /get?svc=a12&inventory=available&product_id=g1231 60 | 0.873364421 CLI 639.769 undefined 0 GET /get?svc=b22&inventory=available&product_id=g1231 61 | 1.515958655 CLI 2668.286 undefined 0 GET /delay/2?svc=g11&inventory=not_available&product_id=g1231 62 | 4.188127138 CLI 642.796 undefined 0 GET /get?svc=a78&inventory=not_available&product_id=g1231 63 | 4.833378060 CLI 633.348 undefined 0 GET /get?svc=b43&inventory=available&product_id=g1231 64 | ``` 65 | 66 | We might not be able to fix the `g11` service, but we can definitely fix the fact we're making the requests serially. You can open the [index.js](nodey/routes/index.js) file and modify the `/inventory` route to use `async.map` instead of `async.mapSeries`. This should immediately improve the client's performance, because we're not waiting for each backend service to return before calling the next one. 67 | 68 | - - - 69 | 70 | #### Bonus 71 | 72 | Alternative approaches for tracing request latency are more invasive. Some options you could try include: 73 | 74 | * Hooking the `request` function with something like [`request-debug`](https://github.com/request/request-debug), and measuring the latency yourself. 75 | 76 | * Using a local proxy to pipe your requests through, and then inspecting them in the proxy's interface. 77 | 78 | * Using `tcpdump` or another sniffer to record the traffic and analyze it later. With something like `tshark`, you can get HTTP latency summaries and perform various aggregations. (Technically, this can be done with a BPF program in real-time, too; it's just that someone has to write a BPF program to parse HTTP requests and responses.) 79 | 80 | Here's an example of a tshark command that captures packets and displays a summary of HTTP requests by HTTP host and URL: 81 | 82 | ``` 83 | # tshark -z http_req,tree -Y http 84 | ``` 85 | 86 | - - - 87 | 88 | -------------------------------------------------------------------------------- /nodey/app.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var path = require('path'); 3 | var favicon = require('serve-favicon'); 4 | var logger = require('morgan'); 5 | var cookieParser = require('cookie-parser'); 6 | var bodyParser = require('body-parser'); 7 | 8 | var index = require('./routes/index'); 9 | var users = require('./routes/users'); 10 | 11 | var app = express(); 12 | 13 | // view engine setup 14 | app.set('views', path.join(__dirname, 'views')); 15 | app.set('view engine', 'jade'); 16 | 17 | // uncomment after placing your favicon in /public 18 | //app.use(favicon(path.join(__dirname, 'public', 'favicon.ico'))); 19 | app.use(logger('dev')); 20 | app.use(bodyParser.json()); 21 | app.use(bodyParser.urlencoded({ extended: false })); 22 | app.use(cookieParser()); 23 | app.use(express.static(path.join(__dirname, 'public'))); 24 | 25 | app.use('/', index); 26 | app.use('/users', users); 27 | 28 | // catch 404 and forward to error handler 29 | app.use(function(req, res, next) { 30 | var err = new Error('Not Found'); 31 | err.status = 404; 32 | next(err); 33 | }); 34 | 35 | // error handler 36 | app.use(function(err, req, res, next) { 37 | // set locals, only providing error in development 38 | var dev = req.app.get('env') === 'development'; 39 | res.locals.message = dev ? err.message : 'An error occurred'; 40 | res.locals.error = req.app.get('env') === 'development' ? err : {}; 41 | 42 | // render the error page 43 | res.status(err.status || 500); 44 | res.render('error'); 45 | }); 46 | 47 | module.exports = app; 48 | -------------------------------------------------------------------------------- /nodey/bin/www: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var app = require('../app'); 8 | var debug = require('debug')('temp:server'); 9 | var http = require('http'); 10 | 11 | /** 12 | * Get port from environment and store in Express. 13 | */ 14 | 15 | var port = normalizePort(process.env.PORT || '3000'); 16 | app.set('port', port); 17 | 18 | /** 19 | * Create HTTP server. 20 | */ 21 | 22 | var server = http.createServer(app); 23 | 24 | /** 25 | * Listen on provided port, on all network interfaces. 26 | */ 27 | 28 | server.listen(port); 29 | server.on('error', onError); 30 | server.on('listening', onListening); 31 | 32 | /** 33 | * Normalize a port into a number, string, or false. 34 | */ 35 | 36 | function normalizePort(val) { 37 | var port = parseInt(val, 10); 38 | 39 | if (isNaN(port)) { 40 | // named pipe 41 | return val; 42 | } 43 | 44 | if (port >= 0) { 45 | // port number 46 | return port; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | /** 53 | * Event listener for HTTP server "error" event. 54 | */ 55 | 56 | function onError(error) { 57 | if (error.syscall !== 'listen') { 58 | throw error; 59 | } 60 | 61 | var bind = typeof port === 'string' 62 | ? 'Pipe ' + port 63 | : 'Port ' + port; 64 | 65 | // handle specific listen errors with friendly messages 66 | switch (error.code) { 67 | case 'EACCES': 68 | console.error(bind + ' requires elevated privileges'); 69 | process.exit(1); 70 | break; 71 | case 'EADDRINUSE': 72 | console.error(bind + ' is already in use'); 73 | process.exit(1); 74 | break; 75 | default: 76 | throw error; 77 | } 78 | } 79 | 80 | /** 81 | * Event listener for HTTP server "listening" event. 82 | */ 83 | 84 | function onListening() { 85 | var addr = server.address(); 86 | var bind = typeof addr === 'string' 87 | ? 'pipe ' + addr 88 | : 'port ' + addr.port; 89 | debug('Listening on ' + bind); 90 | } 91 | -------------------------------------------------------------------------------- /nodey/inventory.lst: -------------------------------------------------------------------------------- 1 | # Main inventory 2 | http://httpbin.org/get?svc=a12&inventory=available 3 | http://httpbin.org/get?svc=b22&inventory=available 4 | 5 | # Prime inventory service 6 | http://httpbin.org/delay/2?svc=g11&inventory=not_available 7 | 8 | # Backup warehouses 9 | http://httpbin.org/get?svc=a78&inventory=not_available 10 | http://httpbin.org/get?svc=b43&inventory=available 11 | -------------------------------------------------------------------------------- /nodey/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodey", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "start": "node ./bin/www" 7 | }, 8 | "dependencies": { 9 | "async": "~2.1.5", 10 | "body-parser": "~1.16.0", 11 | "cookie-parser": "~1.4.3", 12 | "debug": "~2.6.0", 13 | "express": "~4.14.1", 14 | "jade": "~1.11.0", 15 | "morgan": "~1.7.0", 16 | "mysql": "~2.13.0", 17 | "request": "~2.81.0", 18 | "serve-favicon": "~2.3.2" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /nodey/public/stylesheets/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding: 50px; 3 | font: 14px "Lucida Grande", Helvetica, Arial, sans-serif; 4 | } 5 | 6 | a { 7 | color: #00B7FF; 8 | } 9 | -------------------------------------------------------------------------------- /nodey/routes/index.js: -------------------------------------------------------------------------------- 1 | var async = require('async'); 2 | var express = require('express'); 3 | var fs = require('fs'); 4 | var HashTable = require('../utils/hashtable'); 5 | var mysql = require('mysql'); 6 | var pos = require('../utils/position'); 7 | var request = require('request'); 8 | var router = express.Router(); 9 | 10 | var positions = new HashTable(pos.hasher, pos.comparer, 1337); 11 | 12 | function prime_product(product_id) { 13 | return product_id.startsWith('g'); 14 | } 15 | 16 | function dynamic_product(product_id) { 17 | return product_id.startsWith('dyn'); 18 | } 19 | 20 | function dynamic_inventory_service() { 21 | return 'http://' + Math.trunc(Math.random() * 100000).toString() 22 | + '.example.org/inventory?svc=sdyn199'; 23 | } 24 | 25 | function inventory_services(product_id) { 26 | var services = fs 27 | .readFileSync(__dirname + '/../inventory.lst', 'utf-8') 28 | .split('\n') 29 | .filter(Boolean) 30 | .filter(function(s) { return !s.startsWith('#'); }); 31 | if (!prime_product(product_id)) { 32 | services.splice(2, 1); 33 | } 34 | if (dynamic_product(product_id)) { 35 | services.push(dynamic_inventory_service()); 36 | } 37 | return services.map(function(s) { return s + '&product_id=' + product_id; }); 38 | } 39 | 40 | function createConnection() { 41 | return mysql.createConnection({ 42 | host: 'localhost', 43 | user: 'newuser', 44 | password: 'password', 45 | database: 'acme' 46 | }); 47 | } 48 | 49 | function loadUsers(cb) { 50 | var connection = createConnection(); 51 | var sql = 'SELECT id, name FROM users'; 52 | connection.query(sql, function(error, results, fields) { 53 | if (error) cb(error); 54 | else { 55 | cb(null, results); 56 | } 57 | }); 58 | connection.end(); 59 | } 60 | 61 | function authenticate_stats(key, cb) { 62 | setTimeout(function() { 63 | if (key == 'mykey') 64 | cb(null); 65 | else 66 | cb(new Error('invalid key')); 67 | }, 1000); 68 | } 69 | 70 | function getProduct(conn, id, cb) { 71 | conn.query('CALL getproduct(' + id + ')', function(error, results, fields) { 72 | if (error) cb(error); 73 | else cb(null, results[0]); 74 | }); 75 | } 76 | 77 | function loadProducts(user, cb) { 78 | var connection = createConnection(); 79 | var sql = 'SELECT id FROM products WHERE userid = ?'; 80 | var products = []; 81 | connection.query(sql, [user.id], function(error, results, fields) { 82 | if (error) { 83 | cb(error); 84 | connection.end(); 85 | } else { 86 | async.eachSeries(results, function(item, acb) { 87 | getProduct(connection, item.id, function(err, product) { 88 | if (err) acb(err); 89 | else { 90 | products.push(product); 91 | acb(null); 92 | } 93 | }); 94 | }, function(err) { 95 | if (err) cb(err); 96 | else { 97 | cb(null, products); 98 | } 99 | connection.end(); 100 | }); 101 | } 102 | }); 103 | } 104 | 105 | /* GET home page. */ 106 | router.get('/', function(req, res, next) { 107 | res.render('index', { title: 'Express' }); 108 | }); 109 | 110 | router.get('/about', function(req, res, next) { 111 | var data = fs.readFileSync('/etc/nodey.conf', 'utf8'); 112 | res.render('about', { title: 'About', data: data }); 113 | }); 114 | 115 | router.get('/stats', function(req, res, next) { 116 | var len = 10*1024*1024; 117 | fs.writeFileSync('nodey.stats', new Array(len).join('q')); 118 | res.json({ ok: true, written: len }); 119 | }); 120 | 121 | router.put('/stats', function(req, res, next) { 122 | var stats = req.body; 123 | authenticate_stats(stats.auth, function(err) { 124 | if (err) 125 | throw err; 126 | res.sendStatus(201); 127 | }); 128 | }); 129 | 130 | router.get('/products', function(req, res, next) { 131 | var users = []; 132 | loadUsers(function(error, users) { 133 | if (error) res.sendStatus(500); 134 | else { 135 | async.eachSeries(users, function(user, acb) { 136 | loadProducts(user, function(error, products) { 137 | if (error) acb(error); 138 | else { 139 | users.push({ user: user.id, products: products }); 140 | acb(null); 141 | } 142 | }); 143 | }, function(err) { 144 | if (err) { 145 | res.sendStatus(500); 146 | } 147 | else res.json(users); 148 | }); 149 | } 150 | }); 151 | }); 152 | 153 | router.get('/inventory', function(req, res, next) { 154 | var services = inventory_services(req.query.product_id); 155 | async.mapSeries(services, request, function(err, results) { 156 | res.json({ 157 | data: results.filter(Boolean).map(function(r) { 158 | var body = JSON.parse(r.body); 159 | return body.args; 160 | }), 161 | error: err ? "an error occurred enumerating one of the inventory services" 162 | : null 163 | }); 164 | }); 165 | }); 166 | 167 | router.post('/position', function(req, res, next) { 168 | var x = req.query.x, y = req.query.y, z = req.query.z; 169 | var position = new pos.Position(x, y, z); 170 | if (positions.get(position) != null) 171 | res.sendStatus(400); 172 | else { 173 | positions.put(position, 1); 174 | res.sendStatus(201); 175 | } 176 | }); 177 | 178 | module.exports = router; 179 | -------------------------------------------------------------------------------- /nodey/routes/users.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto'); 2 | var express = require('express'); 3 | var fs = require('fs'); 4 | var router = express.Router(); 5 | 6 | var users = []; 7 | 8 | function clean_user(name, info) { 9 | var processed_salt = ''; 10 | for (var i = 0; i < 100000; ++i) { 11 | var size = Math.floor(Math.random() * info.salt.length); 12 | processed_salt += info.salt.slice(0, size); 13 | } 14 | return { name: name, len: processed_salt.length }; 15 | } 16 | 17 | function EmailTemplateNode(index) { 18 | this.index = index; 19 | this.index_desc = index.toString(); 20 | } 21 | 22 | function EmailTemplate(template) { 23 | this.data = fs.readFileSync(__dirname + template, 'utf8'); 24 | this.nodes = []; 25 | var node_count = Math.random()*1000; 26 | for (var i = 0; i < node_count; ++i) { 27 | this.nodes.push(new EmailTemplateNode(i)); 28 | } 29 | } 30 | 31 | function fetch_template() { 32 | return new EmailTemplate('/../public/template.html'); 33 | } 34 | 35 | /* GET users listing. */ 36 | router.get('/', function(req, res, next) { 37 | var clean_users = []; 38 | for (var user in users) { 39 | clean_users.push(clean_user(user, users[user])); 40 | } 41 | res.json(clean_users); 42 | }); 43 | 44 | router.post('/subscribe', function(req, res, next) { 45 | var subscriptionTemplate = null; 46 | function refreshSubscription() { 47 | var original = subscriptionTemplate; 48 | var first_test = function() { 49 | if (original) 50 | console.log("Replacing original subscription."); 51 | }; 52 | subscriptionTemplate = { 53 | template: fetch_template(), 54 | send: function() { 55 | console.log("Sending updated subscription."); 56 | } 57 | }; 58 | first_test(); 59 | } 60 | setInterval(refreshSubscription, 1000); 61 | res.sendStatus(200); 62 | }); 63 | 64 | router.post('/new', function(req, res, next) { 65 | var username = req.query.username || ''; 66 | var password = req.query.password || ''; 67 | 68 | username = username.replace(/[!@#$%^&*]/g, ''); 69 | 70 | if (!username || !password || users.username) { 71 | return res.sendStatus(400); 72 | } 73 | 74 | var salt = crypto.randomBytes(512).toString('base64'); 75 | var hash = crypto.pbkdf2Sync(password, salt, 10000, 512, 'sha512'); 76 | 77 | users[username] = { 78 | salt: salt, 79 | hash: hash 80 | }; 81 | 82 | res.sendStatus(200); 83 | }); 84 | 85 | router.post('/auth', function(req, res, next) { 86 | var username = req.query.username || ''; 87 | var password = req.query.password || ''; 88 | 89 | username = username.replace(/[!@#$%^&*]/g, ''); 90 | 91 | if (!username || !password || !users[username]) { 92 | return res.sendStatus(400); 93 | } 94 | 95 | var hash = crypto.pbkdf2Sync( 96 | password, users[username].salt, 10000, 512, 'sha512'); 97 | 98 | if (users[username].hash.toString() === hash.toString()) { 99 | res.sendStatus(200); 100 | } else { 101 | res.sendStatus(401); 102 | } 103 | }); 104 | 105 | module.exports = router; 106 | -------------------------------------------------------------------------------- /nodey/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export NODE_ENV=production 4 | 5 | function kill_ns { 6 | if ps -ef | grep -v grep | grep -q -F './slodns'; then 7 | ps -ef | grep './slodns' | grep -v grep | awk '{ print $2 }' | \ 8 | xargs sudo kill -9 9 | fi 10 | } 11 | 12 | function setup_ns { 13 | if ! grep -q -F 127.0.0.1 /etc/resolv.conf; then 14 | echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" \ 15 | | sudo tee /etc/resolv.conf 16 | fi 17 | kill_ns 18 | if ! [ -f ./slodns ]; then 19 | wget -q https://raw.githubusercontent.com/goldshtn/slodns/master/slodns \ 20 | -O slodns 21 | chmod u+x ./slodns 22 | fi 23 | sudo ./slodns -p 53 -d 1000 -j 500 >/dev/null & 24 | disown 25 | } 26 | 27 | function bench { 28 | for x in `seq 0 50`; do 29 | for y in `seq 0 50`; do 30 | start_ts=`date +%s%N` 31 | curl -X POST "http://localhost:3000/position?x=$x&y=$y&z=0" 32 | end_ts=`date +%s%N` 33 | runtime=$(((end_ts-start_ts)/1000)) 34 | printf "running time: %dus\n" $runtime 35 | done 36 | done 37 | } 38 | 39 | if [ "$1" == "perf" ]; then 40 | FLAGS="--perf_basic_prof" 41 | elif [ "$1" == "prof" ]; then 42 | FLAGS="--prof" 43 | elif [ "$1" == "core" ]; then 44 | FLAGS="--abort-on-uncaught-exception" 45 | elif [ "$1" == "dns" ]; then 46 | setup_ns 47 | elif [ "$1" == "killdns" ]; then 48 | kill_ns 49 | exit 50 | elif [ "$1" == "bench" ]; then 51 | bench 52 | exit 53 | else 54 | FLAGS="$@" 55 | fi 56 | 57 | pkill node 58 | node $FLAGS bin/www >/dev/null & 59 | 60 | sleep 1 61 | curl -X POST 'http://localhost:3000/users/new?username=foo&password=pass' 62 | curl -X POST 'http://localhost:3000/users/new?username=bar&password=pass' 63 | -------------------------------------------------------------------------------- /nodey/utils/hashtable.js: -------------------------------------------------------------------------------- 1 | function HashTable(hasher, comparer, n) { 2 | this._hasher = hasher; 3 | this._comparer = comparer; 4 | this._buckets = new Array(n || 131); 5 | this._count = 0; 6 | } 7 | 8 | HashTable.prototype.put = function(key, value) { 9 | var k = this._hasher(key) % this._buckets.length; 10 | var bucket = this._buckets[k]; 11 | if (bucket == null) { 12 | bucket = new Array(); 13 | this._buckets[k] = bucket; 14 | } else { 15 | for (var i = 0; i < bucket.length; ++i) { 16 | if (this._comparer(bucket[i].key, key)) { 17 | bucket[i].value = value; 18 | return; 19 | } 20 | } 21 | } 22 | bucket.push({ key: key, value: value }); 23 | ++this._count; 24 | }; 25 | 26 | HashTable.prototype.get = function(key) { 27 | var k = this._hasher(key) % this._buckets.length; 28 | var bucket = this._buckets[k]; 29 | if (bucket == null) 30 | return null; 31 | for (var i = 0; i < bucket.length; ++i) { 32 | if (this._comparer(bucket[i].key, key)) { 33 | return bucket[i].value; 34 | } 35 | } 36 | return null; 37 | }; 38 | 39 | HashTable.prototype.size = function() { 40 | return this._count; 41 | }; 42 | 43 | HashTable.prototype.remove = function(key) { 44 | var k = this._hasher(key) % this._buckets.length; 45 | var bucket = this._buckets[k]; 46 | if (bucket == null) 47 | return; 48 | for (var i = 0; i < bucket.length; ++i) { 49 | if (this._comparer(bucket[i].key, key)) { 50 | bucket.splice(i, 1); 51 | --this._count; 52 | return; 53 | } 54 | } 55 | }; 56 | 57 | HashTable.prototype.clear = function(key) { 58 | this._count = 0; 59 | this._buckets = new Array(this._buckets.length); 60 | }; 61 | 62 | HashTable.prototype.dump = function() { 63 | console.log('hashtable with ' + this._count + ' elements:'); 64 | for (var i = 0; i < this._buckets.length; ++i) { 65 | var bucket = this._buckets[i]; 66 | if (bucket != null) { 67 | console.log(' bucket ' + i + ' with ' + bucket.length + ' elements:'); 68 | for (var j = 0; j < bucket.length; ++j) { 69 | console.log(' element ' + j + ': key = ' + bucket[j].key + 70 | ', value = ' + bucket[j].value); 71 | } 72 | } 73 | } 74 | }; 75 | 76 | module.exports = HashTable; 77 | -------------------------------------------------------------------------------- /nodey/utils/position.js: -------------------------------------------------------------------------------- 1 | function Position(x, y, z) { 2 | this._x = x; 3 | this._y = y; 4 | this._z = z; 5 | this._extra = Array(10001).join('*'); 6 | } 7 | 8 | function hasher(pos) { 9 | return pos._z; 10 | } 11 | 12 | function comparer(pos1, pos2) { 13 | return pos1._extra == pos2._extra && 14 | pos1._x == pos2._x && pos1._y == pos2._y && pos1._z == pos2._z; 15 | } 16 | 17 | module.exports = { 18 | Position: Position, 19 | hasher: hasher, 20 | comparer: comparer 21 | }; 22 | -------------------------------------------------------------------------------- /nodey/views/about.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h1= title 5 | p This beautiful app was built very quickly. 6 | -------------------------------------------------------------------------------- /nodey/views/error.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h1= message 5 | h2= error.status 6 | pre #{error.stack} 7 | -------------------------------------------------------------------------------- /nodey/views/index.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h1= title 5 | p Welcome to #{title} 6 | a(href="/about") About 7 | -------------------------------------------------------------------------------- /nodey/views/layout.jade: -------------------------------------------------------------------------------- 1 | doctype html 2 | html 3 | head 4 | title= title 5 | link(rel='stylesheet', href='/stylesheets/style.css') 6 | body 7 | block content 8 | -------------------------------------------------------------------------------- /pargrep.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | class pargrep { 10 | std::string dir_; 11 | std::regex regexp_; 12 | std::vector results_; 13 | public: 14 | pargrep(std::string dir, std::string regexp) 15 | : dir_(std::move(dir)), regexp_(regexp) { 16 | } 17 | 18 | void run() { 19 | std::vector files { 20 | // TODO use actual files from the provided directory 21 | "pargrep.cc", 22 | "pargrep.cc", 23 | "pargrep.cc", 24 | "pargrep.cc" 25 | }; 26 | #pragma omp parallel for 27 | for (int i = 0; i < files.size(); ++i) { 28 | do_one_file(files[i]); 29 | } 30 | std::copy(results_.begin(), results_.end(), 31 | std::ostream_iterator(std::cout, "\n")); 32 | } 33 | private: 34 | void do_one_file(std::string const& filename) { 35 | std::cout << omp_get_thread_num() << " " << filename << "\n"; 36 | std::string line; 37 | std::ifstream ifs(filename.c_str()); 38 | while (std::getline(ifs, line)) { 39 | if (std::regex_search(line, regexp_)) { 40 | results_.push_back(filename + ": " + line); 41 | } 42 | } 43 | } 44 | }; 45 | 46 | void usage() { 47 | std::cerr << "USAGE: pargrep \n"; 48 | exit(1); 49 | } 50 | 51 | int main(int argc, char* argv[]) { 52 | if (argc < 3) { 53 | usage(); 54 | } 55 | 56 | omp_set_num_threads(4); 57 | pargrep grepper(argv[1], argv[2]); 58 | grepper.run(); 59 | 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /parprimes.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | int is_prime(int n) { 6 | int i; 7 | 8 | if (n <= 2) return 1; 9 | if (n % 2 == 0) return 0; 10 | 11 | for (i = 3; i < n; i += 2) { 12 | if (n % i == 0) return 0; 13 | } 14 | 15 | return 1; 16 | } 17 | 18 | struct search_range { 19 | int start; 20 | int end; 21 | int count; 22 | pthread_mutex_t *mutex; 23 | }; 24 | 25 | void primes_loop(struct search_range *range) 26 | { 27 | int i; 28 | 29 | for (i = range->start; i < range->end; ++i) 30 | { 31 | if (is_prime(i)) 32 | { 33 | pthread_mutex_lock(range->mutex); 34 | ++(range->count); 35 | pthread_mutex_unlock(range->mutex); 36 | } 37 | } 38 | } 39 | 40 | void *primes_thread(void *ctx) { 41 | struct search_range *range; 42 | 43 | range = (struct search_range *)ctx; 44 | primes_loop(range); 45 | 46 | return NULL; 47 | } 48 | 49 | int main(int argc, char* argv[]) { 50 | int nthreads, max; 51 | pthread_t *threads; 52 | struct search_range *ranges; 53 | int i; 54 | pthread_mutex_t mutex; 55 | 56 | puts("Hit ENTER to start."); 57 | getchar(); 58 | 59 | pthread_mutex_init(&mutex, NULL); 60 | 61 | if (argc >= 3) { 62 | nthreads = atoi(argv[1]); 63 | max = atoi(argv[2]); 64 | } 65 | else { 66 | nthreads = 2; 67 | max = 100000; 68 | } 69 | 70 | threads = (pthread_t *)malloc(sizeof(pthread_t) * nthreads); 71 | ranges = (struct search_range *)malloc(sizeof(struct search_range) * nthreads); 72 | for (i = 0; i < nthreads; ++i) { 73 | ranges[i].start = i * (max / nthreads); 74 | ranges[i].end = (i + 1) * (max / nthreads); 75 | ranges[i].count = 0; 76 | ranges[i].mutex = &mutex; 77 | pthread_create(&threads[i], NULL, primes_thread, &ranges[i]); 78 | } 79 | 80 | for (i = 0; i < nthreads; ++i) { 81 | pthread_join(threads[i], NULL); 82 | printf("thread %d found %d primes\n", i, ranges[i].count); 83 | } 84 | 85 | free(ranges); 86 | free(threads); 87 | 88 | return 0; 89 | } 90 | 91 | -------------------------------------------------------------------------------- /perf-opens.md: -------------------------------------------------------------------------------- 1 | ### Using `perf` Tools: Broken File Opens 2 | 3 | In this lab, you will experiment with diagnosing an application that fails to start correctly by using some of the tools in the perf-tools toolkit. 4 | 5 | - - - 6 | 7 | #### Task 1: Compile and Run the Application 8 | 9 | First, run the following command to compile `server.c` into the server application you'll be diagnosing: 10 | 11 | ``` 12 | $ gcc -g -fno-omit-frame-pointer -O0 server.c -o server 13 | ``` 14 | 15 | Now, run `./server`. It should print a message saying that it's starting up, but it never completes initialization. 16 | 17 | - - - 18 | 19 | #### Task 2: Perform Basic Diagnostics 20 | 21 | Because the application process seems to be stuck, let's try to see what it's doing. Run `top` -- you should see the app near the top of the output, consuming some CPU. Next, run the following command to see a report of the process' user and system CPU utilization every second: 22 | 23 | ``` 24 | $ pidstat -u -p $(pidof server) 1 25 | ``` 26 | 27 | It looks like the process is spending a bit of time in kernel mode. 28 | 29 | - - - 30 | 31 | #### Task 3: Snoop Syscalls 32 | 33 | > For the rest of this lab, you will need the perf-tools toolkit on your path, or you will need to prefix the commands with the full path to where you placed the [perf-tools](https://github.com/brendangregg/perf-tools) repository. 34 | 35 | If the process is running frequently in kernel mode, it must be making quite a bunch of syscalls. To characterize which syscalls it's making, use the `syscount` tool: 36 | 37 | ``` 38 | # syscount -c -p $(pidof server) 39 | ``` 40 | 41 | > NOTE: There's a BCC tool called syscount, and a perf-tools tool called syscount. If you have both installed and in the path (like in the lab environment), you will need to qualify the specific one -- in this case, the one in the perf-tools folder. 42 | 43 | This collects all syscall events. Press Ctrl+C after a few seconds to stop collection. As you can see, the application is making a lot of `nanosleep()` and `open()` calls, but doesn't seem to be making progress. This looks like a typical retry pattern -- try to open a file, fail, and then try again. 44 | 45 | - - - 46 | 47 | #### Task 4: Snooping Opens 48 | 49 | Fortunately, there a tool for snooping all `open` calls performed by a certain process, including the path being opened and the result of the call. Run the following command to trace opens: 50 | 51 | ``` 52 | # opensnoop -p $(pidof server) 53 | ``` 54 | 55 | The problem becomes apparent -- the application is trying to open the `/etc/tracing-server-example.conf` file, and is getting a -1 result. Indeed, you can confirm that the file does not exist. You can even try creating the file and making sure the server now starts successfully. 56 | 57 | - - - 58 | -------------------------------------------------------------------------------- /perros/.gitignore: -------------------------------------------------------------------------------- 1 | out/ 2 | libagent.so 3 | -------------------------------------------------------------------------------- /perros/Admin.java: -------------------------------------------------------------------------------- 1 | package perros; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Files; 5 | import java.nio.file.Paths; 6 | 7 | class AdminHandler { 8 | private static final String CONFIG_FILE = "/etc/perros.conf"; 9 | 10 | @Get 11 | public void handle(Request request) throws IOException { 12 | String config = new String(Files.readAllBytes(Paths.get(CONFIG_FILE))); 13 | String username = request.queryParams().get("username"); 14 | if (config.contains("approve: " + username)) { 15 | request.ok(); 16 | } else { 17 | request.unauthorized(); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /perros/Auth.java: -------------------------------------------------------------------------------- 1 | package perros; 2 | 3 | import java.io.IOException; 4 | import java.security.MessageDigest; 5 | import java.security.NoSuchAlgorithmException; 6 | import java.util.Arrays; 7 | import java.util.Map; 8 | 9 | class AuthHandler { 10 | @Post 11 | public void handle(Request request) throws IOException { 12 | String username = request.bodyJson().get("username").toString(); 13 | String password = request.bodyJson().get("password").toString(); 14 | 15 | try { 16 | MessageDigest sha = MessageDigest.getInstance("SHA-256"); 17 | byte[] passwordBytes = (getSalt(username) + password).getBytes(); 18 | for (int i = 0; i < 1000; ++i) { 19 | // Do it a few times, for good measure. 20 | byte[] hash = sha.digest(passwordBytes); 21 | } 22 | } catch (NoSuchAlgorithmException e) { 23 | System.out.println("Oops, no SHA-256 support."); 24 | } 25 | 26 | // For now, we're OK with any password :-) 27 | request.ok(); 28 | } 29 | 30 | private String getSalt(String username) { 31 | char[] salt = new char[4096]; 32 | Arrays.fill(salt, '*'); 33 | return new String(salt); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /perros/BadRoute.java: -------------------------------------------------------------------------------- 1 | package perros; 2 | 3 | import java.io.IOException; 4 | 5 | class BadRoute { 6 | private static BadRoute instance = new BadRoute(); 7 | 8 | public static BadRoute getInstance() { return instance; } 9 | } 10 | -------------------------------------------------------------------------------- /perros/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8 2 | 3 | RUN apt update && apt install -y cmake g++ \ 4 | && rm -rf /var/lib/apt/lists/* 5 | 6 | RUN git clone --depth=1 https://github.com/jvm-profiling-tools/perf-map-agent \ 7 | && cd perf-map-agent && cmake . && make 8 | 9 | RUN git clone --depth=1 https://github.com/jvm-profiling-tools/async-profiler \ 10 | && cd async-profiler && make 11 | -------------------------------------------------------------------------------- /perros/Makefile: -------------------------------------------------------------------------------- 1 | ALLROOT=~/tracing-workshop 2 | LABSROOT=$(ALLROOT)/labs 3 | HOST=localhost 4 | PORT=8000 5 | PERFMAPAGENT=$(ALLROOT)/perf-map-agent 6 | FLAMEGRAPH=$(ALLROOT)/FlameGraph 7 | FGOUT=/tmp/fg.svg 8 | JAVACFLAGS=-Xlint:unchecked 9 | JAVAOPTS=-XX:+PreserveFramePointer 10 | APROFILER=$(ALLROOT)/async-profiler 11 | BCC=/usr/share/bcc/tools 12 | MYSQLCONNJAR=$(LABSROOT)/buggy/mysql-connector-java-5.1.40-bin.jar 13 | JVM=$(shell find /usr/lib/jvm -name libjvm.so -print -quit) 14 | CONTAINERNAME=perros 15 | # The allow-perf-event-open.json file is a minor modification of 16 | # https://github.com/moby/moby/blob/master/profiles/seccomp/default.json 17 | # which whitelists the perf_event_open syscall. 18 | DOCKERFLAGS=-p $(PORT):$(PORT) --rm -it -v $(CURDIR)/out:/app -v /tmp:/tmp \ 19 | --name $(CONTAINERNAME) --security-opt seccomp=allow-perf-event-open.json 20 | DOCKERCMD=perros:latest java $(JAVAOPTS) -cp /app perros/App $(PORT) 21 | CGROUPFS=/sys/fs/cgroup 22 | 23 | define flame-graph = 24 | $(FLAMEGRAPH)/flamegraph.pl --colors=java > $(FGOUT) < /tmp/java.stacks 25 | curl --upload-file $(FGOUT) https://transfer.sh 26 | endef 27 | 28 | define java-run = 29 | @java $(JAVAOPTS) \ 30 | -cp "out:out/`basename $(MYSQLCONNJAR)`" perros/App $(PORT) 31 | endef 32 | 33 | define docker-run = 34 | sudo docker run $(DOCKERFLAGS) $(DOCKERCMD) 35 | endef 36 | 37 | dockerbuild: 38 | sudo docker build -t perros:latest - < Dockerfile 39 | 40 | dockersvc: 41 | sudo service docker start 42 | 43 | build: 44 | mkdir -p out 45 | cp $(MYSQLCONNJAR) out/ 46 | javac $(JAVACFLAGS) *.java -d out 47 | 48 | buildagent: 49 | @if test -z "${JAVA_HOME}"; then \ 50 | echo "JAVA_HOME must be set to build the agent."; \ 51 | exit 1; \ 52 | fi; 53 | g++ -std=c++11 -O2 -fPIC -shared -o libagent.so -ldl -lpthread \ 54 | -I${JAVA_HOME}/include -I${JAVA_HOME}/include/linux agent.cc 55 | 56 | run: 57 | $(java-run) 58 | 59 | runext: 60 | $(eval JAVAOPTS=$(JAVAOPTS) -XX:+ExtendedDTraceProbes) 61 | $(java-run) 62 | 63 | attachagent: 64 | $(APROFILER)/build/jattach `pidof java` load $(CURDIR)/libagent.so true 65 | 66 | dumpagent: 67 | $(APROFILER)/build/jattach `pidof java` load $(CURDIR)/libagent.so true dump 68 | 69 | dockerrun: 70 | $(docker-run) 71 | 72 | dockerrun2: 73 | $(eval DOCKERFLAGS=$(DOCKERFLAGS) --cpus=0.5) 74 | $(docker-run) 75 | 76 | dockerrun3: # in the background, run 'make statsbench' 77 | $(eval DOCKERFLAGS=$(DOCKERFLAGS) --memory=100m) 78 | $(eval JAVAOPTS=$(JAVAOPTS) -XshowSettings:vm -XX:+PrintGCDetails) 79 | $(docker-run) 80 | 81 | dockerrun4: 82 | $(eval DOCKERFLAGS=$(DOCKERFLAGS) --memory=100m) 83 | $(eval JAVAOPTS=$(JAVAOPTS) -XshowSettings:vm -XX:+PrintGCDetails \ 84 | -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap \ 85 | -XX:MaxRAMFraction=1) 86 | $(docker-run) 87 | 88 | authbench: 89 | ab -p post-auth.json -c 10 -n 10000 http://$(HOST):$(PORT)/auth 90 | 91 | authrecord: 92 | $(PERFMAPAGENT)/bin/create-java-perf-map.sh `pidof java` 93 | sudo perf record -a -F 97 -g -- sleep 10 94 | sudo perf script -f | $(FLAMEGRAPH)/stackcollapse-perf.pl > /tmp/java.stacks 95 | $(flame-graph) 96 | 97 | registerbenchgood: 98 | ab -p post-register.json -n 100000 http://$(HOST):$(PORT)/register 99 | 100 | registerbenchbad: 101 | ab -p post-register-bad.json -n 1000 http://$(HOST):$(PORT)/register 102 | 103 | registerrecord: 104 | ./perf-std-user.sh 105 | $(APROFILER)/profiler.sh -d 10 -o summary,flat `pidof java` 106 | $(APROFILER)/profiler.sh -d 10 -o collapsed -f /tmp/java.stacks `pidof java` 107 | $(flame-graph) 108 | 109 | adminrepro: 110 | curl "$(HOST):$(PORT)/admin?username=david" -v 111 | 112 | adminsnoop1: 113 | sudo $(BCC)/opensnoop -p `pidof java` -x 114 | 115 | adminsnoop2: 116 | $(PERFMAPAGENT)/bin/create-java-perf-map.sh `pidof java` 117 | sudo $(BCC)/trace -p `pidof java` \ 118 | -U 'r::SyS_open (retval==-2) "failed open"' 119 | 120 | adminfix: 121 | echo 'approve: david' | sudo tee /etc/perros.conf 122 | 123 | usersbench: 124 | $(LABSROOT)/mysql-db.sh 125 | ./usersbench.sh "$(HOST):$(PORT)" 126 | 127 | usersrecord1: 128 | sudo $(BCC)/dbslower mysql -m 1000 129 | 130 | usersrecord2: 131 | sudo $(BCC)/trace 'u::query__exec__start "%s", arg1' -p `pidof mysqld` 132 | 133 | usersrecord3: 134 | $(PERFMAPAGENT)/bin/create-java-perf-map.sh `pidof java` 135 | sudo $(LABSROOT)/mysqlsniff.py -p `pidof java` -f "call getproduct(97)" -S 136 | 137 | statsbench: 138 | ab -n 10000 "$(HOST):$(PORT)/stats?pages=10000" 139 | 140 | statsgc: 141 | sudo $(BCC)/javastat 1 10 142 | 143 | statsrecord1: 144 | ./perf-std-user.sh 145 | $(APROFILER)/profiler.sh -d 10 -e alloc -o summary,flat `pidof java` 146 | $(APROFILER)/profiler.sh -d 10 -e alloc -o collapsed \ 147 | -f /tmp/java.stacks `pidof java` 148 | $(flame-graph) 149 | 150 | statsrecord2: 151 | $(PERFMAPAGENT)/bin/create-java-perf-map.sh `pidof java` dottedclass 152 | sudo $(BCC)/funccount "$(JVM):*AllocTracer*" -p `pidof java` -d 5 153 | sudo $(BCC)/stackcount "$(JVM):*AllocTracer*" -p `pidof java` -f \ 154 | -D 10 > /tmp/java.stacks 155 | $(flame-graph) 156 | 157 | statsrecord3: 158 | @echo Hit Ctrl+C after a few seconds to see the results from javaobjnew. 159 | sudo $(BCC)/javaobjnew `pidof java` -C 10 160 | 161 | statsrecord4: 162 | $(PERFMAPAGENT)/bin/create-java-perf-map.sh `pidof java` dottedclass 163 | sudo $(BCC)/stackcount "u:$(JVM):object__alloc" -p `pidof java` -f \ 164 | -D 10 > /tmp/java.stacks 165 | $(flame-graph) 166 | 167 | throttlerecord: 168 | $(eval CID=`docker inspect $(CONTAINERNAME) --format '{{ .Id }}'`) 169 | cat $(CGROUPFS)/cpu,cpuacct/docker/$(CID)/cpu.stat 170 | sudo $(BCC)/runqlat -p `pidof java` -m 5 1 171 | sudo $(BCC)/cpudist -p `pidof java` -m 5 1 172 | sudo $(BCC)/cpudist -O -p `pidof java` -m 5 1 173 | 174 | throttlerecord2: 175 | sudo docker exec $(CONTAINERNAME) sh -c 'cd perf-map-agent/out && \ 176 | java -cp attach-main.jar:$$JAVA_HOME/lib/tools.jar \ 177 | net.virtualvoid.perf.AttachOnce `pidof java` dottedclass' 178 | sudo $(BCC)/offcputime -f -p `pidof java` 5 > /tmp/java.stacks 179 | $(flame-graph) 180 | 181 | profilecontainer: 182 | # The container inherits these settings from the host 183 | ./perf-std-user.sh 184 | sudo docker exec $(CONTAINERNAME) sh -c '/async-profiler/profiler.sh \ 185 | -d 10 -o collapsed -f /app/java.stacks `pidof java`' 186 | mv out/java.stacks /tmp/java.stacks 187 | $(flame-graph) 188 | 189 | clean: 190 | rm -rf out 191 | -------------------------------------------------------------------------------- /perros/Register.java: -------------------------------------------------------------------------------- 1 | package perros; 2 | 3 | import java.io.IOException; 4 | import java.util.Arrays; 5 | import java.util.Map; 6 | import java.util.regex.Matcher; 7 | import java.util.regex.Pattern; 8 | 9 | class RegisterHandler { 10 | @Post 11 | public void handle(Request request) throws IOException { 12 | String username = request.bodyJson().get("username").toString(); 13 | String email = request.bodyJson().get("email").toString(); 14 | String password = request.bodyJson().get("password").toString(); 15 | 16 | if (!validateEmail(email) || !validatePassword(password)) { 17 | request.badRequest(); 18 | return; 19 | } 20 | 21 | request.finish(201); // Created 22 | } 23 | 24 | private boolean validateEmail(String email) { 25 | Pattern pattern = Pattern.compile("[A-Za-z0-9]+@[A-Za-z0-9]+\\.com"); 26 | Matcher matcher = pattern.matcher(email); 27 | return matcher.find(); 28 | } 29 | 30 | private boolean validatePassword(String password) { 31 | Pattern pattern = Pattern.compile("([A-Za-z0-9]+[A-Za-z0-9]+)+[!^&*#]"); 32 | Matcher matcher = pattern.matcher(password); 33 | return matcher.find(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /perros/Stats.java: -------------------------------------------------------------------------------- 1 | package perros; 2 | 3 | import java.io.StringWriter; 4 | import javax.xml.parsers.*; 5 | import javax.xml.transform.*; 6 | import javax.xml.transform.dom.*; 7 | import javax.xml.transform.stream.*; 8 | import org.w3c.dom.*; 9 | 10 | class StatsHandler { 11 | @Get 12 | public void handle(Request request) throws Exception { 13 | int pages = Integer.parseInt(request.queryParams().get("pages")); 14 | String stats = generateStats(pages); 15 | request.finish(200, stats); 16 | } 17 | 18 | private String generateStats(int pages) throws Exception { 19 | DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 20 | DocumentBuilder db = dbf.newDocumentBuilder(); 21 | Document doc = db.newDocument(); 22 | 23 | Element root = doc.createElement("stats"); 24 | doc.appendChild(root); 25 | 26 | for (int i = 0; i < pages; ++i) { 27 | Element stat = doc.createElement("stat" + Integer.toString(i)); 28 | stat.appendChild(doc.createTextNode("1")); 29 | root.appendChild(stat); 30 | } 31 | 32 | Transformer tr = TransformerFactory.newInstance().newTransformer(); 33 | tr.setOutputProperty(OutputKeys.INDENT, "yes"); 34 | tr.setOutputProperty(OutputKeys.METHOD, "xml"); 35 | tr.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); 36 | tr.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4"); 37 | StringWriter writer = new StringWriter(); 38 | tr.transform(new DOMSource(doc), new StreamResult(writer)); 39 | 40 | return writer.getBuffer().toString(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /perros/Users.java: -------------------------------------------------------------------------------- 1 | package perros; 2 | 3 | import java.io.IOException; 4 | import java.sql.*; 5 | import java.util.ArrayList; 6 | 7 | class User { 8 | private int id; 9 | private String name; 10 | 11 | public static User load(int id) throws Exception { 12 | User user = null; 13 | Statement stmt = DAL.getConnection().createStatement(); 14 | ResultSet res = stmt.executeQuery( 15 | "select * from users where id = " + id); 16 | if (res.next()) { 17 | user = new User(); 18 | user.id = res.getInt("id"); 19 | user.name = res.getString("name"); 20 | } 21 | res.close(); 22 | stmt.close(); 23 | return user; 24 | } 25 | 26 | public Iterable loadProducts() throws Exception { 27 | ArrayList products = new ArrayList<>(); 28 | Statement stmt = DAL.getConnection().createStatement(); 29 | ResultSet res = stmt.executeQuery( 30 | "select id from products where userid = " + id); 31 | while (res.next()) { 32 | Product prod = Product.load(res.getInt("id")); 33 | products.add(prod); 34 | } 35 | res.close(); 36 | stmt.close(); 37 | return products; 38 | } 39 | 40 | public int getId() { return id; } 41 | public String getName() { return name; } 42 | } 43 | 44 | class Product { 45 | private int id; 46 | private int userId; 47 | private String name; 48 | private String description; 49 | private double price; 50 | 51 | public static Product load(int id) throws Exception { 52 | Product product = null; 53 | Statement stmt = DAL.getConnection().createStatement(); 54 | ResultSet res = stmt.executeQuery("call getproduct(" + id + ")"); 55 | if (res.next()) { 56 | product = new Product(); 57 | product.id = res.getInt("id"); 58 | product.userId = res.getInt("userid"); 59 | product.name = res.getString("name"); 60 | product.description = res.getString("description"); 61 | product.price = res.getDouble("price"); 62 | } 63 | res.close(); 64 | stmt.close(); 65 | return product; 66 | } 67 | 68 | public int getId() { return id; } 69 | public String getName() { return name; } 70 | public String getDescription() { return description; } 71 | public double getPrice() { return price; } 72 | } 73 | 74 | class DAL { 75 | private static Connection connection; 76 | private static final String CONN_STRING = 77 | "jdbc:mysql://localhost/acme?user=newuser&password=password"; 78 | 79 | public static Connection getConnection() { 80 | return connection; 81 | } 82 | 83 | public static void init() throws ClassNotFoundException, SQLException { 84 | Class.forName("com.mysql.jdbc.Driver"); 85 | connection = DriverManager.getConnection(CONN_STRING); 86 | } 87 | } 88 | 89 | class UsersHandler { 90 | public UsersHandler() throws Exception { 91 | DAL.init(); 92 | } 93 | 94 | @Get 95 | public void handle(Request request) throws Exception { 96 | int userId = Integer.parseInt(request.queryParams().get("user")); 97 | User user = User.load(userId); 98 | String result = "{ \"user\": \"" + user.getName() + "\", \"products\": ["; 99 | for (Product product : user.loadProducts()) { 100 | result += " { \"id\": " + product.getId() + ", "; 101 | result += " \"name\": \"" + product.getName() + "\", "; 102 | result += " \"price\": " + product.getPrice() + " }, "; 103 | } 104 | result = result.substring(0, result.length() - 2); 105 | result += " ] }"; 106 | request.finish(200, result); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /perros/agent.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | class monitor_tracker { 6 | public: 7 | static monitor_tracker& instance() { 8 | static monitor_tracker s_instance; 9 | return s_instance; 10 | } 11 | void attach(JavaVM *vm); 12 | void enter_start(jthread thread, jobject object); 13 | void enter_end(jthread thread, jobject object); 14 | void dump(); 15 | private: 16 | bool attached_ = false; 17 | }; 18 | 19 | void JNICALL MonitorContendedEnter(jvmtiEnv *jvmti_env, JNIEnv *jni_env, 20 | jthread thread, jobject object) { 21 | monitor_tracker::instance().enter_start(thread, object); 22 | } 23 | 24 | void JNICALL MonitorContendedEntered(jvmtiEnv *jvmti_env, JNIEnv *jni_env, 25 | jthread thread, jobject object) { 26 | monitor_tracker::instance().enter_end(thread, object); 27 | } 28 | 29 | void monitor_tracker::enter_start(jthread thread, jobject object) { 30 | // TODO 31 | } 32 | 33 | void monitor_tracker::enter_end(jthread thread, jobject object) { 34 | // TODO 35 | } 36 | 37 | void monitor_tracker::dump() { 38 | // TODO 39 | std::cout << "[AGENT] dumping monitor contention stats\n"; 40 | } 41 | 42 | void monitor_tracker::attach(JavaVM *vm) { 43 | if (attached_) 44 | return; 45 | 46 | attached_ = true; 47 | std::cout << "[AGENT] attaching to monitor events\n"; 48 | 49 | jvmtiEnv *jvmti; 50 | vm->GetEnv((void**)&jvmti, JVMTI_VERSION_1_0); 51 | 52 | jvmtiCapabilities capabilities = {0}; 53 | capabilities.can_generate_monitor_events = 1; 54 | jvmti->AddCapabilities(&capabilities); 55 | 56 | jvmtiEventCallbacks callbacks = {0}; 57 | callbacks.MonitorContendedEnter = MonitorContendedEnter; 58 | callbacks.MonitorContendedEntered = MonitorContendedEntered; 59 | jvmti->SetEventCallbacks(&callbacks, sizeof(callbacks)); 60 | 61 | jvmti->SetEventNotificationMode(JVMTI_ENABLE, 62 | JVMTI_EVENT_MONITOR_CONTENDED_ENTER, 63 | NULL); 64 | jvmti->SetEventNotificationMode(JVMTI_ENABLE, 65 | JVMTI_EVENT_MONITOR_CONTENDED_ENTERED, 66 | NULL); 67 | } 68 | 69 | void do_options(JavaVM *vm, char *options) { 70 | if (std::string(options) == "dump") { 71 | monitor_tracker::instance().dump(); 72 | } else { 73 | monitor_tracker::instance().attach(vm); 74 | } 75 | } 76 | 77 | JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved) { 78 | do_options(vm, options); 79 | return JNI_OK; 80 | } 81 | 82 | JNIEXPORT void JNICALL Agent_OnUnload(JavaVM *vm) { 83 | } 84 | 85 | JNIEXPORT jint JNICALL Agent_OnAttach(JavaVM* vm, char* options, void* reserved) { 86 | do_options(vm, options); 87 | return JNI_OK; 88 | } 89 | 90 | -------------------------------------------------------------------------------- /perros/contmonitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from bcc import BPF, USDT 4 | import argparse 5 | 6 | parser = argparse.ArgumentParser( 7 | description="Trace JVM monitor contention events and dump a summary", 8 | formatter_class=argparse.RawDescriptionHelpFormatter) 9 | parser.add_argument("pid", type=int, help="the Java process id") 10 | args = parser.parse_args() 11 | 12 | usdt = USDT(pid=args.pid) 13 | usdt.enable_probe("monitor__contended__enter", "trace_enter") 14 | usdt.enable_probe("monitor__contended__entered", "trace_entered") 15 | 16 | bpf = BPF(text=""" 17 | int trace_enter(struct pt_regs *ctx) { 18 | // TODO 19 | return 0; 20 | } 21 | 22 | int trace_entered(struct pt_regs *ctx) { 23 | // TODO 24 | return 0; 25 | } 26 | """, usdt_contexts=[usdt]) 27 | 28 | # TODO 29 | -------------------------------------------------------------------------------- /perros/jattach-container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -ne 1 ]; then 4 | echo "USAGE: $0 " 5 | exit 1 6 | fi 7 | 8 | sudo nsenter -t $1 -m touch /proc/1/cwd/.attach_pid1 9 | sudo kill -SIGQUIT $1 10 | 11 | # The JVM should have responded by creating /tmp/.java_pid1, a UNIX domain 12 | # socket that jattach can now attach to. 13 | 14 | echo "Now run jattach 1 [arguments]" 15 | -------------------------------------------------------------------------------- /perros/perf-std-user.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | sudo bash -c 'echo 1 > /proc/sys/kernel/perf_event_paranoid' 4 | sudo bash -c 'echo 0 > /proc/sys/kernel/kptr_restrict' 5 | -------------------------------------------------------------------------------- /perros/post-auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "username": "dave", 3 | "password": "123456" 4 | } 5 | -------------------------------------------------------------------------------- /perros/post-register-bad.json: -------------------------------------------------------------------------------- 1 | { 2 | "username": "mike", 3 | "email": "mike@gmail.com", 4 | "password": "longnicecomplicatedpassword" 5 | } 6 | -------------------------------------------------------------------------------- /perros/post-register.json: -------------------------------------------------------------------------------- 1 | { 2 | "username": "kate", 3 | "email": "kate@gmail.com", 4 | "password": "longnicecomplicatedpassword123!" 5 | } 6 | -------------------------------------------------------------------------------- /perros/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # 3 | # Assumes most dependencies have already been installed on this image 4 | # by the setup-fedora.sh script. Specifically, these labs depend on: 5 | # bcc 6 | # perf 7 | # perf-tools 8 | # FlameGraph 9 | # And some miscellaneous utils: pidstat, curl, etc. 10 | 11 | # 12 | # Taken from https://docs.docker.com/engine/installation/linux/docker-ce/fedora/#install-docker-ce 13 | # 14 | sudo dnf install -y dnf-plugins-core 15 | sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo 16 | sudo dnf makecache -y fast 17 | sudo dnf install -y docker-ce 18 | sudo systemctl start docker 19 | -------------------------------------------------------------------------------- /perros/usersbench.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | for user in `seq 1 50`; do 4 | begin=$(date +%s%N) 5 | curl -v "${1}/users?user=${user}" >/dev/null 2>&1 6 | end=$(date +%s%N) 7 | echo "user ${user} took $(((end-begin)/1000000)) ms" 8 | done 9 | -------------------------------------------------------------------------------- /pg-slow.sql: -------------------------------------------------------------------------------- 1 | create table if not exists test (value integer); 2 | do 3 | $do$ 4 | begin 5 | for i in 1..1000000 loop 6 | insert into test values (i); 7 | end loop; 8 | end $do$; 9 | 10 | select count(*) from test where value % 17 = 0; 11 | select count(*) from test where (value * 3) % 17 = 0; 12 | select count(*) from test; 13 | select avg(value) from test; 14 | select max(value) from test where value % 3 = 1; 15 | -------------------------------------------------------------------------------- /primes.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int is_trivial_prime(int n) 5 | { 6 | return n <= 2; 7 | } 8 | 9 | int is_divisible(int n, int d) 10 | { 11 | return n % d == 0; 12 | } 13 | 14 | int is_prime(int n) 15 | { 16 | if (is_trivial_prime(n)) 17 | return 1; 18 | if (is_divisible(n, 2)) 19 | return 0; 20 | for (int d = 3; d < n; d += 2) 21 | if (is_divisible(n, d)) 22 | return 0; 23 | return 1; 24 | } 25 | 26 | int main() 27 | { 28 | int count = 0; 29 | #pragma omp parallel for reduction(+:count) 30 | for (int n = 2; n < 200000; ++n) 31 | if (is_prime(n)) 32 | ++count; 33 | printf("Primes found: %d\n", count); 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /server.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void read_config() { 5 | int success = 0; 6 | while (!success) { 7 | FILE *fp; 8 | fp = fopen("/etc/tracing-server-example.conf", "r"); 9 | if (fp) { 10 | success = 1; 11 | printf("[*] Read configuration successfully.\n"); 12 | fclose(fp); 13 | } else { 14 | usleep(1); 15 | } 16 | } 17 | } 18 | 19 | int main() { 20 | printf("[*] Server starting...\n"); 21 | read_config(); 22 | printf("[*] Server started successfully.\n"); 23 | while (1) { 24 | printf("[*] Processing request... "); 25 | sleep(1); 26 | printf("DONE.\n"); 27 | } 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var server = http.createServer(function (req, res) { 3 | res.end('Hello, world!'); 4 | }); 5 | server.listen(8080); 6 | -------------------------------------------------------------------------------- /setuidsnoop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # @lint-avoid-python-3-compatibility-imports 3 | # 4 | # setuidsnoop Trace uid changes issued by the setuid() syscall. 5 | # For Linux, uses BCC, eBPF. Embedded C. 6 | # 7 | # USAGE: setuidsnoop [-h] [-p PID] 8 | # 9 | # Copyright (c) 2016 Brendan Gregg, Sasha Goldshtein. 10 | # Licensed under the Apache License, Version 2.0 (the "License") 11 | # 12 | # 20-Sep-2015 Brendan Gregg Created killsnoop. 13 | # 19-Feb-2016 Allan McAleavy Migrated to BPF_PERF_OUTPUT. 14 | # 16-Oct-2016 Sasha Goldshtein Modified to setuidsnoop. 15 | 16 | from __future__ import print_function 17 | from bcc import BPF 18 | import argparse 19 | from time import strftime 20 | import ctypes as ct 21 | 22 | # arguments 23 | examples = """examples: 24 | ./setuidsnoop # trace all setuid() calls 25 | ./setuidsnoop -p 181 # only trace PID 181 26 | """ 27 | parser = argparse.ArgumentParser( 28 | description="Trace uid changes issued by the setuid() syscall", 29 | formatter_class=argparse.RawDescriptionHelpFormatter, 30 | epilog=examples) 31 | parser.add_argument("-p", "--pid", 32 | help="trace this PID only") 33 | args = parser.parse_args() 34 | debug = 0 35 | 36 | # define BPF program 37 | bpf_text = """ 38 | #include 39 | #include 40 | 41 | struct val_t { 42 | u64 pid; 43 | u32 uid; 44 | char comm[TASK_COMM_LEN]; 45 | }; 46 | 47 | struct data_t { 48 | u64 pid; 49 | u32 uid; 50 | int ret; 51 | char comm[TASK_COMM_LEN]; 52 | }; 53 | 54 | BPF_HASH(infotmp, u32, struct val_t); 55 | BPF_PERF_OUTPUT(events); 56 | 57 | int kprobe__sys_setuid(struct pt_regs *ctx, u32 uid) 58 | { 59 | u32 pid = bpf_get_current_pid_tgid(); 60 | FILTER 61 | 62 | struct val_t val = {.pid = pid}; 63 | if (bpf_get_current_comm(&val.comm, sizeof(val.comm)) == 0) { 64 | val.uid = uid; 65 | infotmp.update(&pid, &val); 66 | } 67 | 68 | return 0; 69 | }; 70 | 71 | int kretprobe__sys_setuid(struct pt_regs *ctx) 72 | { 73 | struct data_t data = {}; 74 | struct val_t *valp; 75 | u32 pid = bpf_get_current_pid_tgid(); 76 | 77 | valp = infotmp.lookup(&pid); 78 | if (valp == 0) { 79 | // missed entry 80 | return 0; 81 | } 82 | 83 | bpf_probe_read(&data.comm, sizeof(data.comm), valp->comm); 84 | data.pid = pid; 85 | data.uid = valp->uid; 86 | data.ret = PT_REGS_RC(ctx); 87 | 88 | events.perf_submit(ctx, &data, sizeof(data)); 89 | infotmp.delete(&pid); 90 | 91 | return 0; 92 | } 93 | """ 94 | if args.pid: 95 | bpf_text = bpf_text.replace('FILTER', 96 | 'if (pid != %s) { return 0; }' % args.pid) 97 | else: 98 | bpf_text = bpf_text.replace('FILTER', '') 99 | if debug: 100 | print(bpf_text) 101 | 102 | # initialize BPF 103 | b = BPF(text=bpf_text) 104 | 105 | TASK_COMM_LEN = 16 # linux/sched.h 106 | 107 | class Data(ct.Structure): 108 | _fields_ = [ 109 | ("pid", ct.c_ulonglong), 110 | ("uid", ct.c_uint), 111 | ("ret", ct.c_int), 112 | ("comm", ct.c_char * TASK_COMM_LEN) 113 | ] 114 | 115 | # header 116 | print("%-9s %-6s %-16s %-6s %s" % ( 117 | "TIME", "PID", "COMM", "UID", "RESULT")) 118 | 119 | # process event 120 | def print_event(cpu, data, size): 121 | event = ct.cast(data, ct.POINTER(Data)).contents 122 | print("%-9s %-6d %-16s %-6d %d" % (strftime("%H:%M:%S"), 123 | event.pid, event.comm, event.uid, event.ret)) 124 | 125 | # loop with callback to print_event 126 | b["events"].open_perf_buffer(print_event) 127 | while 1: 128 | b.kprobe_poll() 129 | -------------------------------------------------------------------------------- /slowy/App.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goldshtn/linux-tracing-workshop/a3a34fe7242a16047f277f28c1e5c1edef29340d/slowy/App.class -------------------------------------------------------------------------------- /slowy/Slowy.java: -------------------------------------------------------------------------------- 1 | package slowy; 2 | 3 | class App { 4 | 5 | private static boolean isDivisible(int n, int d) { 6 | return n % d == 0; 7 | } 8 | 9 | private static boolean isSimplePrime(int n) { 10 | return n <= 2; 11 | } 12 | 13 | private static boolean isPrime(int n) { 14 | if (isSimplePrime(n)) return true; 15 | for (int d = 3; d < n; d += 2) { 16 | if (isDivisible(n, d)) return false; 17 | } 18 | return true; 19 | } 20 | 21 | public static void main(String[] args) throws java.io.IOException { 22 | System.out.println("Press ENTER to start."); 23 | System.in.read(); 24 | int count = 0; 25 | for (int n = 2; n < 200000; ++n) { 26 | if (isPrime(n)) ++count; 27 | } 28 | System.out.println(String.format("Primes found: %d", count)); 29 | System.out.println("Press ENTER to exit."); 30 | System.in.read(); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /whales/Makefile: -------------------------------------------------------------------------------- 1 | SOURCE := $(shell readlink -f ..) 2 | 3 | define parprimes 4 | sudo docker run --name primes -d --rm -v $(SOURCE):/src gcc:4.9 \ 5 | sh -c "gcc -g -std=c11 -fno-omit-frame-pointer -fopenmp \ 6 | /src/parprimes.c -o /parprimes && \ 7 | yes | /parprimes 4 100000000" 8 | endef 9 | 10 | define continue 11 | @read -p "Press any key to continue..." -n 1 -r 12 | endef 13 | 14 | use: 15 | sudo docker run --name stress -d --rm progrium/stress --cpu 2 16 | sudo timeout 5 docker stats || true 17 | $(call continue) 18 | systemd-cgtop -n 5 -d 1 19 | $(call continue) 20 | sudo docker exec -it stress top -n 5 -d 1 21 | sudo docker kill stress 22 | 23 | profile: 24 | $(call parprimes) 25 | sudo perf record -a -g -F 97 -- sleep 10 26 | sudo perf report --stdio 27 | sudo docker kill primes 28 | 29 | profilegood: 30 | $(call parprimes) 31 | sudo /usr/share/bcc/tools/profile -F 97 10 32 | sudo docker kill primes 33 | 34 | io: 35 | sudo docker run --name stress -d --rm \ 36 | --device-write-iops /dev/xvda:10 progrium/stress --hdd 1 37 | sudo /usr/share/bcc/tools/biolatency 10 1 38 | $(call continue) 39 | sudo timeout 10 /usr/share/bcc/tools/fileslower 10 || true 40 | sudo docker kill stress 41 | -------------------------------------------------------------------------------- /wordcount.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | class word_counter; 11 | 12 | class input_reader 13 | { 14 | private: 15 | std::shared_ptr counter_; 16 | public: 17 | void set_counter(std::shared_ptr counter); 18 | std::string next_filename(); 19 | }; 20 | 21 | class word_counter 22 | { 23 | private: 24 | std::vector words_; 25 | bool done_ = false; 26 | std::shared_ptr reader_; 27 | public: 28 | word_counter(std::shared_ptr reader); 29 | bool done() const; 30 | std::map word_count(); 31 | }; 32 | 33 | void input_reader::set_counter(std::shared_ptr counter) 34 | { 35 | counter_ = counter; 36 | } 37 | 38 | std::string input_reader::next_filename() 39 | { 40 | std::string input; 41 | std::cout << "filename or 'quit'> "; 42 | std::getline(std::cin, input); 43 | if (input == "quit") 44 | return ""; 45 | return input; 46 | } 47 | 48 | word_counter::word_counter(std::shared_ptr reader) : reader_(reader) 49 | { 50 | } 51 | 52 | bool word_counter::done() const 53 | { 54 | return done_; 55 | } 56 | 57 | std::map word_counter::word_count() 58 | { 59 | std::string filename = reader_->next_filename(); 60 | if (filename == "") 61 | { 62 | done_ = true; 63 | return {}; 64 | } 65 | std::ifstream in(filename); 66 | std::copy(std::istream_iterator(in), 67 | std::istream_iterator(), 68 | std::back_inserter(words_)); 69 | 70 | std::map counts; 71 | for (auto const& word: words_) { 72 | auto it = counts.find(word); 73 | if (it == counts.end()) 74 | counts[word] = 1; 75 | else 76 | it->second++; 77 | } 78 | return counts; 79 | } 80 | 81 | int main() 82 | { 83 | bool done = false; 84 | while (!done) 85 | { 86 | auto reader = std::make_shared(); 87 | auto counter = std::make_shared(reader); 88 | reader->set_counter(counter); 89 | auto counts = counter->word_count(); 90 | done = counter->done(); 91 | for (auto const& wc : counts) 92 | { 93 | std::cout << wc.first << " " << wc.second << '\n'; 94 | } 95 | } 96 | return 0; 97 | } 98 | --------------------------------------------------------------------------------