├── modules ├── __init__.py ├── sysfork.bpf ├── biolatency.bpf ├── pcpbcc.py ├── sysfork.py ├── biotop.bpf ├── biolatency.py ├── biotop.py ├── tcplife.bpf └── tcplife.py ├── pmrep.conf ├── Remove ├── Install ├── bcc.conf ├── example.txt ├── pmdabcc.1 ├── pmdabcc.python └── README.md /modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pmrep.conf: -------------------------------------------------------------------------------- 1 | [bcc-example] 2 | timestamp = yes 3 | interval = 5s 4 | proc.fd.count = ,,,, 5 | proc.io.write_bytes = ,,kB/s,, 6 | bcc.proc.io.perdev = ,,kB/s,, 7 | bcc.proc.io.net.tcp.rx = ,,MB,raw, 8 | bcc.proc.io.net.tcp.tx = ,,MB,raw, 9 | -------------------------------------------------------------------------------- /modules/sysfork.bpf: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Brendan Gregg. 2 | // Licensed under the Apache License, Version 2.0 (the "License") 3 | 4 | #include 5 | 6 | enum stat_types { 7 | S_COUNT = 1, 8 | S_MAXSTAT 9 | }; 10 | 11 | BPF_ARRAY(stats, u64, S_MAXSTAT); 12 | 13 | static void stats_increment(int key) { 14 | u64 *leaf = stats.lookup(&key); 15 | if (leaf) (*leaf)++; 16 | } 17 | 18 | void do_count(struct pt_regs *ctx) { stats_increment(S_COUNT); } 19 | -------------------------------------------------------------------------------- /Remove: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Copyright (c) 2017 Red Hat. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # Remove the BCC PMDA 16 | # 17 | 18 | . $PCP_DIR/etc/pcp.env 19 | . $PCP_SHARE_DIR/lib/pmdaproc.sh 20 | 21 | iam=bcc 22 | 23 | pmdaSetup 24 | pmdaRemove 25 | exit 26 | -------------------------------------------------------------------------------- /Install: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Copyright (c) 2017 Red Hat. 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # Install the BCC PMDA 16 | # 17 | 18 | . $PCP_DIR/etc/pcp.env 19 | . $PCP_SHARE_DIR/lib/pmdaproc.sh 20 | 21 | iam=bcc 22 | domain=149 23 | python_opt=true 24 | daemon_opt=false 25 | check_delay=5 26 | 27 | pmdaSetup 28 | pmdaInstall 29 | exit 30 | -------------------------------------------------------------------------------- /bcc.conf: -------------------------------------------------------------------------------- 1 | # 2 | # PCP BCC PMDA configuration file - see pmdabcc(1) and PMDA(3) 3 | # 4 | 5 | # NB. By default PMCD will wait for 5 seconds for a PMDA to start. 6 | # Compiling many modules may take longer than that, causing the PMDA 7 | # to be terminated eventually. Either use less modules or adjust the 8 | # check_delay in the Install script to increase the timeout as needed. 9 | 10 | # NB. Since all the modules are part of the same process, modules may 11 | # not attach to the same kprobes as others or latter ones will fail. 12 | # Either disable overlapping modules or create new combined modules. 13 | # Currently overlapping modules are: biolatency/biotop. 14 | 15 | [pmda] 16 | modules = biolatency,sysfork,tcplife 17 | #modules = biotop,sysfork,tcplife 18 | prefix = bcc. 19 | 20 | [biolatency] 21 | module = biolatency 22 | cluster = 0 23 | queued = False 24 | 25 | [biotop] 26 | module = biotop 27 | cluster = 1 28 | debug = True 29 | 30 | [sysfork] 31 | module = sysfork 32 | cluster = 2 33 | 34 | [tcplife] 35 | module = tcplife 36 | cluster = 3 37 | -------------------------------------------------------------------------------- /modules/biolatency.bpf: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Brendan Gregg. 2 | // Licensed under the Apache License, Version 2.0 (the "License") 3 | 4 | #include 5 | #include 6 | 7 | typedef struct disk_key { 8 | char disk[DISK_NAME_LEN]; 9 | u64 slot; 10 | } disk_key_t; 11 | BPF_HASH(start, struct request *); 12 | BPF_HISTOGRAM(dist); 13 | 14 | // time block I/O 15 | int trace_req_start(struct pt_regs *ctx, struct request *req) 16 | { 17 | u64 ts = bpf_ktime_get_ns(); 18 | start.update(&req, &ts); 19 | return 0; 20 | } 21 | 22 | // output 23 | int trace_req_completion(struct pt_regs *ctx, struct request *req) 24 | { 25 | u64 *tsp, delta; 26 | 27 | // fetch timestamp and calculate delta 28 | tsp = start.lookup(&req); 29 | if (tsp == 0) { 30 | return 0; // missed issue 31 | } 32 | delta = bpf_ktime_get_ns() - *tsp; 33 | delta /= 1000; // usec 34 | 35 | // store as histogram 36 | dist.increment(bpf_log2l(delta)); 37 | 38 | start.delete(&req); 39 | return 0; 40 | } 41 | -------------------------------------------------------------------------------- /example.txt: -------------------------------------------------------------------------------- 1 | # cat test.conf 2 | [bcc-example] 3 | timestamp = yes 4 | interval = 5s 5 | proc.fd.count = ,,,, 6 | proc.io.write_bytes = ,,kB/s,, 7 | bcc.proc.io.perdev = ,,kB/s,, 8 | bcc.proc.io.net.tcp.rx = ,,MB,raw, 9 | bcc.proc.io.net.tcp.tx = ,,MB,raw, 10 | # pmrep -c ./test.conf -i '.*15751.*' --samples 10 :bcc-example 11 | p.f.count p.i.write_bytes b.p.i.perdev b.p.i.perdev b.p.i.n.t.rx b.p.i.n.t.tx 12 | 015751 /u 015751 /usr/bin vdb::015751 vda::015751 015751 015751 13 | count kB/s kB/s kB/s MB MB 14 | 10:00:04 23 N/A N/A N/A 166.011 0.069 15 | 10:00:09 23 2234.501 1891.713 0.000 166.011 0.069 16 | 10:00:14 23 2109.045 953.655 0.000 174.980 0.075 17 | 10:00:19 23 2137.472 1192.711 0.000 174.980 0.075 18 | 10:00:24 23 1987.648 1510.867 0.000 174.980 0.075 19 | 10:00:29 23 2099.366 1712.808 0.000 185.435 0.081 20 | 10:00:34 23 2042.308 1251.223 0.000 185.435 0.081 21 | 10:00:39 23 1708.094 1004.761 0.000 266.203 0.087 22 | 10:00:44 23 1524.975 1188.127 0.000 266.203 0.087 23 | 10:00:49 23 1823.358 1123.660 0.000 279.981 0.093 24 | # 25 | -------------------------------------------------------------------------------- /modules/pcpbcc.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2017-2018 Marko Myllynen 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | """ PCP BCC PMDA module base class """ 15 | 16 | # pylint: disable=too-many-instance-attributes 17 | class PCPBCCBase(object): 18 | """ PCP BCC Base Class """ 19 | def __init__(self, module, config, log, err): 20 | """ Constructor """ 21 | self._who = module 22 | self._log = log 23 | self._err = err 24 | 25 | self.bpf = None 26 | self.insts = {} 27 | self.items = [] 28 | 29 | self.pmdaIndom = None # pylint: disable=invalid-name 30 | self.config = config 31 | self.debug = False 32 | 33 | for opt in self.config.options(self._who): 34 | if opt == 'debug': 35 | self.debug = self.config.getboolean(self._who, opt) 36 | 37 | if self.debug: 38 | self.log("Debug logging enabled.") 39 | 40 | def log(self, msg): 41 | """ Log a message """ 42 | self._log(self._who + ": " + msg) 43 | 44 | def err(self, msg): 45 | """ Log an error """ 46 | self._err(self._who + ": " + msg) 47 | 48 | def metrics(self): 49 | """ Get metric definitions """ 50 | raise NotImplementedError 51 | 52 | def helpers(self, pmdaIndom): # pylint: disable=invalid-name 53 | """ Register helper function references """ 54 | self.pmdaIndom = pmdaIndom 55 | 56 | def compile(self): 57 | """ Compile BPF """ 58 | raise NotImplementedError 59 | 60 | def refresh(self): 61 | """ Refresh BPF data """ 62 | raise NotImplementedError 63 | 64 | def bpfdata(self, item, inst): 65 | """ Return BPF data as PCP metric value """ 66 | raise NotImplementedError 67 | 68 | def cleanup(self): 69 | """ Clean up at exit """ 70 | if self.bpf is not None: 71 | self.bpf.cleanup() 72 | self.bpf = None 73 | self.log("BPF/kprobes detached.") 74 | -------------------------------------------------------------------------------- /modules/sysfork.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2017-2018 Marko Myllynen 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | """ PCP BCC PMDA sysfork module """ 15 | 16 | # pylint: disable=invalid-name 17 | 18 | from ctypes import c_int 19 | from bcc import BPF 20 | 21 | from modules.pcpbcc import PCPBCCBase 22 | from pcp.pmapi import pmUnits 23 | from cpmapi import PM_TYPE_U64, PM_SEM_COUNTER, PM_COUNT_ONE 24 | from cpmapi import PM_ERR_AGAIN 25 | 26 | # 27 | # BPF program 28 | # 29 | bpf_src = "modules/sysfork.bpf" 30 | 31 | # 32 | # PCP BCC PMDA constants 33 | # 34 | MODULE = 'sysfork' 35 | METRIC = 'proc.sysfork' 36 | units_count = pmUnits(0, 0, 1, 0, 0, PM_COUNT_ONE) 37 | 38 | # 39 | # PCP BCC Module 40 | # 41 | class PCPBCCModule(PCPBCCBase): 42 | """ PCP BCC biotop module """ 43 | def __init__(self, config, log, err): 44 | """ Constructor """ 45 | PCPBCCBase.__init__(self, MODULE, config, log, err) 46 | 47 | self.value = 0 48 | 49 | self.log("Initialized.") 50 | 51 | def metrics(self): 52 | """ Get metric definitions """ 53 | name = METRIC 54 | self.items.append( 55 | # Name - reserved - type - semantics - units - help 56 | (name, None, PM_TYPE_U64, PM_SEM_COUNTER, units_count, 'fork rate'), 57 | ) 58 | return False, self.items 59 | 60 | def compile(self): 61 | """ Compile BPF """ 62 | try: 63 | self.bpf = BPF(src_file=bpf_src) 64 | self.bpf.attach_kprobe(event="sched_fork", fn_name="do_count") 65 | self.log("Compiled.") 66 | except Exception as error: # pylint: disable=broad-except 67 | self.err(str(error)) 68 | self.err("Module NOT active!") 69 | self.bpf = None 70 | 71 | def refresh(self): 72 | """ Refresh BPF data """ 73 | if self.bpf is None: 74 | return 75 | 76 | self.value = self.bpf["stats"][c_int(1)].value 77 | 78 | def bpfdata(self, item, inst): 79 | """ Return BPF data as PCP metric value """ 80 | try: 81 | return [self.value, 1] 82 | except Exception: # pylint: disable=broad-except 83 | return [PM_ERR_AGAIN, 0] 84 | -------------------------------------------------------------------------------- /modules/biotop.bpf: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Netflix, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License") 3 | 4 | #include 5 | #include 6 | 7 | // for saving process info by request 8 | struct who_t { 9 | u32 pid; 10 | char name[TASK_COMM_LEN]; 11 | }; 12 | 13 | // the key for the output summary 14 | struct info_t { 15 | u32 pid; 16 | int rwflag; 17 | int major; 18 | int minor; 19 | char name[TASK_COMM_LEN]; 20 | }; 21 | 22 | // the value of the output summary 23 | struct val_t { 24 | u64 bytes; 25 | u64 us; 26 | u32 io; 27 | }; 28 | 29 | BPF_HASH(start, struct request *); 30 | BPF_HASH(whobyreq, struct request *, struct who_t); 31 | BPF_HASH(counts, struct info_t, struct val_t); 32 | 33 | // cache PID and comm by-req 34 | int trace_pid_start(struct pt_regs *ctx, struct request *req) 35 | { 36 | struct who_t who = {}; 37 | 38 | if (bpf_get_current_comm(&who.name, sizeof(who.name)) == 0) { 39 | who.pid = bpf_get_current_pid_tgid(); 40 | whobyreq.update(&req, &who); 41 | } 42 | 43 | return 0; 44 | } 45 | 46 | // time block I/O 47 | int trace_req_start(struct pt_regs *ctx, struct request *req) 48 | { 49 | u64 ts; 50 | 51 | ts = bpf_ktime_get_ns(); 52 | start.update(&req, &ts); 53 | 54 | return 0; 55 | } 56 | 57 | // output 58 | int trace_req_completion(struct pt_regs *ctx, struct request *req) 59 | { 60 | u64 *tsp; 61 | 62 | // fetch timestamp and calculate delta 63 | tsp = start.lookup(&req); 64 | if (tsp == 0) { 65 | return 0; // missed tracing issue 66 | } 67 | 68 | struct who_t *whop; 69 | struct val_t *valp, zero = {}; 70 | u64 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000; 71 | 72 | // setup info_t key 73 | struct info_t info = {}; 74 | info.major = req->rq_disk->major; 75 | info.minor = req->rq_disk->first_minor; 76 | /* 77 | * The following deals with a kernel version change (in mainline 4.7, although 78 | * it may be backported to earlier kernels) with how block request write flags 79 | * are tested. We handle both pre- and post-change versions here. Please avoid 80 | * kernel version tests like this as much as possible: they inflate the code, 81 | * test, and maintenance burden. 82 | */ 83 | #ifdef REQ_WRITE 84 | info.rwflag = !!(req->cmd_flags & REQ_WRITE); 85 | #elif defined(REQ_OP_SHIFT) 86 | info.rwflag = !!((req->cmd_flags >> REQ_OP_SHIFT) == REQ_OP_WRITE); 87 | #else 88 | info.rwflag = !!((req->cmd_flags & REQ_OP_MASK) == REQ_OP_WRITE); 89 | #endif 90 | 91 | whop = whobyreq.lookup(&req); 92 | if (whop == 0) { 93 | // missed pid who, save stats as pid 0 94 | valp = counts.lookup_or_init(&info, &zero); 95 | } else { 96 | info.pid = whop->pid; 97 | __builtin_memcpy(&info.name, whop->name, sizeof(info.name)); 98 | valp = counts.lookup_or_init(&info, &zero); 99 | } 100 | 101 | // save stats 102 | valp->us += delta_us; 103 | valp->bytes += req->__data_len; 104 | valp->io++; 105 | 106 | start.delete(&req); 107 | whobyreq.delete(&req); 108 | 109 | return 0; 110 | } 111 | -------------------------------------------------------------------------------- /modules/biolatency.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2017-2018 Marko Myllynen 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | """ PCP BCC PMDA biolatency module """ 15 | 16 | # Configuration options 17 | # Name - type - default 18 | # 19 | # queued - boolean - False : include OS queued time in I/O time 20 | 21 | # pylint: disable=invalid-name, line-too-long 22 | 23 | from ctypes import c_int 24 | from bcc import BPF 25 | 26 | from modules.pcpbcc import PCPBCCBase 27 | from pcp.pmapi import pmUnits 28 | from cpmapi import PM_TYPE_U64, PM_SEM_COUNTER, PM_COUNT_ONE 29 | from cpmapi import PM_ERR_AGAIN 30 | 31 | # 32 | # BPF program 33 | # 34 | bpf_src = "modules/biolatency.bpf" 35 | 36 | # 37 | # PCP BCC PMDA constants 38 | # 39 | MODULE = 'biolatency' 40 | METRIC = 'disk.all.latency' 41 | units_count = pmUnits(0, 0, 1, 0, 0, PM_COUNT_ONE) 42 | 43 | # 44 | # PCP BCC Module 45 | # 46 | class PCPBCCModule(PCPBCCBase): 47 | """ PCP BCC biolatency module """ 48 | def __init__(self, config, log, err): 49 | """ Constructor """ 50 | PCPBCCBase.__init__(self, MODULE, config, log, err) 51 | 52 | self.cache = {} 53 | self.queued = False 54 | 55 | for opt in self.config.options(MODULE): 56 | if opt == 'queued': 57 | self.queued = self.config.getboolean(MODULE, opt) 58 | 59 | if self.queued: 60 | self.log("Including OS queued time in I/O time.") 61 | else: 62 | self.log("Excluding OS queued time from I/O time.") 63 | 64 | self.log("Initialized.") 65 | 66 | def metrics(self): 67 | """ Get metric definitions """ 68 | name = METRIC 69 | self.items.append( 70 | # Name - reserved - type - semantics - units - help 71 | (name, None, PM_TYPE_U64, PM_SEM_COUNTER, units_count, 'block io latency distribution'), 72 | ) 73 | return True, self.items 74 | 75 | def compile(self): 76 | """ Compile BPF """ 77 | try: 78 | self.bpf = BPF(src_file=bpf_src) 79 | if self.queued: 80 | self.bpf.attach_kprobe(event="blk_start_request", fn_name="trace_req_start") 81 | self.bpf.attach_kprobe(event="blk_mq_start_request", fn_name="trace_req_start") 82 | else: 83 | self.bpf.attach_kprobe(event="blk_account_io_start", fn_name="trace_req_start") 84 | self.bpf.attach_kprobe(event="blk_account_io_completion", fn_name="trace_req_completion") 85 | self.log("Compiled.") 86 | except Exception as error: # pylint: disable=broad-except 87 | self.err(str(error)) 88 | self.err("Module NOT active!") 89 | self.bpf = None 90 | 91 | def refresh(self): 92 | """ Refresh BPF data """ 93 | if self.bpf is None: 94 | return 95 | 96 | dist = self.bpf.get_table("dist") 97 | 98 | for k, v in dist.items(): 99 | if k.value == 0: 100 | continue 101 | low = (1 << k.value) >> 1 102 | high = (1 << k.value) - 1 103 | if low == high: 104 | low -= 1 105 | key = str(low) + "-" + str(high) 106 | if key not in self.cache: 107 | self.cache[key] = 0 108 | self.cache[key] += v.value 109 | self.insts[key] = c_int(1) 110 | 111 | dist.clear() 112 | 113 | return self.insts 114 | 115 | def bpfdata(self, item, inst): 116 | """ Return BPF data as PCP metric value """ 117 | try: 118 | key = self.pmdaIndom.inst_name_lookup(inst) 119 | return [self.cache[key], 1] 120 | except Exception: # pylint: disable=broad-except 121 | return [PM_ERR_AGAIN, 0] 122 | -------------------------------------------------------------------------------- /pmdabcc.1: -------------------------------------------------------------------------------- 1 | '\"macro stdmacro 2 | .\" 3 | .\" Copyright (C) 2017-2018 Marko Myllynen 4 | .\" 5 | .\" This program is free software; you can redistribute it and/or modify 6 | .\" it under the terms of the GNU General Public License as published by 7 | .\" the Free Software Foundation; either version 2 of the License, or 8 | .\" (at your option) any later version. 9 | .\" 10 | .\" This program is distributed in the hope that it will be useful, 11 | .\" but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | .\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | .\" GNU General Public License for more details. 14 | .\" 15 | .TH PMDABCC 1 "PCP" "Performance Co-Pilot" 16 | .SH NAME 17 | \f3pmdabcc\f1 \- BCC PMDA 18 | .SH DESCRIPTION 19 | \fBpmdabcc\fP is a Performance Metrics Domain Agent (PMDA) which 20 | extracts live performance data from extended BPF (Berkeley Packet Filter) 21 | programs by using BCC (BPF Compiler Collection) Python frontend. 22 | .PP 23 | \fBpmdabcc\fP itself provides no PCP metrics or BPF code; 24 | instead it loads and acts as a bridge for any number of configured, 25 | separate PCP BCC PMDA Python modules running BPF programs. 26 | Existing BCC Python tools and programs should be possible to be utilized 27 | with PCP BCC PMDA modules with reasonable effort. 28 | .PP 29 | See the BPF and BCC documentation for detailed description of both. 30 | .PP 31 | .SH INSTALLATION 32 | \fBpmdabcc\fP reads a mandatory ini-style configuration file 33 | .IP 34 | .PD 0 35 | .RS +4 36 | .IP \(bu 2 37 | .I \f(CW$PCP_PMDAS_DIR\fP/bcc/bcc.conf 38 | .RE 39 | .PD 40 | .PP 41 | This file contains in its \fB[pmda]\fP section values 42 | for the following PMDA options: 43 | .IP 44 | .PD 0 45 | .RS +4 46 | .IP \(bu 2 47 | modules 48 | .IP \(bu 49 | prefix 50 | .RE 51 | .PD 52 | .PP 53 | \fBpmdabcc\fP reads module-specific configuration for each module 54 | listed in the comma-separated list of \fBmodules\fP (mandatory). 55 | By default, all metrics from the modules will appear under the 56 | \fIbcc\fP Performance Metrics Name Space (PMNS) tree, \fBprefix\fP 57 | (optional) can be used to change this generic prefix. 58 | .PP 59 | For each listed module a corresponding \fB[module]\fP section must 60 | be defined containing at least the following options: 61 | .IP 62 | .PD 0 63 | .RS +4 64 | .IP \(bu 2 65 | module 66 | .IP \(bu 67 | cluster 68 | .RE 69 | .PD 70 | .PP 71 | \fBmodule\fP defines the actual Python module file name to be loaded 72 | during PMDA startup under the \fImodules\fP subdirectory of the PCP 73 | BCC PMDA installation. 74 | \fBcluster\fP specifies the cluster ID for the metrics provided by 75 | the module under the PMNS path. 76 | Optionally, \fBprefix\fP can be defined to override the 77 | generic value described above. 78 | All modules accept but not necessarily use the boolean \fBdebug\fP option. 79 | .PP 80 | Modules may also support additional module-specific configuration options, 81 | refer to each module for their supported options. 82 | .PP 83 | Once the needed setup is ready, you can install the PMDA to load the 84 | configured modules and the BPF programs they utilize. 85 | To install, do the following as root: 86 | .sp 1 87 | .RS +4 88 | .ft B 89 | .nf 90 | # cd $PCP_PMDAS_DIR/bcc 91 | # ./Install 92 | .fi 93 | .ft P 94 | .RE 95 | .sp 1 96 | To uninstall, do the following as root: 97 | .sp 1 98 | .RS +4 99 | .ft B 100 | .nf 101 | # cd $PCP_PMDAS_DIR/bcc 102 | # ./Remove 103 | .fi 104 | .ft P 105 | .RE 106 | .sp 1 107 | \fBpmdabcc\fP is launched by \fBpmcd\fP(1) and should never be 108 | executed directly. 109 | The Install and Remove scripts notify \fBpmcd\fP(1) when the 110 | agent is installed or removed. 111 | .SH FILES 112 | .TP 113 | .I \f(CW$PCP_PMDAS_DIR\fP/bcc/bcc.conf 114 | configuration file for the \fBpmdabcc\fP agent 115 | .TP 116 | .I \f(CW$PCP_PMDAS_DIR\fP/bcc/modules/*.py 117 | PCP BCC PMDA Python modules available for the \fBpmdabcc\fP agent 118 | .TP 119 | .I \f(CW$PCP_PMDAS_DIR\fP/bcc/Install 120 | installation script for the \fBpmdabcc\fP agent 121 | .TP 122 | .I \f(CW$PCP_PMDAS_DIR\fP/bcc/Remove\fP 123 | undo installation script for the \fBpmdabcc\fP agent 124 | .TP 125 | .I \f(CW$PCP_LOG_DIR\fP/pmcd/bcc.log 126 | default log file for messages from the \fBpmdabcc\fP agent 127 | .SH PCP ENVIRONMENT 128 | Environment variables with the prefix \fBPCP_\fP are used to parameterize 129 | the file and directory names used by PCP. 130 | On each installation, the 131 | file \fB/etc/pcp.conf\fP contains the local values for these variables. 132 | The \fB$PCP_CONF\fP variable may be used to specify an alternative 133 | configuration file, as described in \fBpcp.conf\fP(5). 134 | .SH SEE ALSO 135 | .BR PCPIntro (1), 136 | .BR bcc (1), 137 | .BR bpf (1), 138 | and 139 | .BR pmcd (1). 140 | -------------------------------------------------------------------------------- /modules/biotop.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2017-2018 Marko Myllynen 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | """ PCP BCC PMDA biotop module """ 15 | 16 | # pylint: disable=invalid-name, line-too-long 17 | 18 | from ctypes import c_int 19 | from os import kill 20 | from bcc import BPF 21 | 22 | from modules.pcpbcc import PCPBCCBase 23 | from pcp.pmapi import pmUnits 24 | from cpmapi import PM_TYPE_U64, PM_SEM_COUNTER, PM_SPACE_BYTE 25 | from cpmapi import PM_ERR_AGAIN 26 | 27 | # 28 | # BPF program 29 | # 30 | bpf_src = "modules/biotop.bpf" 31 | 32 | # 33 | # PCP BCC PMDA constants 34 | # 35 | MODULE = 'biotop' 36 | METRIC = 'proc.io.perdev' 37 | units_bytes = pmUnits(1, 0, 0, PM_SPACE_BYTE, 0, 0) 38 | 39 | # 40 | # PCP BCC Module 41 | # 42 | class PCPBCCModule(PCPBCCBase): 43 | """ PCP BCC biotop module """ 44 | def __init__(self, config, log, err): 45 | """ Constructor """ 46 | PCPBCCBase.__init__(self, MODULE, config, log, err) 47 | 48 | self.cache = {} 49 | 50 | self.disklookup = None 51 | self.update_disk_info() 52 | 53 | self.log("Initialized.") 54 | 55 | @staticmethod 56 | def pid_alive(pid): 57 | """ Test liveliness of PID """ 58 | try: 59 | kill(int(pid), 0) 60 | return True 61 | except Exception: # pylint: disable=broad-except 62 | return False 63 | 64 | def update_disk_info(self): 65 | """ Update disk info cache """ 66 | # cache disk major,minor -> diskname 67 | self.disklookup = {} 68 | if self.debug: 69 | self.log("Updating disk cache...") 70 | with open('/proc/diskstats') as stats: 71 | for line in stats: 72 | a = line.split() 73 | self.disklookup[a[0] + "," + a[1]] = a[2] 74 | 75 | def metrics(self): 76 | """ Get metric definitions """ 77 | name = METRIC 78 | self.items.append( 79 | # Name - reserved - type - semantics - units - help 80 | (name, None, PM_TYPE_U64, PM_SEM_COUNTER, units_bytes, 'device io per pid'), 81 | ) 82 | return True, self.items 83 | 84 | def compile(self): 85 | """ Compile BPF """ 86 | try: 87 | self.bpf = BPF(src_file=bpf_src) 88 | self.bpf.attach_kprobe(event="blk_account_io_start", fn_name="trace_pid_start") 89 | self.bpf.attach_kprobe(event="blk_start_request", fn_name="trace_req_start") 90 | self.bpf.attach_kprobe(event="blk_mq_start_request", fn_name="trace_req_start") 91 | self.bpf.attach_kprobe(event="blk_account_io_completion", fn_name="trace_req_completion") 92 | self.log("Compiled.") 93 | except Exception as error: # pylint: disable=broad-except 94 | self.err(str(error)) 95 | self.err("Module NOT active!") 96 | self.bpf = None 97 | 98 | def refresh(self): 99 | """ Refresh BPF data """ 100 | if self.bpf is None: 101 | return 102 | 103 | counts = self.bpf.get_table("counts") 104 | 105 | # Clean stale data 106 | for key in list(self.cache): 107 | if not self.pid_alive(key.split("::")[1]): 108 | del self.cache[key] 109 | del self.insts[key] 110 | 111 | # Update current data 112 | for k, v in counts.items(): 113 | disk = str(k.major) + "," + str(k.minor) 114 | 115 | # unnamed devices (e.g. non-device mounts) 116 | if k.major == 0: 117 | continue 118 | elif disk not in self.disklookup: 119 | # check for hot swapped devices 120 | self.update_disk_info() 121 | if disk not in self.disklookup: 122 | self.log("Traced unknown device (major: {} minor: {})".format(k.major, k.minor)) 123 | continue 124 | 125 | key = self.disklookup[disk] + "::" + str(k.pid).zfill(6) 126 | value = v.bytes if key not in self.cache else v.bytes + self.cache[key] 127 | self.cache[key] = value 128 | self.insts[key] = c_int(1) 129 | 130 | counts.clear() 131 | 132 | return self.insts 133 | 134 | def bpfdata(self, item, inst): 135 | """ Return BPF data as PCP metric value """ 136 | try: 137 | key = self.pmdaIndom.inst_name_lookup(inst).zfill(6) 138 | return [self.cache[key], 1] 139 | except Exception: # pylint: disable=broad-except 140 | return [PM_ERR_AGAIN, 0] 141 | -------------------------------------------------------------------------------- /modules/tcplife.bpf: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Netflix, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License") 3 | 4 | #include 5 | #define KBUILD_MODNAME "pcpbcctcplife" 6 | #include 7 | #include 8 | #include 9 | 10 | BPF_HASH(birth, struct sock *, u64); 11 | 12 | // separate data structs for ipv4 and ipv6 13 | struct ipv4_data_t { 14 | // XXX: switch some to u32's when supported 15 | u64 ts_us; 16 | u64 pid; 17 | u64 saddr; 18 | u64 daddr; 19 | u64 ports; 20 | u64 rx_b; 21 | u64 tx_b; 22 | u64 span_us; 23 | char task[TASK_COMM_LEN]; 24 | }; 25 | BPF_PERF_OUTPUT(ipv4_events); 26 | 27 | struct ipv6_data_t { 28 | u64 ts_us; 29 | u64 pid; 30 | unsigned __int128 saddr; 31 | unsigned __int128 daddr; 32 | u64 ports; 33 | u64 rx_b; 34 | u64 tx_b; 35 | u64 span_us; 36 | char task[TASK_COMM_LEN]; 37 | }; 38 | BPF_PERF_OUTPUT(ipv6_events); 39 | 40 | struct id_t { 41 | u32 pid; 42 | char task[TASK_COMM_LEN]; 43 | }; 44 | BPF_HASH(whoami, struct sock *, struct id_t); 45 | 46 | int kprobe__tcp_set_state(struct pt_regs *ctx, struct sock *sk, int state) 47 | { 48 | u32 pid = bpf_get_current_pid_tgid() >> 32; 49 | u16 lport = sk->__sk_common.skc_num; 50 | u16 dport = sk->__sk_common.skc_dport; 51 | 52 | /* 53 | * This tool includes PID and comm context. It's best effort, and may 54 | * be wrong in some situations. It currently works like this: 55 | * - record timestamp on any state < TCP_FIN_WAIT1 56 | * - cache task context on: 57 | * TCP_SYN_SENT: tracing from client 58 | * TCP_LAST_ACK: client-closed from server 59 | * - do output on TCP_CLOSE: 60 | * fetch task context if cached, or use current task 61 | */ 62 | 63 | // capture birth time 64 | if (state < TCP_FIN_WAIT1) { 65 | /* 66 | * Matching just ESTABLISHED may be sufficient, provided no code-path 67 | * sets ESTABLISHED without a tcp_set_state() call. Until we know 68 | * that for sure, match all early states to increase chances a 69 | * timestamp is set. 70 | */ 71 | u64 ts = bpf_ktime_get_ns(); 72 | birth.update(&sk, &ts); 73 | } 74 | 75 | // record PID & comm on SYN_SENT 76 | if (state == TCP_SYN_SENT || state == TCP_LAST_ACK) { 77 | struct id_t me = {.pid = pid}; 78 | bpf_get_current_comm(&me.task, sizeof(me.task)); 79 | whoami.update(&sk, &me); 80 | } 81 | 82 | if (state != TCP_CLOSE) 83 | return 0; 84 | 85 | // calculate lifespan 86 | u64 *tsp, delta_us; 87 | tsp = birth.lookup(&sk); 88 | if (tsp == 0) { 89 | whoami.delete(&sk); // may not exist 90 | return 0; // missed create 91 | } 92 | delta_us = (bpf_ktime_get_ns() - *tsp) / 1000; 93 | birth.delete(&sk); 94 | 95 | // fetch possible cached data 96 | struct id_t *mep; 97 | mep = whoami.lookup(&sk); 98 | if (mep != 0) 99 | pid = mep->pid; 100 | 101 | // get throughput stats. see tcp_get_info(). 102 | u64 rx_b = 0, tx_b = 0, sport = 0; 103 | struct tcp_sock *tp = (struct tcp_sock *)sk; 104 | rx_b = tp->bytes_received; 105 | tx_b = tp->bytes_acked; 106 | 107 | u16 family = sk->__sk_common.skc_family; 108 | 109 | if (family == AF_INET) { 110 | struct ipv4_data_t data4 = {.span_us = delta_us, 111 | .rx_b = rx_b, .tx_b = tx_b}; 112 | data4.ts_us = bpf_ktime_get_ns() / 1000; 113 | data4.saddr = sk->__sk_common.skc_rcv_saddr; 114 | data4.daddr = sk->__sk_common.skc_daddr; 115 | // a workaround until data4 compiles with separate lport/dport 116 | data4.pid = pid; 117 | data4.ports = ntohs(dport) + ((0ULL + lport) << 32); 118 | if (mep == 0) { 119 | bpf_get_current_comm(&data4.task, sizeof(data4.task)); 120 | } else { 121 | bpf_probe_read(&data4.task, sizeof(data4.task), (void *)mep->task); 122 | } 123 | ipv4_events.perf_submit(ctx, &data4, sizeof(data4)); 124 | 125 | } else /* 6 */ { 126 | struct ipv6_data_t data6 = {.span_us = delta_us, 127 | .rx_b = rx_b, .tx_b = tx_b}; 128 | data6.ts_us = bpf_ktime_get_ns() / 1000; 129 | bpf_probe_read(&data6.saddr, sizeof(data6.saddr), 130 | sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); 131 | bpf_probe_read(&data6.daddr, sizeof(data6.daddr), 132 | sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32); 133 | // a workaround until data6 compiles with separate lport/dport 134 | data6.ports = ntohs(dport) + ((0ULL + lport) << 32); 135 | data6.pid = pid; 136 | if (mep == 0) { 137 | bpf_get_current_comm(&data6.task, sizeof(data6.task)); 138 | } else { 139 | bpf_probe_read(&data6.task, sizeof(data6.task), (void *)mep->task); 140 | } 141 | ipv6_events.perf_submit(ctx, &data6, sizeof(data6)); 142 | } 143 | 144 | if (mep != 0) 145 | whoami.delete(&sk); 146 | 147 | return 0; 148 | } 149 | -------------------------------------------------------------------------------- /modules/tcplife.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (C) 2017-2018 Marko Myllynen 3 | # 4 | # This program is free software; you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation; either version 2 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | """ PCP BCC PMDA biotop module """ 15 | 16 | # pylint: disable=invalid-name,too-few-public-methods 17 | 18 | from threading import Lock, Thread 19 | import ctypes as ct 20 | from ctypes import c_int 21 | from os import kill 22 | from bcc import BPF 23 | 24 | from modules.pcpbcc import PCPBCCBase 25 | from pcp.pmapi import pmUnits 26 | from cpmapi import PM_TYPE_U64, PM_SEM_COUNTER, PM_SPACE_BYTE 27 | from cpmapi import PM_ERR_AGAIN 28 | 29 | # 30 | # BPF program 31 | # 32 | bpf_src = "modules/tcplife.bpf" 33 | 34 | # 35 | # PCP BCC PMDA constants 36 | # 37 | MODULE = 'tcplife' 38 | BASENS = 'proc.io.net.tcp.' 39 | units_bytes = pmUnits(1, 0, 0, PM_SPACE_BYTE, 0, 0) 40 | 41 | TASK_COMM_LEN = 16 # linux/sched.h 42 | 43 | class Data_ipv4(ct.Structure): 44 | """ IPv4 data struct """ 45 | _fields_ = [ 46 | ("ts_us", ct.c_ulonglong), 47 | ("pid", ct.c_ulonglong), 48 | ("saddr", ct.c_ulonglong), 49 | ("daddr", ct.c_ulonglong), 50 | ("ports", ct.c_ulonglong), 51 | ("rx_b", ct.c_ulonglong), 52 | ("tx_b", ct.c_ulonglong), 53 | ("span_us", ct.c_ulonglong), 54 | ("task", ct.c_char * TASK_COMM_LEN) 55 | ] 56 | 57 | class Data_ipv6(ct.Structure): 58 | """ IPv6 data struct """ 59 | _fields_ = [ 60 | ("ts_us", ct.c_ulonglong), 61 | ("pid", ct.c_ulonglong), 62 | ("saddr", (ct.c_ulonglong * 2)), 63 | ("daddr", (ct.c_ulonglong * 2)), 64 | ("ports", ct.c_ulonglong), 65 | ("rx_b", ct.c_ulonglong), 66 | ("tx_b", ct.c_ulonglong), 67 | ("span_us", ct.c_ulonglong), 68 | ("task", ct.c_char * TASK_COMM_LEN) 69 | ] 70 | 71 | # 72 | # PCP BCC Module 73 | # 74 | class PCPBCCModule(PCPBCCBase): 75 | """ PCP BCC biotop module """ 76 | def __init__(self, config, log, err): 77 | """ Constructor """ 78 | PCPBCCBase.__init__(self, MODULE, config, log, err) 79 | 80 | self.ipv4_stats = {} 81 | self.ipv6_stats = {} 82 | 83 | self.lock = Lock() 84 | self.thread = Thread(name="bpfpoller", target=self.poller) 85 | self.thread.setDaemon(True) 86 | 87 | self.log("Initialized.") 88 | 89 | @staticmethod 90 | def pid_alive(pid): 91 | """ Test liveliness of PID """ 92 | try: 93 | kill(int(pid), 0) 94 | return True 95 | except Exception: # pylint: disable=broad-except 96 | return False 97 | 98 | def poller(self): 99 | """ BPF poller """ 100 | try: 101 | # pylint: disable=no-member 102 | if 'perf_buffer_poll' in dir(self.bpf): 103 | poll = self.bpf.perf_buffer_poll 104 | else: 105 | poll = self.bpf.kprobe_poll 106 | while self.bpf: 107 | poll() 108 | except Exception as error: # pylint: disable=broad-except 109 | self.err(str(error)) 110 | self.err("BPF kprobe poll failed!") 111 | self.log("Poller thread exiting.") 112 | 113 | def handle_ipv4_event(self, _cpu, data, _size): 114 | """ IPv4 event handler """ 115 | event = ct.cast(data, ct.POINTER(Data_ipv4)).contents 116 | pid = str(event.pid).zfill(6) 117 | self.lock.acquire() 118 | if pid not in self.ipv4_stats: 119 | self.ipv4_stats[pid] = [int(event.tx_b), int(event.rx_b)] 120 | else: 121 | self.ipv4_stats[pid][0] += int(event.tx_b) 122 | self.ipv4_stats[pid][1] += int(event.rx_b) 123 | self.lock.release() 124 | 125 | def handle_ipv6_event(self, _cpu, data, _size): 126 | """ IPv6 event handler """ 127 | event = ct.cast(data, ct.POINTER(Data_ipv6)).contents 128 | pid = str(event.pid).zfill(6) 129 | self.lock.acquire() 130 | if pid not in self.ipv6_stats: 131 | self.ipv6_stats[pid] = [int(event.tx_b), int(event.rx_b)] 132 | else: 133 | self.ipv6_stats[pid][0] += int(event.tx_b) 134 | self.ipv6_stats[pid][1] += int(event.rx_b) 135 | self.lock.release() 136 | 137 | def metrics(self): 138 | """ Get metric definitions """ 139 | name = BASENS 140 | self.items = ( 141 | # Name - reserved - type - semantics - units - help 142 | (name + 'tx', None, PM_TYPE_U64, PM_SEM_COUNTER, units_bytes, 'tcp tx per pid'), 143 | (name + 'rx', None, PM_TYPE_U64, PM_SEM_COUNTER, units_bytes, 'tcp rx per pid'), 144 | ) 145 | return True, self.items 146 | 147 | def compile(self): 148 | """ Compile BPF """ 149 | try: 150 | self.bpf = BPF(src_file=bpf_src) 151 | self.bpf["ipv4_events"].open_perf_buffer(self.handle_ipv4_event, page_cnt=64) 152 | self.bpf["ipv6_events"].open_perf_buffer(self.handle_ipv6_event, page_cnt=64) 153 | self.thread.start() 154 | self.log("Compiled.") 155 | except Exception as error: # pylint: disable=broad-except 156 | self.err(str(error)) 157 | self.err("Module NOT active!") 158 | self.bpf = None 159 | 160 | def refresh(self): 161 | """ Refresh BPF data """ 162 | if self.bpf is None: 163 | return 164 | 165 | self.insts = {} 166 | 167 | self.lock.acquire() 168 | for pid in list(self.ipv4_stats): 169 | if not self.pid_alive(pid): 170 | del self.ipv4_stats[pid] 171 | else: 172 | self.insts[pid] = c_int(1) 173 | for pid in list(self.ipv6_stats): 174 | if not self.pid_alive(pid): 175 | del self.ipv6_stats[pid] 176 | else: 177 | self.insts[pid] = c_int(1) 178 | self.lock.release() 179 | 180 | return self.insts 181 | 182 | def bpfdata(self, item, inst): 183 | """ Return BPF data as PCP metric value """ 184 | try: 185 | self.lock.acquire() 186 | key = self.pmdaIndom.inst_name_lookup(inst) 187 | value = 0 188 | if key in self.ipv4_stats: 189 | value += self.ipv4_stats[key][item] 190 | if key in self.ipv6_stats: 191 | value += self.ipv6_stats[key][item] 192 | self.lock.release() 193 | return [value, 1] 194 | except Exception: # pylint: disable=broad-except 195 | self.lock.release() 196 | return [PM_ERR_AGAIN, 0] 197 | -------------------------------------------------------------------------------- /pmdabcc.python: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pmpython 2 | # 3 | # Copyright (C) 2017-2018 Marko Myllynen 4 | # 5 | # This program is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | """ PCP BCC Performance Metrics Domain Agent """ 16 | 17 | try: 18 | import configparser 19 | except ImportError: 20 | import ConfigParser as configparser 21 | import importlib 22 | import atexit 23 | import sys 24 | import os 25 | 26 | from collections import OrderedDict 27 | 28 | from pcp.pmapi import pmContext as PCP 29 | from pcp.pmda import PMDA, pmdaIndom, pmdaMetric 30 | from cpmapi import PM_INDOM_NULL 31 | from cpmapi import PM_ERR_VALUE 32 | 33 | # Module references 34 | CONFIG = 1 35 | MODULE = 2 36 | CLUSTER = 3 37 | PREFIX = 4 38 | METRICS = 5 39 | HELPERS = 6 40 | COMPILE = 7 41 | REFRESH = 8 42 | BPFDATA = 9 43 | CLEANUP = 10 44 | INDOM = 11 45 | INSTS = 12 46 | 47 | class BCCPMDA(PMDA): 48 | """ PCP BCC PMDA """ 49 | def __init__(self, name, domain): 50 | """ Constructor """ 51 | PMDA.__init__(self, name, domain) 52 | 53 | self.modules = OrderedDict() 54 | self.clusters = OrderedDict() 55 | self.prefix = "bcc." 56 | 57 | self.read_config_file() 58 | self.connect_pmcd() 59 | 60 | self.init_modules() 61 | self.register_metrics() 62 | self.register_helpers() 63 | self.compile_modules() 64 | 65 | self.set_refresh(self.bcc_refresh) 66 | self.set_fetch_callback(self.bcc_fetch_callback) 67 | 68 | @atexit.register 69 | def cleanup(): # pylint: disable=unused-variable 70 | """ Clean up """ 71 | if not os.environ.get('PCP_PYTHON_DOMAIN') and not os.environ.get('PCP_PYTHON_PMNS'): 72 | self.log("Cleaning up modules:") 73 | for module in self.modules: 74 | self.log(module) 75 | self.modules[module][CLEANUP]() 76 | self.log("Modules cleaned up.") 77 | 78 | def read_config_file(self): 79 | """ Read configuration file """ 80 | conffile = PCP.pmGetConfig('PCP_PMDAS_DIR') 81 | conffile += '/' + self.read_name() + '/' + self.read_name() + '.conf' 82 | 83 | if not os.path.isfile(conffile): 84 | self.err("Configuration file %s not found, aborting." % conffile) 85 | sys.exit(1) 86 | 87 | config = configparser.SafeConfigParser() 88 | config.read(conffile) 89 | if config.has_section('pmda'): 90 | for opt in config.options('pmda'): 91 | if opt == 'modules': 92 | if config.get('pmda', opt): 93 | for module in config.get('pmda', opt).split(","): 94 | self.modules[module] = {} 95 | elif opt == 'prefix': 96 | self.prefix = config.get('pmda', opt) 97 | else: 98 | self.err("Invalid directive '%s' in %s, aborting." % (opt, conffile)) 99 | sys.exit(1) 100 | else: 101 | self.err("No [pmda] section found, aborting.") 102 | sys.exit(1) 103 | 104 | if not self.modules: 105 | self.err("No modules configured, aborting.") 106 | sys.exit(1) 107 | 108 | self.log("Enabled modules:") 109 | self.log(str([module for module in self.modules])) 110 | 111 | self.log("Configuring modules:") 112 | for module in self.modules: 113 | self.log(module) 114 | self.read_module_config(config, module) 115 | self.log("Modules configured.") 116 | 117 | def read_module_config(self, config, module): 118 | """ Read common module configuration """ 119 | if config.has_section(module): 120 | for opt in config.options(module): 121 | if opt == 'module': 122 | self.modules[module][CONFIG] = config 123 | self.modules[module][MODULE] = config.get(module, opt) 124 | if opt == 'cluster': 125 | try: 126 | self.modules[module][CLUSTER] = int(config.get(module, opt)) 127 | except ValueError: 128 | self.err("Integer expected for 'cluster', aborting.") 129 | sys.exit(1) 130 | if opt == 'prefix': 131 | self.modules[module][PREFIX] = config.get(module, opt) 132 | else: 133 | self.err("No [%s] section found, aborting." % module) 134 | sys.exit(1) 135 | 136 | if PREFIX not in self.modules[module]: 137 | self.modules[module][PREFIX] = self.prefix 138 | 139 | if MODULE not in self.modules[module] or CLUSTER not in self.modules[module]: 140 | self.err("Both 'module' and 'cluster' are mandatory, aborting.") 141 | sys.exit(1) 142 | 143 | def init_modules(self): 144 | """ Initialize modules """ 145 | self.log("Initializing modules:") 146 | for module in self.modules: 147 | self.log(module) 148 | try: 149 | mod = importlib.import_module('modules.%s' % self.modules[module][MODULE]) 150 | except ImportError as error: 151 | self.err(str(error)) 152 | self.err("Module %s not found, aborting." % module) 153 | sys.exit(1) 154 | obj = mod.PCPBCCModule(self.modules[module][CONFIG], self.log, self.err) 155 | self.modules[module][METRICS] = obj.metrics 156 | self.modules[module][HELPERS] = obj.helpers 157 | self.modules[module][COMPILE] = obj.compile 158 | self.modules[module][REFRESH] = obj.refresh 159 | self.modules[module][BPFDATA] = obj.bpfdata 160 | self.modules[module][CLEANUP] = obj.cleanup 161 | self.log("Modules initialized.") 162 | 163 | def register_metrics(self): 164 | """ Register metrics """ 165 | self.log("Registering metrics:") 166 | for module in self.modules: 167 | self.log(module) 168 | cluster = int(self.modules[module][CLUSTER]) 169 | indom, metrics = self.modules[module][METRICS]() 170 | self.modules[module][INDOM] = PM_INDOM_NULL 171 | if indom: 172 | self.modules[module][INDOM] = self.indom(cluster) 173 | self.modules[module][INSTS] = pmdaIndom(self.modules[module][INDOM], {}) 174 | self.add_indom(self.modules[module][INSTS]) 175 | for item, _ in enumerate(metrics): 176 | self.add_metric(self.modules[module][PREFIX] + metrics[item][0], 177 | pmdaMetric(self.pmid(cluster, item), metrics[item][2], 178 | self.modules[module][INDOM], 179 | metrics[item][3], metrics[item][4]), 180 | metrics[item][5], metrics[item][5]) 181 | self.clusters[cluster] = module 182 | self.log("Metrics registered.") 183 | 184 | def register_helpers(self): 185 | """ Register helper function references """ 186 | self.log("Registering helpers:") 187 | for module in self.modules: 188 | if self.modules[module][INDOM] != PM_INDOM_NULL: 189 | self.log(module) 190 | self.modules[module][HELPERS](self.modules[module][INSTS]) 191 | self.log("Helpers registered.") 192 | 193 | def compile_modules(self): 194 | """ Compile modules """ 195 | if not os.environ.get('PCP_PYTHON_DOMAIN') and not os.environ.get('PCP_PYTHON_PMNS'): 196 | self.log("Compiling modules:") 197 | for module in self.modules: 198 | self.log(module) 199 | self.modules[module][COMPILE]() 200 | self.log("Modules compiled.") 201 | 202 | def bcc_refresh(self, cluster): 203 | """ Refresh """ 204 | module = self.clusters[cluster] 205 | insts = {} 206 | try: 207 | insts = self.modules[module][REFRESH]() 208 | except Exception as error: # pylint: disable=broad-except 209 | self.err(str(error)) 210 | if self.modules[module][INDOM] != PM_INDOM_NULL: 211 | self.modules[module][INSTS].set_instances(self.modules[module][INDOM], insts) 212 | self.replace_indom(self.modules[module][INDOM], insts) 213 | 214 | def bcc_fetch_callback(self, cluster, item, inst): 215 | """ Fetch callback """ 216 | module = self.clusters[cluster] 217 | try: 218 | return self.modules[module][BPFDATA](item, inst) 219 | except Exception as error: # pylint: disable=broad-except 220 | self.err(str(error)) 221 | return [PM_ERR_VALUE, 0] 222 | 223 | if __name__ == '__main__': 224 | BCCPMDA('bcc', 149).run() 225 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PCP BCC PMDA 2 | 3 | [![License: Apache v2](https://img.shields.io/badge/license-Apache%20v2-brightgreen.svg)](https://www.apache.org/licenses/LICENSE-2.0) 4 | [![License: GPLv2](https://img.shields.io/badge/license-GPLv2-brightgreen.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) 5 | [![License: GPLv3](https://img.shields.io/badge/license-GPLv3-brightgreen.svg)](https://www.gnu.org/licenses/gpl-3.0) 6 | 7 | ## Introduction 8 | 9 | This repository contains a plugin to connect eBPF/BCC Python modules 10 | to Performance Co-Pilot performance framework/toolkit to provide a 11 | unified interface and advanced tools for processing BPF/BCC data. 12 | 13 | _(This plugin has now been merged to PCP upstream and recent releases.)_ 14 | 15 | ## eBPF 16 | 17 | From https://www.oreilly.com/ideas/ebpf-and-systems-performance: 18 | 19 | > eBPF is a weird Linux kernel technology that powers low-overhead custom analysis tools, which can be run in production to find performance wins that no other tool can. With it, we can pull out millions of new metrics from the kernel and applications, and explore running software like never before. It’s a superpower. 20 | 21 | ## BCC 22 | 23 | From https://github.com/iovisor/bcc: 24 | 25 | > BCC is a toolkit for creating efficient kernel tracing and manipulation programs, and includes several useful tools and examples. It makes use of extended BPF (Berkeley Packet Filters), formally known as eBPF, a new feature that was first added to Linux 3.15. Much of what BCC uses requires Linux 4.1 and above. 26 | 27 | > BCC makes BPF programs easier to write, with kernel instrumentation in C (and includes a C wrapper around LLVM), and front-ends in Python and lua. It is suited for many tasks, including performance analysis and network traffic control. 28 | 29 | ## PCP 30 | 31 | > The Performance Co-Pilot (PCP, http://www.pcp.io/) system is a toolkit for collecting, archiving, and processing performance metrics from multiple operating systems. A typical Linux PCP installation offers over 1,000 metrics by default and is in turn extensible with its own plugins, or PMDAs ("Performance Metrics Domain Agents"). In addition to very complete /proc based statistics, readily available PCP PMDAs provide support for such system and application level components as 389 Directory Server, Apache, Ceph, containers, GFS2, Gluster, HAProxy, Java, libvirt, MySQL, NFS, Oracle, Postfix, PostgreSQL, Samba, and Sendmail, among others. 32 | 33 | ## Problem to Solve 34 | 35 | While BCC has made creating new BPF programs easier and the BCC project 36 | offers a wide variety of such tools (https://github.com/iovisor/bcc), 37 | basically all these programs are individual, disjoint utilities that are 38 | mostly meant for interactive use. This is not a suitable approach to 39 | collect, monitor and analyze performance data in larger environments 40 | where there are hundreds, if not thousands, installations and where 41 | human intervention is unfeasible at best. 42 | 43 | While PCP offers a unified interface to a great number of performance 44 | metrics, advanced command line utilities for analyzing live or archived 45 | metrics (e.g., http://pcp.io/man/man1/pmrep.1.html), and exporters to 46 | external systems like 47 | [Elasticsearch](http://pcp.io/man/man1/pcp2elasticsearch.1.html), 48 | [Graphite](http://pcp.io/man/man1/pcp2graphite.1.html), 49 | [Prometheus](https://prometheus.io/), and 50 | [Zabbix](http://pcp.io/man/man1/pcp2zabbix.1.html), it lacks the ability 51 | to easily collect detailed to performance data from an in-kernel tracing 52 | toolkit like eBPF/BCC. 53 | 54 | There is a need to connect eBPF/BCC programs to a unified performance 55 | metrics framework like PCP. There is a need to connect PCP easily to an 56 | in-kernel tracing toolkit like eBPF/BCC. 57 | 58 | ## Solution 59 | 60 | PCP BCC PMDA is a plugin which extracts live performance data from eBPF 61 | programs by using the BCC (BPF Compiler Collection) Python frontend and 62 | provides them to any PCP client for archiving, monitoring, exporting, 63 | and analysis purposes. It loads and acts as a bridge for any number of 64 | configured, separate PCP BCC PMDA Python modules running BPF programs. 65 | Existing BCC Python tools and modules should be possible to be utilized 66 | with PCP BCC PMDA modules with reasonable effort. 67 | 68 | Initially, four BCC programs were available to be used by PCP, these 69 | serve as examples how to access different data structures used by the 70 | programs. PCP upstream today has more than twenty modules included. 71 | 72 | * [pidpersec](https://github.com/iovisor/bcc/blob/master/tools/pidpersec.py) 73 | as [sysfork.py](modules/sysfork.py) 74 | * Merely a simple Hello World type minimalistic example which provides 75 | one single metric (which in itself is not interesting as PCP already 76 | provides the same metric as part of its [proc metrics](http://pcp.io/man/man1/pmdaproc.1.html) 77 | as _kernel.all.sysfork_. 78 | * [biolatency](https://github.com/iovisor/bcc/blob/master/tools/biolatency.py) 79 | as [biolatency.py](modules/biolatency.py) 80 | * Provides block I/O latency distribution statistics. 81 | * [biotop](https://github.com/iovisor/bcc/blob/master/tools/biotop.py) 82 | as [biotop.py](modules/biotop.py) 83 | * Provides block device I/O information by process. 84 | * [tcplife](https://github.com/iovisor/bcc/blob/master/tools/tcplife.py) 85 | as [tcplife.py](modules/tcplife.py) 86 | * Provides per-process TCP statistics. 87 | 88 | ## Screenshot 89 | 90 | With PCP BCC PMDA _biotop_ and _tcplife_ modules enabled, create a simple 91 | [pmrep configuration file](http://pcp.io/man/man5/pmrep.conf.5.html) 92 | for custom performance reporting, start pmrep, and run it for 50 seconds 93 | while running _dnf -y install libreoffice_ (PID 15751) in the background 94 | (vdb has an ext4 fs without LVM): 95 | 96 | ```Shell 97 | # cat << EOF > pmrep.conf 98 | [bcc-example] 99 | timestamp = yes 100 | interval = 5s 101 | proc.fd.count = ,,,, 102 | proc.io.write_bytes = ,,kB/s,, 103 | bcc.proc.io.perdev = ,,kB/s,, 104 | bcc.proc.io.net.tcp.rx = ,,MB,raw, 105 | bcc.proc.io.net.tcp.tx = ,,MB,raw, 106 | EOF 107 | # pmrep -c ./test.conf -i '.*15751.*' --samples 10 :bcc-example 108 | p.f.count p.i.write_bytes b.p.i.perdev b.p.i.perdev b.p.i.n.t.rx b.p.i.n.t.tx 109 | 015751 /u 015751 /usr/bin vdb::015751 vda::015751 015751 015751 110 | count kB/s kB/s kB/s MB MB 111 | 10:00:04 23 N/A N/A N/A 166.011 0.069 112 | 10:00:09 23 2234.501 1891.713 0.000 166.011 0.069 113 | 10:00:14 23 2109.045 953.655 0.000 174.980 0.075 114 | 10:00:19 23 2137.472 1192.711 0.000 174.980 0.075 115 | 10:00:24 23 1987.648 1510.867 0.000 174.980 0.075 116 | 10:00:29 23 2099.366 1712.808 0.000 185.435 0.081 117 | 10:00:34 23 2042.308 1251.223 0.000 185.435 0.081 118 | 10:00:39 23 1708.094 1004.761 0.000 266.203 0.087 119 | 10:00:44 23 1524.975 1188.127 0.000 266.203 0.087 120 | 10:00:49 23 1823.358 1123.660 0.000 279.981 0.093 121 | # 122 | ``` 123 | 124 | ## Upstream Status 125 | 126 | The upstream Pull Request for BCC PMDA plugin was merged to PCP 4.0 127 | release and the plugin is available e.g. on recent Fedora/RHEL releases 128 | as _pcp-pmda-bcc_ package. 129 | 130 | The below installation and testing instructions are still helpful 131 | with earlier PCP releases and on distributions with no BCC PMDA 132 | package available (but see upstream builds at https://bintray.com/pcp/). 133 | 134 | ## Installation 135 | 136 | * Tested on latest Fedora 27 with: 137 | * kernel-4.14.16-300.fc27.x86_64 138 | * bcc-tools-0.5.0-1.fc27 139 | * pcp-3.12.2-1.fc27.x86_64 140 | * pcp-system-tools-3.12.2-1.fc27.x86_64 141 | * python3-pcp-3.12.2-1.fc27.x86_64 142 | * Put SELinux into Permissive mode (for the time being): 143 | * setenforce 0 144 | * Install the required packages and start the _pmcd_ daemon: 145 | * yum install bcc-tools pcp pcp-system-tools python3-pcp 146 | * systemctl enable --now pmcd 147 | * For a PCP Quick Guide, see: 148 | * http://pcp.io/docs/guide.html 149 | * Test the setup with something trivial (e.g., mimic vmstat with pmrep): 150 | * pmrep :vmstat 151 | * Copy this repository as the PCP BCC PMDA directory: 152 | * cp -r pcp-bcc-pmda /var/lib/pcp/pmdas/bcc 153 | * Configure and enable the plugin: 154 | * cd /var/lib/pcp/pmdas/bcc 155 | * man ./pmdabcc.1 156 | * vi bcc.conf 157 | * ./Install 158 | * Verify operation from logs and by fetching metrics data (data may not 159 | be instantly available on an idle system): 160 | * less /var/log/pcp/pmcd/bcc.log 161 | * pminfo -f bcc 162 | * pmrep bcc.proc.sysfork 163 | * Export and analyse: 164 | * pcp2{csv,json,elasticsearch,graphite,influxdb,json,xlsx,xml,zabbix} 165 | * Enhance and extend: 166 | * less modules/*.bpf modules/*.py 167 | * vi modules/test.bpf modules/test.py bcc.conf 168 | * ./Remove ; ./Install ; 169 | 170 | ## Discussion / Open Items 171 | 172 | * Security / accesss restrictions 173 | * Some of the modules should not be enabled without care as they may 174 | obviously provide sensitive information that should not be available 175 | for non-privileged users - see [pmcd(1)](http://pcp.io/man/man1/pmcd.1.html) 176 | for information on PMCD access control configuration 177 | * Drop copy-pasted BPF programs (in PCP upstream) and import BPF code on the 178 | fly once the newly added _-e_ option is available on all relevant distros 179 | * Since PMCD uses stdout for control messages, debug output from BPF/BCC 180 | can't be enabled, however errors appearing on stderr will appear in 181 | the PMDA log 182 | * https://github.com/performancecopilot/pcp/issues/365 183 | * Update _pcp-selinux_ as appropriate 184 | * https://github.com/performancecopilot/pcp/issues/388 185 | * For potentially improved performance a rewrite in C could be considered 186 | * Would lose the ease of Python for very unclear gain; not planned 187 | 188 | ## License 189 | 190 | GPLv2+ (PCP BCC PMDA), Apache v2 (BCC/BPF programs) 191 | --------------------------------------------------------------------------------