├── scripts ├── net │ ├── rds_egress_TP_example.txt │ ├── rds_bcopy_metric_example.txt │ ├── rds_conn2irq_example.txt │ ├── rds_check_tx_stall_example.txt │ ├── arp_origin_example.txt │ ├── rds_egress_TP.d │ ├── rds_rdma_lat.d │ ├── ping_lat_example.txt │ ├── rds_conn2irq.d │ ├── rds_bcopy_metric.d │ ├── rds_tx_funccount_example.txt │ ├── rds_check_tx_stall.d │ ├── rds_rdma_xfer_rate_example.txt │ ├── rds_rdma_xfer_rate.d │ ├── rds_tx_funccount.d │ ├── rds_rdma_lat_example.txt │ ├── arp_origin.d │ └── ping_lat.d ├── core │ ├── spinlock_time_example.txt │ └── spinlock_time.d ├── io │ ├── scsi_queue.d │ ├── scsi_latency_example.txt │ ├── nvme_io_comp.d │ ├── scsi_latency.d │ └── nvme_io_comp_example.txt └── Makefile ├── CONTRIBUTORS ├── tools ├── memstate │ ├── memstate_lib │ │ ├── __init__.py │ │ ├── rss.py │ │ ├── constants.py │ │ ├── swap.py │ │ ├── logfile.py │ │ ├── hugepages.py │ │ ├── slabinfo.py │ │ ├── buddyinfo.py │ │ ├── base.py │ │ ├── pss.py │ │ └── meminfo.py │ ├── Makefile │ └── memstate.man ├── kstack │ ├── Makefile │ └── kstack.man ├── scanfs │ ├── Makefile │ └── scanfs.man ├── vmcore-utils │ ├── Makefile │ ├── vmcore_sz.man │ └── vmcore_sz.py ├── syswatch │ ├── Makefile │ ├── syswatch.8.scd │ └── syswatch.8 └── lkce │ ├── Makefile │ ├── lkce.man │ └── scripts │ └── kdump_report.py ├── SECURITY.md ├── sbom_generation.yaml ├── oled.man ├── CONTRIBUTING.md ├── Makefile ├── README.md └── oled.py /scripts/net/rds_egress_TP_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: track egress drop reason along with pid and comm. 2 | 3 | Arguments: None [Enable rds egress trace point] 4 | 5 | Output format: reason= 6 | 7 | Sample output: Shown below 8 | 9 | # ./rds_egress_TP.d 10 | 2022 Oct 21 03:04:45 ctrl+c to stop 11 | 2022 Oct 21 03:05:50 pid=373319 comm=rds-stress [<192.168.0.7,192.168.0.5,0>] reason=rdsinfo pending 12 | 2022 Oct 21 03:10:52 pid=373319 comm=rds-stress [<192.168.0.7,192.168.0.5,0>] reason=rdsinfo pending 13 | 2022 Oct 21 03:13:48 pid=373319 comm=rds-stress [<192.168.0.7,192.168.0.5,0>] reason=rdsinfo pending 14 | 15 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Alina Yurenko 2 | Arumugam Kolappan 3 | Aruna Ramakrishna 4 | Cesar Roque 5 | Jeffery Yoder 6 | Jose Lombera 7 | Manjunath Patil 8 | Mridula Shastry 9 | Nagappan Ramasamy Palaniappan 10 | Oleksandra Pavlusieva 11 | Partha Sarathi Satapathy 12 | Praveen Kumar Kannoju 13 | Srikanth C S 14 | Srinivas Eeda 15 | Wengang Wang 16 | -------------------------------------------------------------------------------- /scripts/net/rds_bcopy_metric_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: This script is used to trace and display the send/recv bytes per connection 2 | 3 | Arguments: None 4 | 5 | Output format: [,, ] 6 | 7 | Sample output: Shown below 8 | 9 | # ./rds_bcopy_metric.d | grep "192.168.0.8,192.168.0.6" 10 | [<192.168.0.8,192.168.0.6,0> send] 16 11 | [<192.168.0.8,192.168.0.6,0> recv] 16 12 | [<192.168.0.8,192.168.0.6,0> send] 16 13 | [<192.168.0.8,192.168.0.6,0> recv] 16 14 | [<192.168.0.8,192.168.0.6,0> send] 16 15 | [<192.168.0.8,192.168.0.6,0> recv] 16 16 | [<192.168.0.8,192.168.0.6,0> send] 16 17 | [<192.168.0.8,192.168.0.6,0> recv] 16 18 | [<192.168.0.8,192.168.0.6,0> send] 16 19 | [<192.168.0.8,192.168.0.6,0> recv] 16 20 | [<192.168.0.8,192.168.0.6,0> send] 16 21 | [<192.168.0.8,192.168.0.6,0> recv] 16 22 | [<192.168.0.8,192.168.0.6,0> send] 16 23 | [<192.168.0.8,192.168.0.6,0> recv] 16 24 | -------------------------------------------------------------------------------- /scripts/net/rds_conn2irq_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: Display rds connection and its related qpn, cqn and ird lines 2 | 3 | Arguments: None 4 | 5 | Output format: [, , ] conn= 6 | ic= qpn= dqpn= scqn= scqirq= rcqn= rcqirq= 10 | 11 | Note: This scripts works only on CX-5/RoCE based machine 12 | 13 | Sample output: Shown below. 14 | 15 | # ./rds_conn2irq.d 16 | 2022 Sep 1 03:26:05 print rds conn to irq mapping. Run rds-info -I parallelly to populate. ctrl+c to stop and print summary 17 | 18 | [<192.168.100.125,192.168.100.63,0>] conn=ffff8d93e96e8148 ic=ffff8d35fa43c000 qpn=156 dqpn=0 scqn=1210 scqirq=546 rcqn=1211 rcqirq=545 19 | ^C 20 | Summary: 21 | irqn count 22 | 545 1 23 | -------------------------------------------------------------------------------- /scripts/net/rds_check_tx_stall_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: This script monitors the completions on all the connections and 2 | calculates the time difference between the consecutive completions. If this 3 | time difference exceeds the threshold time (in microseconds), given as 4 | input argument, message will be printed providing the details of the 5 | connection parameters. 6 | 7 | Arguments: Threshold time in microseconds. (Minimum: 500) 8 | 9 | Output format: [, ] last send completion: usecs ago 10 | 11 | Sample output: Shown below. 12 | 13 | Test 1: 14 | 15 | ./rds_check_tx_stall.d 200 16 | Error: Argument should be >= 500 ( microseconds) [2022 Oct 31 13:39:24] 17 | 18 | Test 2: 19 | 20 | ./rds_check_tx_stall.d 1000000 21 | 2022 Oct 31 13:37:16 Print each time there is a gap that exceeds the threshold, ctrl+c to stop. 22 | [192.168.18.5,192.168.18.8,2] last send completion: 1000211 usecs ago 23 | [192.168.18.5,192.168.18.8,3] last send completion: 1000359 usecs ago 24 | [192.168.18.5,192.168.18.8,3] last send completion: 1000233 usecs ago 25 | [192.168.18.5,192.168.18.8,3] last send completion: 1000153 usecs ago 26 | ^C 27 | 28 | -------------------------------------------------------------------------------- /scripts/net/arp_origin_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: This script detects and prints the details of all ARP requests, 2 | replies, and ignored packets on all the network interfaces present on the 3 | system where this script is being executed. 4 | 5 | Arguments: None. 6 | 7 | Output format: each line lists 8 | 9 | 10 | 11 | 12 | arp 13 | sip:, 14 | tip: 15 | 16 | Sample output: Shown below. 17 | 18 | # ./arp_origin.d 19 | 2022 Nov 1 12:36:54 ib0 Recv arp request sip: 192.168.18.8, tip: 192.168.18.5 20 | 2022 Nov 1 12:36:54 ib0 Send arp reply sip: 192.168.18.5, tip: 192.168.18.8 21 | 2022 Nov 1 12:36:54 ib0 Send arp request: sip: 192.168.18.5, tip: 192.168.18.8 22 | 2022 Nov 1 12:36:54 ib0 Recv arp reply sip: 192.168.18.8 tip: 192.168.18.5 23 | 2022 Nov 1 12:36:54 ib1 Recv arp request (Ignoring) sip: 192.168.18.8, tip: 192.168.18.5 24 | 2022 Nov 1 12:36:55 ib0 Recv arp request sip: 192.168.18.8, tip: 192.168.18.5 25 | 2022 Nov 1 12:36:55 ib0 Send arp reply sip: 192.168.18.5, tip: 192.168.18.8 26 | 2022 Nov 1 12:36:55 ib0 Send arp request: sip: 192.168.18.5, tip: 192.168.18.8 27 | 2022 Nov 1 12:36:55 ib0 Recv arp reply sip: 192.168.18.8 tip: 192.168.18.5 28 | 2022 Nov 1 12:36:55 ib1 Recv arp request (Ignoring) sip: 192.168.18.8, tip: 192.168.18.5 29 | 30 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Public exports of memstate_lib module.""" 23 | 24 | # flake8: noqa 25 | 26 | from .base import Base 27 | from .base import LockFile 28 | from .buddyinfo import Buddyinfo 29 | from .logfile import Logfile 30 | from .hugepages import Hugepages 31 | from .meminfo import Meminfo 32 | from .rss import Rss 33 | from .numa import Numa 34 | from .pss import Pss 35 | from .slabinfo import Slabinfo 36 | from .swap import Swap 37 | from . import constants 38 | -------------------------------------------------------------------------------- /tools/kstack/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | all: 22 | @echo "all:$(CURDIR)" 23 | 24 | clean: 25 | @echo "clean:$(CURDIR)" 26 | 27 | install: 28 | @echo "install:$(CURDIR)" 29 | install --mode=755 kstack $(BINDIR)/kstack 30 | gzip -c $(CURDIR)/kstack.man > $(MANDIR)/oled-kstack.8.gz 31 | chmod 644 $(MANDIR)/oled-kstack.8.gz 32 | @echo "KSTACK successfully installed!" 33 | 34 | uninstall: 35 | @echo "uninstall:$(CURDIR)" 36 | rm -f $(MANDIR)/oled-kstack.8.gz 37 | rm -f $(BINDIR)/kstack 38 | @echo "KSTACK successfully uninstalled!" 39 | -------------------------------------------------------------------------------- /tools/scanfs/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | all: 22 | @echo "all:$(CURDIR)" 23 | 24 | clean: 25 | @echo "clean:$(CURDIR)" 26 | 27 | install: 28 | @echo "install:$(CURDIR)" 29 | install -m 755 $(CURDIR)/scanfs.py $(BINDIR)/scanfs 30 | gzip -c $(CURDIR)/scanfs.man > $(MANDIR)/oled-scanfs.8.gz 31 | chmod 644 $(MANDIR)/oled-scanfs.8.gz 32 | @echo "scanfs successfully installed!" 33 | 34 | uninstall: 35 | @echo "uninstall:$(CURDIR)" 36 | rm -f $(BINDIR)/scanfs 37 | rm -f $(MANDIR)/oled-scanfs.8.gz 38 | @echo "scanfs successfully uninstalled!" 39 | -------------------------------------------------------------------------------- /tools/vmcore-utils/Makefile: -------------------------------------------------------------------------------- 1 | # Copyrigcanht (c) 2023, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | all: 22 | @echo "all:$(CURDIR)" 23 | 24 | clean: 25 | @echo "clean:$(CURDIR)" 26 | 27 | install: 28 | @echo "install:$(CURDIR)" 29 | install -m 755 $(CURDIR)/vmcore_sz.py $(BINDIR)/vmcore_sz 30 | gzip -c $(CURDIR)/vmcore_sz.man > $(MANDIR)/oled-vmcore_sz.8.gz 31 | chmod 644 $(MANDIR)/oled-vmcore_sz.8.gz 32 | @echo "vmcore-utils successfully installed!" 33 | 34 | uninstall: 35 | @echo "uninstall:$(CURDIR)" 36 | rm -f $(BINDIR)/vmcore_sz 37 | rm -f $(MANDIR)/oled-vmcore_sz.8.gz 38 | @echo "vmcore-utils successfully uninstalled!" 39 | -------------------------------------------------------------------------------- /scripts/core/spinlock_time_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: Track 'the time interrupt is disabled' due to spin_lock_irq*() 2 | kernel functions. Takes an argument[holdtime in ms] and prints the process 3 | (and its call stack) which disables interrupt more than the given time. 4 | 5 | Argument: Time in ms (Minimum value is 2) 6 | 7 | Output format: 8 | time= ms lock= pid= cpu= cmd= 9 | followed by stack dump 10 | 11 | Sample output: Shown below. 12 | 13 | # ./spinlock_time.d 1 14 | Error: Argument should be >= 2 ( millisec) [2023 Jan 18 10:15:38] 15 | 16 | 17 | # ./spinlock_time.d 2 18 | Processes holding spinlock_irq longer than 2 ms [2023 Jan 18 10:21:28] 19 | ---------------------------------------------------------------------- 20 | 21 | [2023 Jan 18 10:21:32] time=2 ms lock=ffffc41bff5083a0 pid=28352 cpu=39 cmd=osysmond.bin 22 | 23 | vmlinux`do_invalid_op+0x20 24 | vmlinux`invalid_op+0x11a 25 | vmlinux`_raw_spin_unlock_irqrestore+0x14 26 | iova`fq_flush_timeout+0x66 27 | vmlinux`call_timer_fn+0x3c 28 | vmlinux`run_timer_softirq+0x209 29 | vmlinux`__do_softirq+0xe1 30 | vmlinux`irq_exit+0xf6 31 | vmlinux`smp_apic_timer_interrupt+0x91 32 | vmlinux`apic_timer_interrupt+0x1c6 33 | vmlinux`__raw_spin_unlock+0x10 34 | vmlinux`dput+0xc9 35 | vmlinux`proc_fill_cache+0x81 36 | vmlinux`proc_readfd_common+0xe8 37 | vmlinux`proc_readfd+0x15 38 | vmlinux`iterate_dir+0x98 39 | vmlinux`SyS_getdents+0x98 40 | vmlinux`do_syscall_64+0x79 41 | vmlinux`entry_SYSCALL_64+0x191 42 | 43 | -------------------------------------------------------------------------------- /tools/syswatch/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | all: 22 | @echo "all:$(CURDIR)" 23 | 24 | clean: 25 | @echo "clean:$(CURDIR)" 26 | 27 | syswatch.8: syswatch.8.scd 28 | scdoc < $< > $@ 29 | 30 | # Whenever syswatch.8.scd is modified, `make man` must be run and the resulting 31 | # changes to syswatch.8 must be committed together with syswatch.8.scd 32 | man: syswatch.8 33 | 34 | install: 35 | @echo "install:$(CURDIR)" 36 | install --mode=755 $(CURDIR)/syswatch $(BINDIR)/syswatch 37 | gzip -c $(CURDIR)/syswatch.8 > $(MANDIR)/oled-syswatch.8.gz 38 | chmod 644 $(MANDIR)/oled-syswatch.8.gz 39 | @echo "SYSWATCH successfully installed!" 40 | 41 | uninstall: 42 | @echo "uninstall:$(CURDIR)" 43 | rm -f $(MANDIR)/oled-syswatch.8.gz 44 | rm -f $(BINDIR)/syswatch 45 | @echo "SYSWATCH successfully uninstalled!" 46 | -------------------------------------------------------------------------------- /tools/lkce/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | LKCEDIR := $(OLEDDIR)/lkce 22 | LKCE_KDUMPDIR := $(OLEDDIR)/lkce/lkce_kdump.d 23 | 24 | all: 25 | @echo "all:$(CURDIR)" 26 | 27 | clean: 28 | @echo "clean:$(CURDIR)" 29 | 30 | install: 31 | @echo "install:$(CURDIR)" 32 | mkdir -p $(LKCEDIR) 33 | mkdir -p $(LKCE_KDUMPDIR) 34 | mkdir -p $(MANDIR) 35 | install -m 755 $(CURDIR)/scripts/lkce.py $(BINDIR)/lkce 36 | install -m 755 $(CURDIR)/scripts/kdump_report.py $(LKCE_KDUMPDIR)/kdump_report 37 | gzip -c $(CURDIR)/lkce.man > $(MANDIR)/oled-lkce.8.gz 38 | chmod 644 $(MANDIR)/oled-lkce.8.gz 39 | @echo "lkce successfully installed" 40 | 41 | uninstall: 42 | @echo "uninstall:$(CURDIR)" 43 | $(BINDIR)/lkce disable || : 44 | rm -rf $(LKCE_KDUMPDIR) 45 | rm -rf $(LKCEDIR) 46 | rm -f $(BINDIR)/lkce 47 | rm -f $(MANDIR)/oled-lkce.8.gz 48 | @echo "lkce successfully uninstalled" 49 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting security vulnerabilities 2 | 3 | Oracle values the independent security research community and believes that 4 | responsible disclosure of security vulnerabilities helps us ensure the security 5 | and privacy of all our users. 6 | 7 | Please do NOT raise a GitHub Issue to report a security vulnerability. If you 8 | believe you have found a security vulnerability, please submit a report to 9 | [secalert_us@oracle.com][1] preferably with a proof of concept. Please review 10 | some additional information on [how to report security vulnerabilities to Oracle][2]. 11 | We encourage people who contact Oracle Security to use email encryption using 12 | [our encryption key][3]. 13 | 14 | We ask that you do not use other channels or contact the project maintainers 15 | directly. 16 | 17 | Non-vulnerability related security issues including ideas for new or improved 18 | security features are welcome on GitHub Issues. 19 | 20 | ## Security updates, alerts and bulletins 21 | 22 | Security updates will be released on a regular cadence. Many of our projects 23 | will typically release security fixes in conjunction with the 24 | Oracle Critical Patch Update program. Additional 25 | information, including past advisories, is available on our [security alerts][4] 26 | page. 27 | 28 | ## Security-related information 29 | 30 | We will provide security related information such as a threat model, considerations 31 | for secure use, or any known security issues in our documentation. Please note 32 | that labs and sample code are intended to demonstrate a concept and may not be 33 | sufficiently hardened for production use. 34 | 35 | [1]: mailto:secalert_us@oracle.com 36 | [2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html 37 | [3]: https://www.oracle.com/security-alerts/encryptionkey.html 38 | [4]: https://www.oracle.com/security-alerts/ 39 | -------------------------------------------------------------------------------- /tools/vmcore-utils/vmcore_sz.man: -------------------------------------------------------------------------------- 1 | .TH OLED VMCORE_SZ 8 "Sep 2023" "Oracle Linux Enhanced Diagnostics" 2 | 3 | .SH NAME 4 | oled vmcore_sz - Estimating vmcore size before kernel dump. 5 | 6 | .SH SYNOPSIS 7 | oled vmcore_sz [-h] [-d DUMP_LEVEL] 8 | 9 | .SH DESCRIPTION 10 | The vmcore_sz takes the dump_level as an argument and estimates the 11 | vmcore size if a kernel dump is obtained at that moment. It displays 12 | the total number of pages, pages need to be excluded depending on 13 | the dump level and the expected VMcore size in bytes. If the dump 14 | level is not specified, the default configured in "/etc/kdump.conf" 15 | will be used. 16 | 17 | oled vmcore_sz takes dump level (in range 0 to 31) as an argument 18 | to display the expected VMcore size in bytes. 19 | 20 | .SH EXAMPLE 21 | #oled vmcore_sz 22 | .PP 23 | Dump level is not specified. Using default/configured - [31] 24 | .PP 25 | ---------------------------------------------- 26 | .br 27 | Dump level = 31 28 | .br 29 | Exclude zero pages : 56568 30 | .br 31 | Exclude non private cache : 1594405 32 | .br 33 | Exclude private cache : 34 34 | .br 35 | Exclude user pages : 143813 36 | .br 37 | Exclude free pages : 2076424 38 | .PP 39 | Total Pages : 4134552 40 | .br 41 | Pages to be dumped : 263308 42 | .PP 43 | ---------------------------------------------- 44 | .br 45 | Expected vmcore size in bytes : 1078509568 46 | .br 47 | ---------------------------------------------- 48 | .br 49 | 50 | .SH OPTIONS 51 | .TP 52 | -h, --help 53 | Displays a help message for vmvmcore_sz command. 54 | 55 | .TP 56 | -d DUMP_LEVEL, --dump_level DUMP_LEVEL 57 | Calculate vmcore size based on user provided DUMP_LEVEL. DUMP_LEVEL is an integer in range 0..31 58 | 59 | .SH REPORTING BUGS 60 | .TP 61 | Please contact Oracle Linux Support to report any bugs for OLED tools. 62 | 63 | .SH AUTHOR 64 | .TP 65 | Partha Sarathi Satapathy 66 | 67 | .SH CO-AUTHOR 68 | .TP 69 | Srikanth C S 70 | -------------------------------------------------------------------------------- /scripts/net/rds_egress_TP.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -qs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author(s): Manjunath Patil 27 | * Purpose: track egress drop reason along with pid and comm. 28 | * Prerequisites: Refer to the file rds_egress_TP_example.txt 29 | * Sample output: Refer to the file rds_egress_TP_example.txt 30 | */ 31 | 32 | dtrace:::BEGIN 33 | { 34 | printf("%Y ctrl+c to stop\n", walltimestamp); 35 | } 36 | 37 | /* 38 | * trace_rds_drop_egress(rm, rs, conn, cpath, 39 | * conn ? &conn->c_laddr : NULL, 40 | * conn ? &conn->c_faddr : NULL, 41 | * reason); 42 | */ 43 | fbt:rds:trace_event_raw_event_rds_drop_egress:entry 44 | / arg3 != NULL / 45 | { 46 | this->conn = (struct rds_connection *)arg3; 47 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 48 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 49 | this->tos = this->conn->c_tos; 50 | this->reason = stringof(arg7); 51 | 52 | printf("%Y pid=%d comm=%s [<%s,%s,%d>] reason=%s\n", 53 | walltimestamp, pid, execname, 54 | inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, 55 | this->reason); 56 | } 57 | -------------------------------------------------------------------------------- /sbom_generation.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. 2 | 3 | # This OCI DevOps build specification file [1] generates a Software Bill of Materials (SBOM) of the repository. 4 | # The file is needed to run checks for third-party vulnerabilities and business approval according to Oracle’s GitHub policies. 5 | # [1] https://docs.oracle.com/en-us/iaas/Content/devops/using/build_specs.htm 6 | 7 | version: 0.1 8 | component: build 9 | timeoutInSeconds: 1000 10 | shell: bash 11 | env: 12 | variables: 13 | PYTHON_CMD: "python3" 14 | CDXGEN_DEBUG_MODE: "debug" 15 | steps: 16 | - type: Command 17 | name: "Download the version 10.10.0 of cdxgen globally" 18 | command: | 19 | npm install -g @cyclonedx/cdxgen@10.10.0 20 | - type: Command 21 | name: "Workaround to let cdxgen run on nodejs 16" 22 | command: | 23 | # cdxgen relies on a fourth-party dependency that cannot be executed in a Node.js environment running version 16 24 | # (as installed on the build runner instance) 25 | # This is a workaround to ensure cdxgen functions correctly, even in an older Node.js environment. 26 | cd /node/node-v16.14.2-linux-x64/lib/node_modules/@cyclonedx/cdxgen && \ 27 | npm install cheerio@v1.0.0-rc.12 28 | - type: Command 29 | name: "Generate SBOM for Python " 30 | command: | 31 | # Search the test or dev requirements files, so that test and dev py packages can be excluded in the generated SBOM 32 | files=$(find . -type f -regex ".*$test.*requirements\|requirements.*test\|dev.*requirements\|requirements.*dev$.*\.txt") && \ 33 | if [ -n "$files" ]; then \ 34 | cdxgen -t python -o artifactSBOM.json --spec-version 1.4 \ 35 | --exclude "*{requirements,dev,test}*{requirements,dev,test}*.txt" --project-name "$(basename $OCI_PRIMARY_SOURCE_URL)" --no-recurse 36 | else \ 37 | cdxgen -t python -o artifactSBOM.json --spec-version 1.4 --project-name "$(basename $OCI_PRIMARY_SOURCE_URL)" --no-recurse 38 | fi \ 39 | outputArtifacts: 40 | - name: artifactSBOM 41 | type: BINARY 42 | location: ${OCI_PRIMARY_SOURCE_DIR}/artifactSBOM.json -------------------------------------------------------------------------------- /oled.man: -------------------------------------------------------------------------------- 1 | .TH OLED 8 "Nov 2024" "Oracle Linux Enhanced Diagnostics" 2 | 3 | .SH NAME 4 | oled - Interface to invoke other diagnostic tools 5 | 6 | .SH SYNOPSIS 7 | oled [-h | --help | -v | --version] [sub_command] [options] 8 | 9 | .SH DESCRIPTION 10 | OLED (Oracle Linux Enhanced Diagnostics) is a collection of diagnostic tools 11 | and scripts. The \fBoled\fR command is used to invoke these other tools/scripts. 12 | See usage below, for available options. 13 | 14 | .TP 15 | # oled --help 16 | usage: oled {-h | -v | COMMAND [ARGS]} 17 | 18 | Valid commands: 19 | lkce -- Linux Kernel Core Extractor 20 | memstate -- Capture and analyze memory usage statistics 21 | kstack -- Gather kernel stack based on the process status or PID 22 | syswatch -- Execute user-provided commands based on the CPU utilization 23 | scanfs -- Scan KVM images for corruption, supports XFS and EXT4 24 | vmcore_sz -- Estimating vmcore size before kernel dump 25 | 26 | optional arguments: 27 | -h, --help show this help message and exit 28 | -v, --version show program's version number and exit 29 | 30 | NOTE: Must run as root. 31 | 32 | Please check the man page of each sub-command to learn more about that tool and 33 | its usage. 34 | 35 | .SH EXAMPLES 36 | .TP 37 | 38 | # oled memstate -h 39 | usage: oled memstate [-h] [-p [PID]] [-s [FILE]] [-n [FILE]] [-a] [-v] 40 | [-f [INTERVAL]] 41 | 42 | memstate: Capture and analyze memory usage data on this system. 43 | 44 | optional arguments: 45 | -h, --help show this help message and exit 46 | -p [PID], --pss [PID] 47 | display per-process memory usage 48 | -s [FILE], --slab [FILE] 49 | analyze/display slab usage 50 | -n [FILE], --numa [FILE] 51 | analyze/display NUMA stats 52 | -a, --all display all data 53 | -v, --verbose verbose data capture; combine with other options 54 | -f [INTERVAL], --frequency [INTERVAL] 55 | interval at which data should be collected (default: 56 | 30s) 57 | 58 | .SH REPORTING BUGS 59 | .TP 60 | Please contact Oracle Linux Support to report any bugs for OLED tools. 61 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to this repository 2 | 3 | We welcome your contributions! There are multiple ways to contribute. 4 | 5 | ## Opening issues 6 | 7 | For bugs or enhancement requests, please file a GitHub issue unless it's 8 | security related. When filing a bug remember that the better written the bug 9 | is, the more likely it is to be fixed. If you think you've found a security 10 | vulnerability, do not raise a GitHub issue and follow the instructions in our 11 | [security policy](./SECURITY.md). 12 | 13 | ## Contributing code 14 | 15 | We welcome your code contributions. Before submitting code via a pull request, 16 | you will need to have signed the [Oracle Contributor Agreement][OCA] (OCA) and 17 | your commits need to include the following line using the name and e-mail 18 | address you used to sign the OCA: 19 | 20 | ```text 21 | Signed-off-by: Your Name 22 | ``` 23 | 24 | This can be automatically added to pull requests by committing with 25 | `--signoff` or `-s`, e.g. 26 | 27 | ```text 28 | git commit --signoff 29 | ``` 30 | 31 | Only pull requests from committers that can be verified as having signed the 32 | OCA can be accepted. 33 | 34 | ## Pull request process 35 | 36 | 1. Ensure there is an issue created to track and discuss the fix or enhancement 37 | you intend to submit. 38 | 1. Fork this repository. 39 | 1. Create a branch in your fork to implement the changes. We recommend using 40 | the issue number as part of your branch name, e.g. `1234-fixes`. 41 | 1. Ensure that any documentation is updated with the changes that are required 42 | by your change. 43 | 1. Ensure that any samples are updated if the base image has been changed. 44 | 1. Submit the pull request. *Do not leave the pull request blank*. Explain 45 | exactly what your changes are meant to do and provide simple steps on how to 46 | validate. your changes. Ensure that you reference the issue you created as 47 | well. 48 | 1. We will assign the pull request to 2-3 people for review before it is 49 | merged. 50 | 51 | ## Code of conduct 52 | 53 | Follow the [Golden Rule](https://en.wikipedia.org/wiki/Golden_Rule). If you'd 54 | like more specific guidelines, see the [Contributor Covenant Code of 55 | Conduct][COC]. 56 | 57 | [OCA]: https://oca.opensource.oracle.com 58 | [COC]: https://www.contributor-covenant.org/version/1/4/code-of-conduct/ 59 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | VERSION="0.7" 22 | ARCH := $(shell uname -m) 23 | 24 | subdirs := tools/lkce tools/memstate tools/kstack tools/syswatch tools/scanfs tools/vmcore-utils 25 | 26 | subdirs := $(subdirs) scripts 27 | rev_subdirs := $(shell echo -n "$(subdirs) " | tac -s ' ') 28 | OLEDDIR := $(DESTDIR)/etc/oled 29 | SBINDIR := $(DESTDIR)/usr/sbin 30 | MANDIR := $(DESTDIR)/usr/share/man/man8 31 | OLEDBIN := $(DESTDIR)/usr/libexec/oled-tools 32 | 33 | export OLEDDIR 34 | export MANDIR 35 | 36 | all: 37 | echo $(subdirs) 38 | $(foreach dir,$(subdirs), $(MAKE) BINDIR=$(OLEDBIN) -C $(dir) all || exit 1;) 39 | 40 | clean: 41 | 42 | $(foreach dir,$(subdirs), $(MAKE) BINDIR=$(OLEDBIN) -C $(dir) clean;) 43 | 44 | install: 45 | @echo "install:$(CURDIR)" 46 | $(MAKE) all 47 | mkdir -p $(OLEDDIR) 48 | mkdir -p $(SBINDIR) 49 | mkdir -p $(MANDIR) 50 | mkdir -p $(OLEDBIN) 51 | install -m 755 oled.py $(SBINDIR)/oled 52 | gzip -c oled.man > $(MANDIR)/oled.8.gz; chmod 644 $(MANDIR)/oled.8.gz 53 | $(foreach dir,$(subdirs), $(MAKE) BINDIR=$(OLEDBIN) -C $(dir) install || exit 1;) 54 | @echo "oled-tools installed" 55 | 56 | uninstall: 57 | @echo "uninstall:$(CURDIR)" 58 | $(foreach dir, $(rev_subdirs), $(MAKE) BINDIR=$(OLEDBIN) -C $(dir) uninstall || exit 1;) 59 | rm -f $(MANDIR)/oled.8.gz 60 | rm -f $(SBINDIR)/oled 61 | rmdir $(OLEDBIN) || : 62 | rmdir $(OLEDDIR) || : 63 | @echo "oled-tools uninstalled" 64 | -------------------------------------------------------------------------------- /tools/scanfs/scanfs.man: -------------------------------------------------------------------------------- 1 | .TH OLED SCANFS 8 "May 2023" "Oracle Linux Enhanced Diagnostics" 2 | 3 | .SH NAME 4 | oled scanfs - Scan KVM images for corruption. Supports XFS and EXT4 filesystems. 5 | 6 | .SH SYNOPSIS 7 | oled scanfs [-h] [-c | -s] directory_path 8 | 9 | .SH DESCRIPTION 10 | The scanfs tool takes the path to image files as an argument and scans the XFS and 11 | EXT4 volumes from these images for corruption. The tool takes all the image files under 12 | the given directory path and creates a reflink copy of these image files. Once the 13 | reflink copy is done, it creates loop devices for these files and adds all partitions 14 | and volume groups, renames all the volume groups including the ones that conflict with 15 | the host and scans all the XFS and EXT4 volumes using xfs_repair and e2fsck. 16 | 17 | Sample output: 18 | 19 | Scanfs 1.0 20 | 21 | 22 | Setting up 23 | setting up - Volume1.img 24 | setting up - Volume2.img 25 | Setup Complete 26 | 27 | Checking XFS Filesystems 28 | Scanning /dev/Scan037871-VG/Home 29 | Scanning /dev/Scan037871-VG/Root 30 | Scanning /dev/Scan037871-VG/Sys 31 | Scanning /dev/mapper/loop0p1 32 | 33 | Checking EXT4 Filesystems 34 | 35 | Cleaning up 36 | Cleanup complete. 37 | 38 | Check /EXAVMIMAGES/bug34037871/Scanfs-2023-06-28T09-25-36/summary for more details 39 | 40 | Corruption found in these volumes: 41 | 42 | ************************************ 43 | 44 | xfs - /dev/Scan037871-VG/Sys 45 | 46 | ************************************ 47 | 48 | The output shows the location to log file, which holds more details on the images and 49 | corruption. It lists out the volumes that it finds to be corrupt. 50 | 51 | This tool also supports setup and cleanup - see OPTIONS for more details. 52 | 53 | .SH OPTIONS 54 | .TP 55 | -h, --help 56 | Displays a help message with all the valid options that 57 | can be passed to the tool. 58 | 59 | .TP 60 | -s, --setup 61 | Do not scan the volumes, just sets up the loop devices, partitions and volume 62 | groups and exit. It displays all the images that are set up. 63 | 64 | .TP 65 | -c, --cleanup 66 | Cleanup the already setup environment. Deactivates VGs and remove partition 67 | mappings and loop devices. It shows the reflink images left behind that might 68 | need to be purged. 69 | 70 | .SH REPORTING BUGS 71 | .TP 72 | Please contact Oracle Linux Support to report any bugs for OLED tools. 73 | 74 | .SH AUTHOR 75 | .TP 76 | Srikanth C S 77 | -------------------------------------------------------------------------------- /scripts/io/scsi_queue.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -qs 2 | 3 | /* 4 | * Copyright (c) 2024, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | * 24 | * Author(s): Rajan Shanmugavelu 25 | * Purpose: to measure SCSI IO queue time in milliseconds (ms). 26 | * 27 | * The DTrace 'fbt' and 'profile' modules need to be loaded 28 | * ('modprobe -a fbt profile') for UEK5. 29 | * Sample output: Refer to the file scsi_queue_example.txt 30 | */ 31 | 32 | #pragma D option dynvarsize=100m 33 | #pragma D option strsize=25 34 | 35 | dtrace:::BEGIN 36 | { 37 | printf("Tracing... Hit Ctrl-C to end.\n"); 38 | } 39 | 40 | fbt::scsi_queue_insert:entry 41 | { 42 | start_time[arg0] = timestamp; 43 | } 44 | 45 | ::scsi_dispatch_cmd_start 46 | /this->start = start_time[arg0]/ 47 | { 48 | this->elapsed = (timestamp - this->start) / 1000; 49 | this->cmd = (struct scsi_cmnd *) arg0; 50 | this->scsi_device = (struct scsi_device *)this->cmd->device; 51 | this->device = (struct device) this->scsi_device->sdev_gendev; 52 | this->sd = stringof(this->cmd->request->rq_disk->disk_name); 53 | this->dev_name = stringof(this->device.kobj.name); 54 | 55 | @avg[this->sd, this->dev_name] = avg(this->elapsed/1000); 56 | @plot[this->sd, this->dev_name] = lquantize(this->elapsed / 1000, 0, 1000, 100); 57 | 58 | start_time[arg0] = 0; 59 | } 60 | 61 | dtrace:::END 62 | { 63 | printf("%Y", walltimestamp); 64 | printf("Wait queue time by disk (ms):\n"); 65 | printa("\n %-12s %-20s\n%@d", @plot); 66 | printf("\n\n %-12s %-20s %12s\n", "DEVICE", "NAME", "AVG_WAIT(us)"); 67 | printa(" %-12s %-20s %@12d\n", @avg); 68 | } 69 | 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # oled-tools 2 | 3 | Oracle Linux Enhanced Diagnostic (OLED) tools is a collection of tools, 4 | scripts, configs, etc. that collect and analyze data about the health of the 5 | system in order to root cause and resolve any system issues. 6 | 7 | ## Components 8 | 9 | The oled-tools repo includes the following debug tools/scripts that aid in 10 | gathering additional debug data from the system. Please review the 11 | corresponding man pages for more details. 12 | 13 | - lkce: Extracts data from a vmcore or from within the kdump kernel after a 14 | crash 15 | - memstate: Captures and analyzes various memory usage statistics on the 16 | running system 17 | - filecache: List the paths of the biggest files present in the page cache 18 | (note: this is present only on the x86-64 architecture) 19 | - dentrycache: Lists a sample of file paths which have active dentries in the 20 | dentry hash table 21 | (note: this is present only on the x86-64 architecture) 22 | - kstack: Collects the kernel stack trace for selected processes, based on 23 | status or PID 24 | - syswatch: Execute user-provided commands when CPU utilization reaches a 25 | threshold 26 | - scanfs: Scan KVM images for corruption, supports XFS and EXT4 27 | - vmcore-utils: Estimating vmcore size before kernel dump 28 | 29 | ## Installation 30 | 31 | ### Build dependencies 32 | 33 | - `make` 34 | - `python3` 35 | - `zlib-devel` 36 | - `bzip2-devel` 37 | - `elfutils-devel` 38 | 39 | ### Runtime dependencies 40 | 41 | - `python3` 42 | - `zlib` 43 | - `bzip2-libs` 44 | - `elfutils-libs` 45 | 46 | ### Build and install 47 | 48 | ```bash 49 | $ git clone https://github.com/oracle/oled-tools.git 50 | $ cd oled-tools 51 | $ make 52 | $ make install 53 | ``` 54 | 55 | `lkce` requires additional setup the first time - it is recommended to run 56 | the following command after oled-tools is installed: 57 | 58 | ```bash 59 | $ [ -f /etc/oled/lkce/lkce.conf ] || sudo oled lkce configure --default 60 | ``` 61 | 62 | ## Usage 63 | 64 | `oled-tools` must be run as root. After it has been installed, it can be run 65 | with 66 | 67 | ``` 68 | sudo oled [args] 69 | ``` 70 | 71 | Run `sudo oled --help` to see a list of subcommands supported and other options 72 | and `sudo oled {-h | --help}` to see help of specific commands. 73 | You can also consult man pages `oled(8)` and `oled-(8)`. 74 | 75 | ## Contributing 76 | 77 | This project welcomes contributions from the community. Before submitting a 78 | pull request, please [review our contribution guide](./CONTRIBUTING.md) 79 | 80 | ## Security 81 | 82 | Please consult the [security guide](./SECURITY.md) for our responsible security 83 | vulnerability disclosure process 84 | 85 | ## License 86 | 87 | oled-tools is licensed under [GPLv2](LICENSE.txt). 88 | -------------------------------------------------------------------------------- /scripts/net/rds_rdma_lat.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author: Manjunath Patil 27 | * Purpose: Trace and display the rdma read/write in bytes and the 28 | * corresponding latency (in usec) per connection as in the below format 29 | * Prerequisites: Refer to rds_rdma_lat_example.txt 30 | * Sample output: Refer to rds_rdma_lat_example.txt 31 | */ 32 | 33 | #define DT_VERSION_NUMBER_(M, m, u) \ 34 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 35 | 36 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 37 | #pragma D option lockmem=unlimited 38 | #endif 39 | 40 | #define container_of(__ptr, __type, __member) ((__type *)((unsigned long long)__ptr - (unsigned long long)offsetof(__type, __member))) 41 | 42 | dtrace:::BEGIN 43 | { 44 | printf("%Y %16s %-16s %s\n", walltimestamp, "[ op]", 45 | "Bytes", "Latency(in usecs)"); } 46 | 47 | fbt:rds:rds_cmsg_rdma_args:entry 48 | { 49 | this->rm = (struct rds_message *)arg1; 50 | this->op = (struct rm_rdma_op *)&this->rm->rdma; 51 | track_rdma[this->op] = timestamp; 52 | } 53 | 54 | fbt::rds_ib_send_unmap_rdma*:entry 55 | / track_rdma[(struct rm_rdma_op *) arg1] / 56 | { 57 | this->op = (struct rm_rdma_op *)arg1; 58 | this->rm = (struct rds_message *)container_of(this->op, struct rds_message, rdma); 59 | 60 | this->cpath = (struct rds_conn_path *)this->rm->m_conn_path; 61 | this->conn = (struct rds_connection *)this->cpath->cp_conn; 62 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 63 | this->fip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 64 | this->tos = this->conn->c_tos; 65 | 66 | printf("[<%s,%s,%d> %c] bytes=%d lat(usec)=%u \n", 67 | inet_ntoa(this->sip), inet_ntoa(this->fip), this->tos, 68 | this->op->op_write ? 'w' : 'r', this->op->op_bytes, 69 | (timestamp-track_rdma[this->op])/1000); 70 | track_rdma[this->op] = 0; 71 | } 72 | -------------------------------------------------------------------------------- /scripts/net/ping_lat_example.txt: -------------------------------------------------------------------------------- 1 | ping 192.168.18.9 -I ib0 -c 5 2 | PING 192.168.18.9 (192.168.18.9) from 192.168.18.5 ib0: 56(84) bytes of data. 3 | 64 bytes from 192.168.18.9: icmp_seq=1 ttl=64 time=0.636 ms 4 | 64 bytes from 192.168.18.9: icmp_seq=2 ttl=64 time=0.366 ms 5 | 64 bytes from 192.168.18.9: icmp_seq=3 ttl=64 time=0.167 ms 6 | 64 bytes from 192.168.18.9: icmp_seq=4 ttl=64 time=0.135 ms 7 | 64 bytes from 192.168.18.9: icmp_seq=5 ttl=64 time=0.120 ms 8 | 9 | --- 192.168.18.9 ping statistics --- 10 | 5 packets transmitted, 5 received, 0% packet loss, time 4084ms 11 | rtt min/avg/max/mdev = 0.120/0.284/0.636/0.197 ms 12 | [root@rds-test1 ~]# 13 | 14 | [root@rds-test1 dtrace]# dtrace -Cqs ping_lat.d 15 | icmp id: 14434, icmp sequence: 256, src ip: 192.168.18.5, dst ip: 192.168.18.9 16 | 17 | routine delta(ns) 18 | 19 | ip_send_skb 20 | dev_hard_start_xmit 190382 21 | ipoib_start_xmit 44855 22 | ipoib_cm_send 84644 23 | icmp_rcv 322743 24 | 25 | request-reply latency: 642624 26 | ------------------------------------------------------------------ 27 | icmp id: 14434, icmp sequence: 512, src ip: 192.168.18.5, dst ip: 192.168.18.9 28 | 29 | routine delta(ns) 30 | 31 | ip_send_skb 32 | dev_hard_start_xmit 96159 33 | ipoib_start_xmit 44214 34 | ipoib_cm_send 44720 35 | icmp_rcv 173769 36 | 37 | request-reply latency: 358862 38 | ------------------------------------------------------------------ 39 | icmp id: 14434, icmp sequence: 768, src ip: 192.168.18.5, dst ip: 192.168.18.9 40 | 41 | routine delta(ns) 42 | 43 | ip_send_skb 44 | dev_hard_start_xmit 18217 45 | ipoib_start_xmit 8269 46 | ipoib_cm_send 8729 47 | icmp_rcv 129234 48 | 49 | request-reply latency: 164449 50 | ------------------------------------------------------------------ 51 | icmp id: 14434, icmp sequence: 1024, src ip: 192.168.18.5, dst ip: 192.168.18.9 52 | 53 | routine delta(ns) 54 | 55 | ip_send_skb 56 | dev_hard_start_xmit 12989 57 | ipoib_start_xmit 5900 58 | ipoib_cm_send 4801 59 | icmp_rcv 110251 60 | 61 | request-reply latency: 133941 62 | ------------------------------------------------------------------ 63 | icmp id: 14434, icmp sequence: 1280, src ip: 192.168.18.5, dst ip: 192.168.18.9 64 | 65 | routine delta(ns) 66 | 67 | ip_send_skb 68 | dev_hard_start_xmit 11200 69 | ipoib_start_xmit 4504 70 | ipoib_cm_send 4559 71 | icmp_rcv 100701 72 | 73 | request-reply latency: 120964 74 | ------------------------------------------------------------------ 75 | 76 | -------------------------------------------------------------------------------- /scripts/net/rds_conn2irq.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author: Manjunath Patil 27 | * Purpose: Display rds connection and its related qpn, cqn and irq lines 28 | * Prerequisites: Refer to the file rds_conn2irq_example.txt 29 | * Sample output: Refer to the file rds_conn2irq_example.txt 30 | */ 31 | 32 | #define DT_VERSION_NUMBER_(M, m, u) \ 33 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 34 | 35 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 36 | #pragma D option lockmem=unlimited 37 | #endif 38 | 39 | dtrace:::BEGIN 40 | { 41 | printf("%Y print rds conn to irq mapping. Run rds-info -I parallelly to populate. ctrl+c to stop and print summary\n", walltimestamp); 42 | } 43 | 44 | fbt:rds_rdma:rds_ib_conn_info_visitor:entry 45 | / seen[(struct rds_connection *)arg0] != 1 / 46 | { 47 | this->conn = (struct rds_connection *)arg0; 48 | this->ic = (struct rds_ib_connection *)this->conn->c_path[0].cp_transport_data; 49 | 50 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 51 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 52 | this->tos = this->conn->c_tos; 53 | 54 | this->scq = this->ic->i_scq; 55 | this->m_ibcq = (struct mlx5_ib_cq *)this->scq; 56 | this->send_mcq = (struct mlx5_core_cq *)&this->m_ibcq->mcq; 57 | 58 | this->rcq = this->ic->i_rcq; 59 | this->m_ibcq = (struct mlx5_ib_cq *)this->rcq; 60 | this->recv_mcq = (struct mlx5_core_cq *)&this->m_ibcq->mcq; 61 | 62 | printf("[<%s,%s,%d>] conn=%p ic=%p qpn=%u dqpn=%u scqn=%u scqirq=%u rcqn=%u rcqirq=%u\n", 63 | inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, this->conn, this->ic, 64 | this->ic->i_qp_num, this->ic->i_dst_qp_num, 65 | this->send_mcq->cqn, this->send_mcq->irqn, 66 | this->recv_mcq->cqn, this->recv_mcq->irqn); 67 | 68 | @irq_dist[this->send_mcq->irqn] = count(); 69 | @irq_dist[this->recv_mcq->irqn] = count(); 70 | seen[this->conn] = 1; 71 | } 72 | 73 | dtrace:::END 74 | { 75 | printf("Summary:\n"); 76 | printf("%12s %8s\n", "irqn", "count"); 77 | printa("%12d %@8d\n", @irq_dist); 78 | } 79 | -------------------------------------------------------------------------------- /tools/memstate/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | PYTHON_SITEDIR=$(shell python3 -c "import site; print(site.getsitepackages()[-1])") 22 | MEMSTATE_LIB=$(DESTDIR)$(PYTHON_SITEDIR)/memstate_lib 23 | CURLIB=$(CURDIR)/memstate_lib 24 | 25 | all: 26 | @echo "all:$(CURDIR)" 27 | 28 | clean: 29 | @echo "clean:$(CURDIR)" 30 | 31 | install: 32 | @echo "install:$(CURDIR)" 33 | @echo "$(MEMSTATE_LIB)" 34 | mkdir -p $(MEMSTATE_LIB) 35 | install -m 755 $(CURDIR)/memstate.py $(BINDIR)/memstate 36 | install -m 755 $(CURLIB)/__init__.py $(MEMSTATE_LIB)/__init__.py 37 | install -m 755 $(CURLIB)/base.py $(MEMSTATE_LIB)/base.py 38 | install -m 755 $(CURLIB)/buddyinfo.py $(MEMSTATE_LIB)/buddyinfo.py 39 | install -m 755 $(CURLIB)/constants.py $(MEMSTATE_LIB)/constants.py 40 | install -m 755 $(CURLIB)/hugepages.py $(MEMSTATE_LIB)/hugepages.py 41 | install -m 755 $(CURLIB)/logfile.py $(MEMSTATE_LIB)/logfile.py 42 | install -m 755 $(CURLIB)/meminfo.py $(MEMSTATE_LIB)/meminfo.py 43 | install -m 755 $(CURLIB)/numa.py $(MEMSTATE_LIB)/numa.py 44 | install -m 755 $(CURLIB)/pss.py $(MEMSTATE_LIB)/pss.py 45 | install -m 755 $(CURLIB)/rss.py $(MEMSTATE_LIB)/rss.py 46 | install -m 755 $(CURLIB)/slabinfo.py $(MEMSTATE_LIB)/slabinfo.py 47 | install -m 755 $(CURLIB)/swap.py $(MEMSTATE_LIB)/swap.py 48 | gzip -c $(CURDIR)/memstate.man > $(MANDIR)/oled-memstate.8.gz 49 | chmod 644 $(MANDIR)/oled-memstate.8.gz 50 | @echo "memstate successfully installed!" 51 | 52 | uninstall: 53 | @echo "uninstall:$(CURDIR)" 54 | rm -f $(BINDIR)/memstate 55 | rm -f $(MEMSTATE_LIB)/base.py 56 | rm -f $(MEMSTATE_LIB)/buddyinfo.py 57 | rm -f $(MEMSTATE_LIB)/constants.py 58 | rm -f $(MEMSTATE_LIB)/hugepages.py 59 | rm -f $(MEMSTATE_LIB)/logfile.py 60 | rm -f $(MEMSTATE_LIB)/meminfo.py 61 | rm -f $(MEMSTATE_LIB)/numa.py 62 | rm -f $(MEMSTATE_LIB)/pss.py 63 | rm -f $(MEMSTATE_LIB)/rss.py 64 | rm -f $(MEMSTATE_LIB)/slabinfo.py 65 | rm -f $(MEMSTATE_LIB)/swap.py 66 | rm -f $(MEMSTATE_LIB)/__init__.py 67 | rm -f $(MEMSTATE_LIB)/*.pyc 68 | rm -f $(MANDIR)/oled-memstate.8.gz 69 | rmdir $(MEMSTATE_LIB) || : 70 | @echo "memstate successfully uninstalled!" 71 | -------------------------------------------------------------------------------- /scripts/net/rds_bcopy_metric.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author: Manjunath Patil 27 | * Purpose: This script is used to trace and display the send and recv bytes 28 | * per connection 29 | * Prerequisites: Refer to the file rds_bcopy_metric_example.txt 30 | * Sample output: Refer to the file rds_bcopy_metric_example.txt 31 | */ 32 | 33 | #define DT_VERSION_NUMBER_(M, m, u) \ 34 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 35 | 36 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 37 | #pragma D option lockmem=unlimited 38 | #endif 39 | 40 | dtrace:::BEGIN 41 | { 42 | printf("%Y %16s %10s \n", walltimestamp, "[ op]", "MB/s"); 43 | } 44 | 45 | fbt::rds_sendmsg:entry 46 | { 47 | this->socket = (struct socket *)arg0; 48 | this->sock = this->socket->sk; 49 | self->rs = (struct rds_sock *)this->sock; 50 | self->payload_len = (size_t)arg2; 51 | } 52 | 53 | fbt::rds_sendmsg:return 54 | / self->rs && self->payload_len == (size_t)arg1 / 55 | { 56 | this->conn = (struct rds_connection *)self->rs->rs_conn; 57 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 58 | this->fip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 59 | this->tos = this->conn->c_tos; 60 | 61 | @payloads[inet_ntoa(this->sip), inet_ntoa(this->fip), this->tos, "send"] = sum(self->payload_len); 62 | self->rs = 0; 63 | self->payload_len = 0; 64 | } 65 | 66 | fbt::rds_ib_inc_copy_to_user:entry 67 | { 68 | self->inc = (struct rds_incoming *)arg0; 69 | } 70 | 71 | fbt::rds_ib_inc_copy_to_user:return 72 | / self->inc && arg1 > 0 / 73 | { 74 | this->copied = (int)arg1; 75 | this->conn = (struct rds_connection *)self->inc->i_conn; 76 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 77 | this->fip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 78 | this->tos = this->conn->c_tos; 79 | 80 | @payloads[inet_ntoa(this->sip), inet_ntoa(this->fip), this->tos, "recv"] = sum(this->copied); 81 | self->inc = 0; 82 | } 83 | 84 | tick-10s 85 | { 86 | /* convert bytes/10 secs to MB/s */ 87 | normalize(@payloads, 10*1024*1024); 88 | printa("[<%s,%s,%d> %s] %-@d\n", @payloads); 89 | clear(@payloads); 90 | } 91 | -------------------------------------------------------------------------------- /scripts/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, Oracle and/or its affiliates. 2 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 | # 4 | # This code is free software; you can redistribute it and/or modify it 5 | # under the terms of the GNU General Public License version 2 only, as 6 | # published by the Free Software Foundation. 7 | # 8 | # This code is distributed in the hope that it will be useful, but WITHOUT 9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 | # version 2 for more details (a copy is included in the LICENSE file that 12 | # accompanied this code). 13 | # 14 | # You should have received a copy of the GNU General Public License version 15 | # 2 along with this work; if not, see . 16 | # 17 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 18 | # or visit www.oracle.com if you need additional information or have any 19 | # questions. 20 | 21 | SCRIPTSDIR := $(BINDIR)/scripts 22 | DOCSDIR := $(BINDIR)/scripts/docs 23 | 24 | all: 25 | @echo "all:$(CURDIR)" 26 | 27 | clean: 28 | @echo "clean:$(CURDIR)" 29 | 30 | install: 31 | @echo "install:$(CURDIR)" 32 | mkdir -p $(SCRIPTSDIR) 33 | mkdir -p $(DOCSDIR) 34 | #net 35 | install -m 755 $(CURDIR)/net/arp_origin.d $(SCRIPTSDIR) 36 | install -m 755 $(CURDIR)/net/arp_origin_example.txt $(DOCSDIR) 37 | install -m 755 $(CURDIR)/net/rds_bcopy_metric.d $(SCRIPTSDIR) 38 | install -m 755 $(CURDIR)/net/rds_bcopy_metric_example.txt $(DOCSDIR) 39 | install -m 755 $(CURDIR)/net/rds_check_tx_stall.d $(SCRIPTSDIR) 40 | install -m 755 $(CURDIR)/net/rds_check_tx_stall_example.txt $(DOCSDIR) 41 | install -m 755 $(CURDIR)/net/rds_conn2irq.d $(SCRIPTSDIR) 42 | install -m 755 $(CURDIR)/net/rds_conn2irq_example.txt $(DOCSDIR) 43 | install -m 755 $(CURDIR)/net/rds_egress_TP.d $(SCRIPTSDIR) 44 | install -m 755 $(CURDIR)/net/rds_egress_TP_example.txt $(DOCSDIR) 45 | install -m 755 $(CURDIR)/net/rds_rdma_lat.d $(SCRIPTSDIR) 46 | install -m 755 $(CURDIR)/net/rds_rdma_lat_example.txt $(DOCSDIR) 47 | install -m 755 $(CURDIR)/net/rds_rdma_xfer_rate.d $(SCRIPTSDIR) 48 | install -m 755 $(CURDIR)/net/rds_rdma_xfer_rate_example.txt $(DOCSDIR) 49 | install -m 755 $(CURDIR)/net/rds_tx_funccount.d $(SCRIPTSDIR) 50 | install -m 755 $(CURDIR)/net/rds_tx_funccount_example.txt $(DOCSDIR) 51 | install -m 755 $(CURDIR)/net/ping_lat.d $(SCRIPTSDIR) 52 | install -m 755 $(CURDIR)/net/ping_lat_example.txt $(DOCSDIR) 53 | #io 54 | install -m 755 $(CURDIR)/io/nvme_io_comp.d $(SCRIPTSDIR) 55 | install -m 755 $(CURDIR)/io/nvme_io_comp_example.txt $(DOCSDIR) 56 | install -m 755 $(CURDIR)/io/scsi_latency.d $(SCRIPTSDIR) 57 | install -m 755 $(CURDIR)/io/scsi_latency_example.txt $(DOCSDIR) 58 | install -m 755 $(CURDIR)/io/scsi_queue.d $(SCRIPTSDIR) 59 | install -m 755 $(CURDIR)/io/scsi_queue_example.txt $(DOCSDIR) 60 | #core 61 | install -m 755 $(CURDIR)/core/spinlock_time.d $(SCRIPTSDIR) 62 | install -m 755 $(CURDIR)/core/spinlock_time_example.txt $(DOCSDIR) 63 | @echo "scripts successfully installed" 64 | 65 | uninstall: 66 | @echo "uninstall:$(CURDIR)" 67 | rm -rf $(SCRIPTSDIR) 68 | rm -rf $(DOCSDIR) 69 | @echo "scripts successfully uninstalled" 70 | -------------------------------------------------------------------------------- /scripts/core/spinlock_time.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2023, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author(s): Rama Nichanamatlu, Arumugam Kolappan 27 | * Purpose: Track 'the time interrupt is disabled' due to spin_lock_irq*() 28 | * kernel functions. Takes an argument[holdtime in ms] and prints the process 29 | * (and its call stack) which disables interrupt more than the given time. 30 | * Argument: Time in ms (Minimum value is 2) 31 | * 32 | * Start Bug: 32080059 33 | * Prerequisites: Refer to the file spinlock_time_example.txt 34 | * Sample output: Refer to the file spinlock_time_example.txt 35 | */ 36 | 37 | #define DT_VERSION_NUMBER_(M, m, u) \ 38 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 39 | 40 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 41 | #pragma D option lockmem=unlimited 42 | #endif 43 | 44 | #pragma D option cleanrate=50hz 45 | #pragma D option dynvarsize=16000000 46 | #pragma D option bufsize=16m 47 | 48 | dtrace:::BEGIN 49 | / $1 < 2 / 50 | { 51 | printf("Error: Argument should be >= 2 ms [%Y]\n", walltimestamp); 52 | exit(0); 53 | } 54 | 55 | dtrace:::BEGIN 56 | { 57 | printf("Processes holding spinlock_irq longer than %d ms [%Y]\n", $1, walltimestamp); 58 | printf("----------------------------------------------------------------------\n"); 59 | hold_time = $1 * 1000 * 1000; 60 | } 61 | 62 | /* 63 | * Use cpu number along with lock_pointer in the hash key (time[arg0, cpu]) to 64 | * handle the scenario: Second cpu acquires a lock before release_probe is fired 65 | * after first cpu unlocked it. 66 | */ 67 | lockstat:vmlinux:*spin_lock_irq*:spin-acquire 68 | { 69 | time[arg0, cpu] = timestamp; 70 | } 71 | 72 | lockstat:vmlinux:*:spin-release 73 | / time[arg0, cpu] && (this->nsec = timestamp - time[arg0, cpu]) >= hold_time / 74 | { 75 | printf("\n[%Y] time=%d ms lock=%p pid=%d cpu=%d cmd=%s\n", 76 | walltimestamp, this->nsec/1000000, 77 | arg0, pid, cpu, curthread->comm); 78 | stack(); 79 | time[arg0, cpu] = 0; 80 | 81 | /* Resetting the clause variable 'this->nsec' seems helping to avoid 82 | * 'dynamic variable drops with non-empty dirty list' 83 | * with Oracle Linux DTrace 1.0 84 | */ 85 | this->nsec = 0; 86 | } 87 | 88 | lockstat:vmlinux:*:spin-release 89 | / time[arg0, cpu] / 90 | { 91 | time[arg0, cpu] = 0; 92 | this->nsec = 0; 93 | } 94 | -------------------------------------------------------------------------------- /scripts/net/rds_tx_funccount_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: This script is used to trace and display rate of calls for 2 | send_xmit, ib_xmit and send_cqe_handler per 10 sec. 3 | 4 | Arguments: None 5 | 6 | Output format: [] no_of_calls/10sec 7 | 8 | Sample output: Shown below 9 | 10 | # ./rds_tx_funccount.d 11 | 2022 Oct 26 23:52:16: rate of calls for send_xmit, ib_xmit and send_cqe_handler. ctrl+c to stop 12 | --- 2022 Oct 26 23:52:26 --- 13 | [<192.168.0.5,192.168.0.23,2> 03ib_xmits] 3 14 | [<192.168.0.5,192.168.0.23,3> 03ib_xmits] 3 15 | [<192.168.0.5,192.168.0.23,3> 04send_cqe_handler] 3 16 | [<192.168.0.5,192.168.0.23,5> 01send_msg] 3 17 | [<192.168.0.5,192.168.0.24,2> 03ib_xmits] 3 18 | [<192.168.0.5,192.168.0.24,2> 04send_cqe_handler] 3 19 | [<192.168.0.5,192.168.0.24,4> 03ib_xmits] 3 20 | [<192.168.0.5,192.168.0.24,4> 04send_cqe_handler] 3 21 | [<192.168.0.5,192.168.0.24,5> 03ib_xmits] 3 22 | [<192.168.0.5,192.168.0.24,5> 04send_cqe_handler] 3 23 | [<192.168.0.5,192.168.0.25,2> 03ib_xmits] 3 24 | [<192.168.0.5,192.168.0.25,3> 03ib_xmits] 3 25 | [<192.168.0.5,192.168.0.25,5> 01send_msg] 3 26 | [<192.168.0.5,192.168.0.27,2> 04send_cqe_handler] 3 27 | [<192.168.0.5,192.168.0.27,3> 03ib_xmits] 3 28 | [<192.168.0.5,192.168.0.27,3> 04send_cqe_handler] 3 29 | [<192.168.0.5,192.168.0.25,0> 04send_cqe_handler] 71 30 | [<192.168.0.6,192.168.0.26,0> 02send_xmits] 72 31 | [<192.168.0.5,192.168.0.8,0> 01send_msg] 74 32 | [<192.168.0.5,192.168.0.8,0> 03ib_xmits] 75 33 | [<192.168.0.5,192.168.0.8,0> 04send_cqe_handler] 83 34 | [<192.168.0.5,192.168.0.27,0> 02send_xmits] 90 35 | [<192.168.0.6,192.168.0.24,0> 02send_xmits] 92 36 | [<192.168.0.6,192.168.0.28,0> 02send_xmits] 110 37 | [<192.168.0.5,192.168.0.23,0> 02send_xmits] 113 38 | [<192.168.0.6,192.168.0.8,0> 02send_xmits] 131 39 | [<192.168.0.5,192.168.0.25,0> 02send_xmits] 134 40 | [<192.168.0.5,192.168.0.8,0> 02send_xmits] 158 41 | --- 2022 Oct 26 23:52:36 --- 42 | [<192.168.0.6,192.168.0.28,5> 01send_msg] 1 43 | [<192.168.0.6,192.168.0.7,0> 01send_msg] 2 44 | [<192.168.0.5,192.168.0.23,2> 03ib_xmits] 3 45 | [<192.168.0.5,192.168.0.23,2> 04send_cqe_handler] 3 46 | [<192.168.0.5,192.168.0.23,3> 03ib_xmits] 3 47 | [<192.168.0.5,192.168.0.23,3> 04send_cqe_handler] 3 48 | [<192.168.0.5,192.168.0.23,5> 03ib_xmits] 3 49 | [<192.168.0.5,192.168.0.24,2> 03ib_xmits] 3 50 | [<192.168.0.5,192.168.0.24,2> 04send_cqe_handler] 3 51 | [<192.168.0.6,192.168.0.28,4> 03ib_xmits] 10 52 | [<192.168.0.6,192.168.0.28,4> 04send_cqe_handler] 10 53 | [<192.168.0.5,192.168.0.23,4> 01send_msg] 11 54 | [<192.168.0.5,192.168.0.25,4> 01send_msg] 11 55 | [<192.168.0.5,192.168.0.7,0> 03ib_xmits] 11 56 | [<192.168.0.5,192.168.0.27,4> 01send_msg] 12 57 | [<192.168.0.5,192.168.0.7,0> 04send_cqe_handler] 12 58 | [<192.168.0.5,192.168.0.23,4> 03ib_xmits] 13 59 | [<192.168.0.5,192.168.0.23,4> 04send_cqe_handler] 13 60 | [<192.168.0.5,192.168.0.25,4> 03ib_xmits] 13 61 | [<192.168.0.5,192.168.0.25,4> 04send_cqe_handler] 13 62 | [<192.168.0.5,192.168.0.27,4> 03ib_xmits] 14 63 | [<192.168.0.6,192.168.0.24,4> 01send_msg] 14 64 | [<192.168.0.5,192.168.0.27,4> 04send_cqe_handler] 15 65 | [<192.168.0.6,192.168.0.24,4> 03ib_xmits] 16 66 | [<192.168.0.6,192.168.0.24,4> 04send_cqe_handler] 17 67 | [<192.168.0.6,192.168.0.28,4> 02send_xmits] 20 68 | [<192.168.0.5,192.168.0.8,0> 02send_xmits] 142 69 | ^C 70 | -------------------------------------------------------------------------------- /scripts/net/rds_check_tx_stall.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author(s): Rama Nichanamatlu, Praveen Kumar Kannoju. 27 | * Purpose: This script monitors the completions on all the connections and 28 | * calculates the time difference between the consecutive completions. If 29 | * this time difference exceeds the threshold time (in microseconds), given 30 | * as input argument, message will be printed providing the details of the 31 | * connection parameters. Minimum input value is 500 microseconds. 32 | * Prerequisites: Refer to the file rds_check_tx_stall_example.txt 33 | * Sample output: Refer to the file rds_check_tx_stall_example.txt 34 | */ 35 | 36 | #define DT_VERSION_NUMBER_(M, m, u) \ 37 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 38 | 39 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 40 | #pragma D option lockmem=unlimited 41 | #endif 42 | 43 | uint64_t rds_connection[struct rds_connection *]; 44 | 45 | dtrace:::BEGIN 46 | / $1 < 500 / 47 | { 48 | printf("\n Error: Argument should be >= 500 ( microseconds) [%Y]\n\n", walltimestamp); 49 | printf("-----------------------------------------------------------------------------\n"); 50 | exit(0); 51 | } 52 | 53 | dtrace:::BEGIN 54 | { 55 | time_to_wait = $1; 56 | printf("%Y Print each time there is a gap that exceeds the threshold, ctrl+c to stop.\n", walltimestamp); 57 | } 58 | 59 | fbt:rds_rdma:rds_ib_send_cqe_handler:entry 60 | { 61 | this->ic = (struct rds_ib_connection *)arg0; 62 | this->conn = (struct rds_connection *)this->ic->conn; 63 | 64 | self->latency = rds_connection[this->conn] > 0 ? (timestamp - rds_connection[this->conn])/1000 : 0; 65 | rds_connection[this->conn] = timestamp; 66 | } 67 | 68 | fbt:rds_rdma:rds_ib_send_cqe_handler:entry 69 | / self->latency > time_to_wait / 70 | { 71 | this->ic = (struct rds_ib_connection *)arg0; 72 | this->conn = (struct rds_connection *)this->ic->conn; 73 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 74 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 75 | this->tos = this->conn->c_tos; 76 | 77 | printf("[%s,%s,%d] last send completion: %d usecs ago\n", 78 | inet_ntoa(this->sip), inet_ntoa(this->dip), 79 | this->tos, self->latency); 80 | } 81 | -------------------------------------------------------------------------------- /scripts/io/scsi_latency_example.txt: -------------------------------------------------------------------------------- 1 | Below are sample output, the script gathers SCSI IO statistics given a sampling 2 | interval and displays the summary ouptut in histogram format. The script takes 3 | 2 arguments, the first argument is sampling interval in seconds for latency, 4 | the second argument is latency threshold time in milli-seconds for IO completion. 5 | 6 | The left column of the histogram 'value' output is the latency time in milli-seconds 7 | and the right 'count' are the number of IO commands completed. 8 | 9 | # scsi-latency.d 5 1 (5 being sample interval, and 1 is threshold for IO completion in ms). 10 | sdbz 11 | value ------------- Distribution ------------- count 12 | -1 | 0 13 | 0 |@@@@@@@@@@@@@@@@@@@@@@ 16616 14 | 1 |@@@@@@@@@@@@@@@@ 12411 15 | 2 |@@ 1479 16 | 4 | 229 17 | 8 | 10 18 | 16 | 0 19 | (ms) 20 | sdef 21 | value ------------- Distribution ------------- count 22 | -1 | 0 23 | 0 |@@@@@@@@@@@@@@@@@@@@@@@@@@ 29766 24 | 1 |@@@@@@@@@@@@ 14042 25 | 2 |@ 1073 26 | 4 | 186 27 | 8 | 2 28 | 16 | 0 29 | (ms) 30 | 31 | Below is a sample of abnormally higher latency of IO completion. 32 | 33 | # scsi-latency.d 5 50 34 | 2022 Oct 21 15:49:16 35 | ======================================================= 36 | sddb 7:0:2:229 37 | value ------------- Distribution ------------- count 38 | -1 | 0 39 | 0 |@@ 2 40 | 1 | 0 41 | 2 |@@@@@ 5 42 | 4 |@ 1 43 | 8 |@@ 2 44 | 16 | 0 45 | 32 | 0 46 | 64 | 0 47 | 128 | 0 48 | 256 | 0 49 | 512 | 0 50 | 1024 | 0 51 | 2048 | 0 52 | 4096 | 0 53 | 8192 | 0 54 | 16384 |@@@@@@@@@@@@@@@@@@@@@ 20 55 | 32768 |@@@@@@@@@ 9 56 | 65536 | 0 57 | (ms) 58 | ======================================================== 59 | 60 | -------------------------------------------------------------------------------- /oled.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """oled-tools driver""" 23 | 24 | import argparse 25 | import sys 26 | import os 27 | from typing import Sequence 28 | 29 | # Oracle Linux Enhanced Diagnostic Tools 30 | MAJOR = "0" 31 | MINOR = "7" 32 | 33 | BINDIR = "/usr/libexec/oled-tools" 34 | 35 | # Valid oled subcomands 36 | OLED_CMDS = ( 37 | "kstack", "lkce", "memstate", "syswatch", "scanfs", "vmcore_sz") 38 | 39 | 40 | def parse_args(args: Sequence[str]) -> argparse.Namespace: 41 | """Parse CLI arguments.""" 42 | parser = argparse.ArgumentParser( 43 | formatter_class=argparse.RawDescriptionHelpFormatter, 44 | usage="%(prog)s {-h | -v | COMMAND [ARGS]}", 45 | description="""\ 46 | Valid commands: 47 | lkce -- Linux Kernel Core Extractor 48 | memstate -- Capture and analyze memory usage statistics 49 | kstack -- Gather kernel stack based on the process status or PID 50 | syswatch -- Execute user-provided commands based on the CPU utilization 51 | scanfs -- Scan KVM images for corruption, supports XFS and EXT4 52 | vmcore_sz -- Estimating vmcore size before kernel dump 53 | """, 54 | epilog="NOTE: Must run as root.") 55 | 56 | parser.add_argument( 57 | "-v", "--version", action="version", 58 | version=( 59 | f"Oracle Linux Enhanced Diagnostics (oled) v{MAJOR}.{MINOR} " 60 | "(developer preview release)")) 61 | parser.add_argument( 62 | "command", metavar="COMMAND", choices=OLED_CMDS, 63 | help=argparse.SUPPRESS) 64 | parser.add_argument( 65 | "args", metavar="ARGS", nargs=argparse.REMAINDER, 66 | help=argparse.SUPPRESS) 67 | 68 | return parser.parse_args(args) 69 | 70 | 71 | def main(args: Sequence[str]) -> None: 72 | """Main function.""" 73 | 74 | options = parse_args(args) 75 | 76 | # only allow root to execute subcommands 77 | if os.getuid() != 0: 78 | print("Run as root only", file=sys.stderr) 79 | sys.exit(1) 80 | 81 | cmd = options.command 82 | 83 | exec_path = os.path.join(BINDIR, cmd) 84 | prog_name = f"oled-{cmd}" 85 | exec_args = [prog_name] + options.args 86 | 87 | try: 88 | os.execv(exec_path, exec_args) # nosec 89 | except Exception: 90 | print(f"Error executing: {exec_path} {exec_args}", file=sys.stderr) 91 | raise 92 | 93 | 94 | if __name__ == "__main__": 95 | main(sys.argv[1:]) 96 | -------------------------------------------------------------------------------- /scripts/net/rds_rdma_xfer_rate_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: The script shows the rdma transfer rate[MB/s] for each rds connection. 2 | It also shows histograms of inverse bandwidth in nsecs/KB [Avg number of nsecs 3 | to transfer 1KB of data]. 4 | 5 | Arguments: None 6 | 7 | Output format: [src-ip, dst-ip, tos, ] 8 | 9 | Sample output: Shown below. 10 | # ./rds_rdma_xfer_rate.d 11 | 2022 Oct 25 10:25:17 12 | [ op] MB/s 13 | 2022 Oct 25 10:25:27 14 | [<192.168.100.125,192.168.100.124,0> r] 1059 15 | 2022 Oct 25 10:25:37 16 | [<192.168.100.125,192.168.100.124,0> r] 1059 17 | 2022 Oct 25 10:25:47 18 | [<192.168.100.125,192.168.100.124,0> r] 1056 19 | 2022 Oct 25 10:25:57 20 | [<192.168.100.125,192.168.100.124,0> r] 1049 21 | ^C << ---------- Press Cntr+c and stop the script to get histogram output. 22 | 2022 Oct 25 10:26:06 - Histogram of inverse bandwidth reported in nsecs/KB 23 | [<192.168.100.125,192.168.100.124,0> r] 24 | 25 | value ------------- Distribution ------------- count 26 | 128 | 0 27 | 256 |@@ 15827 28 | 512 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 373157 29 | 1024 | 3040 30 | 2048 | 0 31 | 32 | Transfer rdma bytes of 1 MB size. 33 | # rds-stress 34 | waiting for incoming connection on 0.0.0.0:4000 35 | accepted connection from 192.168.100.124:34403 on 192.168.100.125:4000 36 | negotiated options, tasks will start in 2 seconds 37 | Starting up.... 38 | tsks tx/s rx/s tx+rx K/s mbi K/s mbo K/s tx us/c rtt us cpu % 39 | 1 0 2411 2556.60 2467191.36 0.00 0.00 0.00 -1.00 40 | 1 0 2403 2548.11 2458041.90 0.00 0.00 0.00 -1.00 41 | 1 0 2502 2653.08 2559294.20 0.00 0.00 0.00 -1.00 42 | 1 0 2382 2525.92 2436633.90 0.00 0.00 0.00 -1.00 43 | 1 0 2396 2540.57 2449799.92 0.00 0.00 0.00 -1.00 44 | 45 | # rds-stress -r 192.168.100.124 -s 192.168.100.125 -D 1M -o -M 1 46 | connecting to 192.168.100.125:4000 47 | negotiated options, tasks will start in 2 seconds 48 | Starting up.... 49 | tsks tx/s rx/s tx+rx K/s mbi K/s mbo K/s tx us/c rtt us cpu % 50 | 1 2410 0 2557.84 0.00 2466458.36 75.45 402.28 -1.00 51 | 1 2401 0 2548.27 0.00 2458193.82 74.44 404.07 -1.00 52 | 1 2499 0 2652.33 0.00 2558571.75 72.08 387.50 -1.00 53 | 1 2380 0 2525.99 0.00 2436703.32 74.09 407.52 -1.00 54 | 55 | # ./rds_rdma_xfer_rate.d 56 | 2022 Oct 25 10:32:05 57 | [ op] MB/s 58 | 2022 Oct 25 10:32:15 59 | [<192.168.100.125,192.168.100.124,0> r] 2365 60 | 2022 Oct 25 10:32:25 61 | [<192.168.100.125,192.168.100.124,0> r] 2424 62 | 2022 Oct 25 10:32:35 63 | [<192.168.100.125,192.168.100.124,0> r] 2398 64 | 2022 Oct 25 10:32:45 65 | [<192.168.100.125,192.168.100.124,0> r] 2443 66 | ^C 67 | 2022 Oct 25 10:32:47 - Histogram of inverse bandwidth reported in nsecs/KB 68 | [<192.168.100.125,192.168.100.124,0> r] 69 | 70 | value ------------- Distribution ------------- count 71 | 64 | 0 72 | 128 | 1 73 | 256 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 101206 74 | 512 | 1 75 | 1024 | 0 76 | -------------------------------------------------------------------------------- /scripts/net/rds_rdma_xfer_rate.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author: Manjunath Patil 27 | * Purpose: The script shows the rdma transfer rate[MB/s] for each rds 28 | * connection. It also shows histograms of inverse bandwidth in nsecs/KB 29 | * [Avg number of nsecs to transfer 1KB of data]. 30 | * Prerequisites: Refer to the file rds_rdma_xfer_rate_example.txt 31 | * Sample output: Refer to the file rds_rdma_xfer_rate_example.txt 32 | */ 33 | 34 | #define DT_VERSION_NUMBER_(M, m, u) \ 35 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 36 | 37 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 38 | #pragma D option lockmem=unlimited 39 | #endif 40 | 41 | #define container_of(__ptr, __type, __member) ((__type *)((unsigned long long)__ptr - (unsigned long long)offsetof(__type, __member))) 42 | 43 | dtrace:::BEGIN 44 | { 45 | printf("%Y\n%-40s %-10s\n", walltimestamp, "[ op]", "MB/s"); 46 | } 47 | 48 | fbt:rds:rds_cmsg_rdma_args:entry 49 | { 50 | this->rm = (struct rds_message *)arg1; 51 | this->op = (struct rm_rdma_op *)&this->rm->rdma; 52 | track_rdma[this->op] = timestamp; 53 | } 54 | 55 | /* rds_ib_send_unmap_rdma.isra.9 */ 56 | fbt::rds_ib_send_unmap_rdma*:entry 57 | / (this->op = (struct rm_rdma_op *)arg1) && track_rdma[this->op] / 58 | { 59 | this->rdma_op = this->op->op_write ? 'w' : 'r'; 60 | this->rm = (struct rds_message *) container_of(this->op, struct rds_message, rdma); 61 | this->cpath = (struct rds_conn_path *)this->rm->m_conn_path; 62 | this->conn = (struct rds_connection *)this->cpath->cp_conn; 63 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 64 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 65 | this->tos = this->conn->c_tos; 66 | 67 | @rdma_xfers[inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, this->rdma_op] = sum(this->op->op_bytes); 68 | 69 | /* Histogram of inverse bandwidth */ 70 | this->quanta = (timestamp - track_rdma[this->op])/(this->op->op_bytes/1024); 71 | @rdma_hist[inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, this->rdma_op] = quantize(this->quanta); 72 | track_rdma[this->op] = 0; 73 | } 74 | 75 | tick-10s 76 | { 77 | printf("%Y\n", walltimestamp); 78 | 79 | /* convert bytes/10 secs to MB/s */ 80 | normalize(@rdma_xfers, 1024*1024*10); 81 | printa("[<%s,%s,%d> %c] %-@d\n", @rdma_xfers); 82 | clear(@rdma_xfers); 83 | } 84 | 85 | /* Histograming inverse bandwidth. Printed when dtrace is ended */ 86 | dtrace:::END 87 | { 88 | printf("%Y - Histogram of inverse bandwidth reported in nsecs/KB\n", walltimestamp); 89 | printa("[<%s,%s,%d> %c] \n%@d\n", @rdma_hist); 90 | } 91 | -------------------------------------------------------------------------------- /tools/kstack/kstack.man: -------------------------------------------------------------------------------- 1 | .TH KSTACK 8 2 | .SH NAME 3 | kstack \- Collect the kernel stack trace for selected processes. 4 | .SH SYNOPSIS 5 | \fB kstack\fR 6 | \fB-a\fR|\fB-DIRSZ -p\fI PID\fR[,...] 7 | [ \fB\-h\fR|\fB--help\fR] 8 | [ \fB\-v\fR|\fB--verbose\fR] 9 | [ \fB-b\fR 10 | [ \fB\-t \fIMINUTES \fR] 11 | [ \fB\-i \fISECONDS \fR] 12 | [ \fB\-d \fIDIRECTORY \fR] 13 | [ \fB\-m \fIMAX_SIZE \fR] 14 | [ \fB\-n \fINUMBER_OF_FILES \fR] 15 | [ \fB\-x \fIMAX_SPACE \fR] ] 16 | .SH DESCRIPTION 17 | \fBkstack\fR is used to capture the kernel stack trace of a selected process or group of processes. \fBkstack\fR first captures all needed data from the /proc file system and then parses the data based on the selection criteria. The data is grouped by stack, so that duplicate stacks are not printed, and processes that share a stack are displayed together. All of the \fBSELECTION OPTIONS\fR, with the exception of \fB-a\fR, can be combined. See \fBEXAMPLES\fR below. 18 | 19 | \fBkstack\fR can be run in foreground or background mode. In foreground mode, the default, a single sample will be displayed to stdout. In background mode, \fB-b\fR, the tool will run for a specified amount of time and gather the kernel stacks at regular intervals. The data will be written to files in /var/oled/kstack. The \fB-d\fR argument can be used to redirect the files to a different location. The log files written in background mode can be further controlled using using the \fB-m\fR, \fB-n\fR, and \fB-x\fR options. 20 | 21 | Because most of the time it will be processes in D state that we are interested in, \fBkstack\fR has an option to print a warning to syslog if a certain number of D state processes are exceeded. 22 | .SH SELECTION OPTIONS 23 | .TP 24 | \fB\-a\fR 25 | All processes. 26 | .TP 27 | \fB\-D\fR 28 | Processes waiting on I/O or a lock. 29 | .TP 30 | \fB\-R\fR 31 | Running processes. 32 | .TP 33 | \fB\-S\fR 34 | Sleeping processes. 35 | .TP 36 | \fB\-Z\fR 37 | Zombie processes. 38 | .TP 39 | \fB\-p\fI PID\fR[,...] 40 | Dump stack traces for \fIPID\fR. More than one \fIPID\fR can be specified if separated by commas. 41 | .SH MISC OPTIONS 42 | .TP 43 | \fB\-h\fR,\fB--help\fr 44 | Display help information. 45 | .TP 46 | \fB\-v\fR,\fB--verbose\fr 47 | Display debugging data. 48 | .SH BACKGROUND OPTIONS 49 | .TP 50 | \fB\-b\fR 51 | Run in background mode. 52 | .TP 53 | \fB\-t \fIMINUTES\fR 54 | Specify the number of \fIMINUTES\fR to run when in background mode. Default is 30. 55 | .TP 56 | \fB\-i \fISECONDS\fR 57 | Specify the number of \fISECONDS\fR between samples. Default is 60. 58 | .SH LOG FILE OPTIONS 59 | .TP 60 | \fB\-d \fIDIRECTORY\fR 61 | The target directory to write log files. The default is /var/oled/kstack. 62 | .TP 63 | \fB\-m \fIMAX_SIZE\fR 64 | The maximum size, in megabytes, for the log file. Once the log file exceeds this size it will be rotated and compressed. Default value is 1Mb. 65 | .TP 66 | \fB\-n \fIFILES_NUMBER\fR 67 | The number of rotated log files to retain. Default value is 5. 68 | .TP 69 | \fB\-x \fIMAX_SPACE\fR 70 | This is the maximum percent used on the target file system. The default is 85(%). If the target file system is at or above this limit, then the program will refuse to run. 71 | .SH ASSUMPTIONS 72 | \fBkstack\fR must be run by root. 73 | .SH EXAMPLES 74 | .TP 75 | To select all running processes: 76 | \fBkstack -R\fR 77 | .TP 78 | To select all running processes and all processes waiting on I/O: 79 | \fBkstack -RD\fR 80 | .TP 81 | To select all processes waiting on I/O and also processes 1234 and 1235: 82 | \fBkstack -Dp 1234,1235\fR 83 | .TP 84 | To select all processes waiting on I/O, every 30 seconds, for 30 minutes: 85 | \fBkstack -D -b -t 30 -i 30\fR 86 | .SH FILES 87 | .TP 88 | /var/oled/kstack 89 | Destination directory for log files when in background mode. 90 | .SH REPORTING BUGS 91 | .TP 92 | Please contact Oracle Linux Support to report any bugs for OLED tools. 93 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/rss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """ 23 | Helper module to extract memory usage (RSS) from /proc//status. 24 | """ 25 | 26 | from collections import OrderedDict 27 | import glob 28 | import re 29 | import time 30 | from memstate_lib import Base 31 | from memstate_lib import constants 32 | 33 | 34 | class Rss(Base): 35 | """ Extracts per-process stats from /proc//status """ 36 | 37 | def __display_top_vmrss(self, num): 38 | """ 39 | Extract per-process VmRSS from /proc//status. 40 | Sort in descending order, and display the top memory hoggers. 41 | @param num: The number of top memory users to print. 42 | """ 43 | # pylint: disable=too-many-locals 44 | rss = {} 45 | num_printed = 0 46 | start_time = time.time() 47 | num_files_scanned = 0 48 | name_search_str = r'^Name:\s*.+' 49 | vmrss_search_str = r'^VmRSS:\s*[0-9]+' 50 | for filestr in glob.glob("/proc/*/status"): 51 | pid = filestr.split("/")[2] 52 | if pid.isdigit() is False: 53 | continue 54 | try: 55 | with open(filestr, "r", encoding="utf8") as status_fd: 56 | data = status_fd.read() 57 | num_files_scanned = num_files_scanned + 1 58 | name = re.search(name_search_str, data) 59 | comm = name.group(0).split(":")[1].strip() 60 | hdr = f"{comm}({pid})" 61 | match_list = re.findall( 62 | vmrss_search_str, data, flags=re.MULTILINE) 63 | for match in match_list: 64 | rss_kb = int(match.split(":")[1].strip()) 65 | rss[hdr] = rss_kb + rss.get(hdr, 0) 66 | except IOError: 67 | pass 68 | end_time = time.time() 69 | self.log_debug( 70 | f"Time taken to extract memory stats from {num_files_scanned}" 71 | f" /proc//status files is {round(end_time - start_time)} " 72 | "second(s).") 73 | rss_sorted = OrderedDict( 74 | sorted(rss.items(), key=lambda x: x[1], reverse=True)) 75 | for comm, rss_kb in rss_sorted.items(): 76 | if num != constants.NO_LIMIT and num_printed >= num: 77 | break 78 | self.print_pretty_kb(comm, rss_kb) 79 | num_printed += 1 80 | 81 | def memstate_check_rss(self, num=constants.NUM_TOP_MEM_USERS): 82 | """Check per-process memory usage (RSS).""" 83 | if num == constants.NO_LIMIT: 84 | hdr = "PROCESS MEMORY USAGE (in KB):" 85 | else: 86 | hdr = f"TOP {num} MEMORY CONSUMERS (in KB):" 87 | print(hdr) 88 | print(f"{'PROCESS(PID)': <30}{'RSS': >12}") 89 | self.__display_top_vmrss(num) 90 | print("") 91 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Common global constants used by memstate.""" 23 | 24 | # 25 | # Begin: Global Constants 26 | # 27 | # Do not change these values! 28 | # 29 | ONE_KB = 1024.00 30 | ONE_MB = ONE_KB * ONE_KB 31 | ONE_GB = ONE_KB * ONE_MB 32 | VMA_KB = 256 * ONE_KB 33 | PERCENT = 0.01 34 | NO_LIMIT = -1 35 | DEFAULT_SHOW_PSS_SUMMARY = -1 36 | # For parsing output from /proc/buddyinfo. Lower orders are 0-3, higher order 37 | # buckets are 4-10. 38 | BUDDYINFO_START_INDEX = 0 39 | LOW_ORDER_MAX_INDEX = 3 40 | HIGH_ORDER_MAX_INDEX = 10 41 | # Hardcoded constants. These could theoretically change in newer UEKs. 42 | FRAG_SIZE_KB = 16 43 | STRUCT_PAGE_SIZE = 64 44 | # The assumption here is that the system can be at most an 8-socket system. 45 | MAX_NUMA_NODES = 8 46 | 47 | # 48 | # End: Global Constants 49 | # 50 | 51 | # 52 | # Begin: Config Values 53 | # 54 | 55 | # 56 | # These values control how many lines of output are printed for different 57 | # categories. Change them if you want more/less output. 58 | # 59 | NUM_TOP_SLAB_CACHES = 10 # N biggest slab caches 60 | NUM_TOP_MEM_USERS = 10 # N top memory consumers (user processes) 61 | NUM_TOP_SWAP_USERS = 10 # N top users of swap space 62 | NUM_TOP_NUMA_MAPS = 20 # N biggest consumers of memory on each NUMA node 63 | 64 | # 65 | # Thresholds for warnings 66 | # 67 | # For instance, print a warning if X slab cache usage exceeds 68 | # SLAB_USE_PERCENT * (TotalRAM - TotalHugepages). These values are a bit 69 | # arbitrary, and represent what we think are "safe" thresholds for most 70 | # workloads. Change these to alter thresholds when warnings are printed. 71 | SLAB_USE_PERCENT = 30 * PERCENT 72 | RDS_USE_PERCENT = 10 * PERCENT 73 | PAGETABLES_USE_PERCENT = 2 * PERCENT 74 | UNACCOUNTED_THRESHOLD = 10 * PERCENT 75 | WSF_THRESHOLD = 100 76 | 77 | # Thresholds for NUMA imbalance checks 78 | # Min MemFree/MemTotal ratio across all NUMA nodes 79 | NUMASTAT_MIN_MEMFREE_PERCENT = 10 * PERCENT 80 | 81 | # max(MemFree[Nx])/min(MemFree[Ny]) ratio 82 | NUMASTAT_DIFF_PERCENT = 200 * PERCENT 83 | 84 | # Buddyinfo fragmentation level 85 | # Percent of free memory that's available in low (0-3) orders 86 | FRAG_LEVEL_LOW_ORDERS = 95 87 | 88 | ORDER_KB = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] # Order-x in KB 89 | 90 | # Output log file, lock file and default execution frequency 91 | LOGROTATEFILE = "/etc/logrotate.d/memstate" 92 | LOGFILE = "/var/oled/memstate/memstate.log" 93 | MEMSTATE_DEBUG_LOG = "/var/oled/memstate/memstate_debug.log" 94 | LOCK_FILE_DIR = "/run/lock/" 95 | LOCK_FILE_DIR_OL6 = "/var/run/" 96 | DEFAULT_INTERVAL = 30 # Unit: seconds 97 | 98 | # This is the max allowed "Use%" field in 'df -h' output for the logfile path; 99 | # the tool will exit if there's not enough space on disk. 100 | MAX_DISKSPACE_UTIL = 85 101 | MIN_DISKSPACE_NEEDED = 50 # Unit: MB 102 | 103 | # 104 | # End: config values 105 | # 106 | -------------------------------------------------------------------------------- /scripts/net/rds_tx_funccount.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author: Manjunath Patil 27 | * Purpose: This script is used to trace and display rate of calls for 28 | * sendmsg, send_xmit, ib_xmit and send_cqe_handler per 10 sec. 29 | * Prerequisites: Refer to the file rds_tx_funccount_example.txt 30 | * Sample output: Refer to the file rds_tx_funccount_example.txt 31 | */ 32 | 33 | #define DT_VERSION_NUMBER_(M, m, u) \ 34 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 35 | 36 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 37 | #pragma D option lockmem=unlimited 38 | #endif 39 | 40 | dtrace:::BEGIN 41 | { 42 | printf("%Y: rate of calls for sendmsg, send_xmit, ib_xmit and send_cqe_handler. ctrl+c to stop\n", walltimestamp); 43 | } 44 | 45 | fbt::rds_sendmsg:entry 46 | { 47 | this->socket = (struct socket *)arg0; 48 | this->sock = this->socket->sk; 49 | self->rs = (struct rds_sock *)this->sock; 50 | self->payload_len = (size_t)arg2; 51 | } 52 | 53 | fbt::rds_sendmsg:return 54 | / self->rs && self->payload_len == (size_t)arg1 / 55 | { 56 | this->conn = (struct rds_connection *)self->rs->rs_conn; 57 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 58 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 59 | this->tos = this->conn->c_tos; 60 | 61 | @counts[inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, "01send_msg"] = count(); 62 | self->rs = 0; 63 | self->payload_len = 0; 64 | } 65 | 66 | /* in case rds_sendmsg() returns with error, clear memory */ 67 | fbt::rds_sendmsg:return 68 | { 69 | self->rs = 0; 70 | self->payload_len = 0; 71 | } 72 | 73 | fbt:rds:rds_send_xmit:entry 74 | { 75 | this->cpath = (struct rds_conn_path *)arg0; 76 | this->conn = (struct rds_connection *)this->cpath->cp_conn; 77 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 78 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 79 | this->tos = this->conn->c_tos; 80 | 81 | @counts[inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, "02send_xmits"] = count(); 82 | } 83 | 84 | fbt:rds_rdma:rds_ib_xmit:entry 85 | { 86 | this->conn = (struct rds_connection *)arg0; 87 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 88 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 89 | this->tos = this->conn->c_tos; 90 | 91 | @counts[inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, "03ib_xmits"] = count(); 92 | } 93 | 94 | fbt:rds_rdma:rds_ib_send_cqe_handler:entry 95 | { 96 | this->ic = (struct rds_ib_connection *)arg0; 97 | this->conn = (struct rds_connection *)this->ic->conn; 98 | this->sip = &this->conn->c_laddr.in6_u.u6_addr32[3]; 99 | this->dip = &this->conn->c_faddr.in6_u.u6_addr32[3]; 100 | this->tos = this->conn->c_tos; 101 | 102 | @counts[inet_ntoa(this->sip), inet_ntoa(this->dip), this->tos, "04send_cqe_handler"] = count(); 103 | } 104 | 105 | END,tick-10s 106 | { 107 | printf("--- %Y ---\n", walltimestamp); 108 | printa("[<%s,%s,%d> %s] %-@d\n", @counts); 109 | clear(@counts); 110 | } 111 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/swap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Helper module to analyze swap state.""" 23 | 24 | from collections import OrderedDict 25 | import glob 26 | import re 27 | import time 28 | from memstate_lib import Base 29 | from memstate_lib import Meminfo 30 | from memstate_lib import constants 31 | 32 | 33 | class Swap(Base): 34 | """ Extracts per-process swap usage from /proc//status """ 35 | 36 | def __display_top_vmswap(self, num): 37 | """ 38 | Sum up all VmSwap per-process from /proc//status. 39 | Sort in descending order, and display the top users of swap. 40 | @param num: The number of top swap users to print. 41 | """ 42 | # pylint: disable=too-many-locals 43 | swap = {} 44 | num_printed = 0 45 | meminfo = Meminfo() 46 | swap_used_kb = meminfo.get_swap_used_kb() 47 | if swap_used_kb == 0: 48 | print("No swap usage found.") 49 | return 50 | start_time = time.time() 51 | num_files_scanned = 0 52 | for filestr in glob.glob("/proc/*/status"): 53 | pid = filestr.split("/")[2] 54 | name_search_str = r'^Name:\s*.+' 55 | vmswap_search_str = r'^VmSwap:\s*[0-9]+' 56 | try: 57 | with open(filestr, "r", encoding="utf8") as status_fd: 58 | data = status_fd.read() 59 | num_files_scanned = num_files_scanned + 1 60 | name = re.search(name_search_str, data) 61 | comm = name.group(0).split(":")[1].strip() 62 | hdr = f"{comm}(pid={pid})" 63 | match_list = re.findall( 64 | vmswap_search_str, data, flags=re.MULTILINE) 65 | for match in match_list: 66 | swap_kb = int(match.split(":")[1].strip()) 67 | swap[hdr] = swap_kb + swap.get(hdr, 0) 68 | time.sleep(0.01) # Sleep for 10 ms to avoid hogging CPU 69 | except OSError: 70 | pass 71 | end_time = time.time() 72 | self.log_debug( 73 | f"Time taken to extract swap usage values from {num_files_scanned}" 74 | f" /proc//status files is {round(end_time - start_time)} " 75 | "second(s).") 76 | swap_sorted = OrderedDict( 77 | sorted(swap.items(), key=lambda x: x[1], reverse=True)) 78 | for comm, swap_kb in swap_sorted.items(): 79 | if num != constants.NO_LIMIT and num_printed >= num: 80 | break 81 | if int(swap_kb) == 0: 82 | if num_printed == 0: 83 | print("No swap usage found.") 84 | break 85 | self.print_pretty_kb(comm, swap_kb) 86 | num_printed += 1 87 | 88 | def memstate_check_swap(self, num=constants.NUM_TOP_SWAP_USERS): 89 | """Check state of swap.""" 90 | if num == constants.NO_LIMIT: 91 | hdr = "SWAP USERS:" 92 | else: 93 | hdr = f"TOP {num} SWAP SPACE CONSUMERS:" 94 | print(hdr) 95 | self.__display_top_vmswap(num) 96 | print("") 97 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/logfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Logging module.""" 23 | 24 | import math 25 | import os 26 | import shutil 27 | import subprocess # nosec 28 | import sys 29 | from memstate_lib import Base 30 | from memstate_lib import constants 31 | 32 | 33 | class Logfile(Base): 34 | """ 35 | Captures output in desired log file and sets up logrotate config 36 | in order to cap disk space consumed by the log file. 37 | """ 38 | 39 | init_done = False 40 | 41 | def __init__(self, interval): 42 | if not Logfile.init_done: 43 | end = constants.LOGFILE.rfind("/") 44 | parent_dir = constants.LOGFILE[0:end] 45 | self.setup_logrotate() 46 | if not os.path.exists(parent_dir): 47 | os.makedirs(parent_dir) 48 | 49 | if not self.__disk_space_available(parent_dir): 50 | msg = ( 51 | "Exiting! Unable to verify that there is enough disk space" 52 | f"available in {parent_dir}; check the man page for more " 53 | "details.") 54 | self.log_debug(msg) 55 | print(msg) 56 | sys.exit(1) 57 | 58 | # pylint: disable-next=consider-using-with 59 | self.log = open(constants.LOGFILE, "a", encoding="utf8") 60 | 61 | msg = ( 62 | f"Capturing memstate data every {interval} seconds, in log: " 63 | f"{constants.LOGFILE}") 64 | print(msg + "; press Ctrl-C to stop.") 65 | self.log_debug(msg + ".") 66 | Logfile.init_done = True 67 | 68 | @staticmethod 69 | def __disk_space_available(path): 70 | disk = shutil.disk_usage(path) 71 | free_mb = math.ceil(disk.free / 2**20) 72 | util_percent = math.ceil(disk.used / disk.total * 100) 73 | 74 | return ((util_percent < constants.MAX_DISKSPACE_UTIL) 75 | and (free_mb >= constants.MIN_DISKSPACE_NEEDED)) 76 | 77 | @staticmethod 78 | def setup_logrotate(): 79 | """Setup logrotate(8) configuration.""" 80 | with open(constants.LOGROTATEFILE, "w", encoding="utf8") as conf_fd: 81 | conf_fd.write( 82 | f"{constants.LOGFILE} {{\n" 83 | "\trotate 20\n" 84 | "\tsize 20M\n" 85 | "\tcopytruncate\n" 86 | "\tcompress\n" 87 | "\tmissingok\n" 88 | "}\n") 89 | 90 | def write(self, message): 91 | """Write message to log file.""" 92 | self.log.write(message) 93 | 94 | @staticmethod 95 | def flush(): 96 | """Flush log file.""" 97 | 98 | @staticmethod 99 | def rotate_file(): 100 | """Rotate log file.""" 101 | subprocess.run( 102 | ("logrotate", constants.LOGROTATEFILE), 103 | check=False, shell=False) # nosec 104 | 105 | def __del__(self): 106 | if Logfile.init_done: 107 | self.log.close() 108 | if os.path.exists(constants.LOGROTATEFILE): 109 | os.remove(constants.LOGROTATEFILE) 110 | -------------------------------------------------------------------------------- /scripts/net/rds_rdma_lat_example.txt: -------------------------------------------------------------------------------- 1 | Purpose: rds_rdma_lat.d used to trace and display the rdma read/write in bytes 2 | and the corresponding latency (in usec) per connection 3 | 4 | Arguments: none 5 | 6 | Output format: [src ip, dest ip, tos, read/write, bytes, latency (usecs)] 7 | 8 | Sample output: Shown below 9 | 10 | # ./rds_rdma_lat.d 11 | 2022 Aug 4 06:57:31 [ op] Bytes Latency(in usecs) 12 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=106 13 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=47 14 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=49 15 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=71 16 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=41 17 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=41 18 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=52 19 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=56 20 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=50 21 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=41 22 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=42 23 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=41 24 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=55 25 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=40 26 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=40 27 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=40 28 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=39 29 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=41 30 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=40 31 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=40 32 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=56 33 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=42 34 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=61 35 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=47 36 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=41 37 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=47 38 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=56 39 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=39 40 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=54 41 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=42 42 | [<192.168.0.6,192.168.0.7,0> r] bytes=1024 lat(usec)=10 43 | [<192.168.0.6,192.168.0.7,0> w] bytes=1024 lat(usec)=10 44 | 45 | 46 | Test with -D 100k size 47 | ---------------------- 48 | # ./rds_rdma_lat.d 49 | 2022 Aug 4 07:03:33 [ op] Bytes Latency(in usecs) 50 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=126 51 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=93 52 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=81 53 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=70 54 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=79 55 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=80 56 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=76 57 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=85 58 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=84 59 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=74 60 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=108 61 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=82 62 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=76 63 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=83 64 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=76 65 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=84 66 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=88 67 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=85 68 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=135 69 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=106 70 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=145 71 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=104 72 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=86 73 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=80 74 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=77 75 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=74 76 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=88 77 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=75 78 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=85 79 | [<192.168.0.6,192.168.0.7,0> w] bytes=102400 lat(usec)=86 80 | [<192.168.0.6,192.168.0.7,0> r] bytes=102400 lat(usec)=84 81 | [<192.168.0.6,192.168.0,0> w] bytes=102400 lat(usec)=75 82 | -------------------------------------------------------------------------------- /scripts/io/nvme_io_comp.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -qs 2 | 3 | /* 4 | * Copyright (c) 2024, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | * 24 | * Author(s): Rajan Shanmugavelu 25 | * Purpose: to measure nvme IO latency in microseconds (us). 26 | * The script requires 2 arguments: 27 | * - the reporting period (in secs) for latency 28 | * - the reporting period (in secs) for command completion 29 | * The DTrace 'fbt' and 'profile' modules need to be loaded 30 | * ('modprobe -a fbt profile') for UEK5. 31 | * Sample output: Refer to the file nvme_io_comp_example.txt 32 | */ 33 | 34 | #pragma D option dynvarsize=100m 35 | #pragma D option strsize=25 36 | 37 | string ncmd[uchar_t]; 38 | string stat[ushort_t]; 39 | 40 | dtrace:::BEGIN 41 | { 42 | printf("%d is latency interval, %d is cmd comp interval\n", $1, $2); 43 | 44 | ncmd[0x00] = "FLUSH"; 45 | ncmd[0x01] = "WRITE"; 46 | ncmd[0x02] = "READ"; 47 | ncmd[0x04] = "WRITE_UNCOR"; 48 | ncmd[0x05] = "COMPARE"; 49 | ncmd[0x08] = "WRITE_ZEROS"; 50 | ncmd[0x09] = "DSM"; 51 | ncmd[0x0c] = "VERIFY"; 52 | ncmd[0x0d] = "RESV_REG"; 53 | ncmd[0x0e] = "RESV_RPT"; 54 | ncmd[0x11] = "RESV_ACQ"; 55 | ncmd[0x15] = "RESV_REL"; 56 | 57 | stat[0x00] = "Success"; 58 | stat[0x01] = "Invalid Opcode"; 59 | stat[0x02] = "Invalid Field"; 60 | stat[0x03] = "DID Conflict"; 61 | stat[0x04] = "Data Xfer Error"; 62 | stat[0x06] = "Internal Error"; 63 | stat[0x07] = "Abort Request"; 64 | stat[0x08] = "Abort Queue"; 65 | stat[0x83] = "Resv Conflict"; 66 | stat[0x280] = "Write Fault"; 67 | stat[0x281] = "Read Error"; 68 | stat[0x287] = "Unwritten Block"; 69 | 70 | printf("Tracing... Hit Ctrl-C to end.\n"); 71 | } 72 | 73 | ::nvme_setup_cmd:entry 74 | { 75 | this->req = (struct request *) arg1; 76 | (this->req->rq_disk != NULL) ? cmnd[this->req] = (struct nvme_command *)arg2 : 0; 77 | (this->req->rq_disk != NULL) ? nvme_req_starttime[this->req] = timestamp : 0; 78 | } 79 | 80 | ::nvme_complete_rq:entry 81 | / this->start = (int64_t) (nvme_req_starttime[(struct request *) arg0]) / 82 | { 83 | this->comp_req = (struct request *) arg0; 84 | this->nvme_cmd = (struct nvme_command *) cmnd[this->comp_req]; 85 | this->comp_diskname = stringof(this->comp_req->rq_disk->disk_name); 86 | this->opcode = (uint8_t) this->nvme_cmd->rw.opcode & 0xff; 87 | this->nvme_req = (struct nvme_request *) (this->comp_req + 1); 88 | this->nvme_req_stat = (uint16_t) (this->nvme_req->status & 0x7ff); 89 | @opcount[this->comp_diskname, ncmd[this->opcode], 90 | stat[this->nvme_req_stat]] = count(); 91 | @lat_time[this->comp_diskname] = quantize((timestamp - this->start) / 1000); 92 | cmnd[this->comp_req] = NULL; 93 | nvme_req_starttime[this->comp_req] = 0; 94 | } 95 | 96 | tick-$1s, END 97 | { 98 | printf("\n\tSample Time : %-25Y\n", walltimestamp); 99 | printf("\t========================================================\n"); 100 | printa("\t\t%s %@5u\t(us)\n", @lat_time); 101 | printf("\t========================================================\n"); 102 | clear(@lat_time); 103 | } 104 | 105 | tick-$2s, END 106 | { 107 | printf("\n\tSample Time : %-25Y\n", walltimestamp); 108 | printf("\t=======================================================\n"); 109 | printf("\tDevice\t Command\t Status\t\tCount\n"); 110 | printf("\t=======================================================\n"); 111 | printa("\t%s %-12s %-20s %@d\n", @opcount); 112 | printf("\t=======================================================\n"); 113 | clear(@opcount); 114 | } 115 | -------------------------------------------------------------------------------- /tools/lkce/lkce.man: -------------------------------------------------------------------------------- 1 | .TH OLED LKCE 8 "Jul 2021" "Oracle Linux Enhanced Diagnostics" 2 | 3 | .SH NAME 4 | oled lkce \- Linux Kernel Core Extractor 5 | 6 | .SH SYNOPSIS 7 | \fB oled lkce\fR [report \fIreport-options\fR] [configure \fIconfig-options\fR] [enable_kexec|disable_kexec|status] [clean|list|help] 8 | 9 | .SH DESCRIPTION 10 | \fBlkce command\fR is used to interact with Linux Kernel Core Extractor[LKCE]. It is used to generate report from vmcore. lkce can run user scripts in kexec kernel. 11 | 12 | Once configured it can generate a report from a kexec kernel when system crashes. This is useful if the size of vmcore is too large to copy or send across for analysis or when generating vmcore is not preferred. 13 | 14 | .SH OPTIONS 15 | .TP 16 | \fBreport \fR 17 | .RS 3 18 | report-options: 19 | .RE 20 | .RS 3 21 | \fB\-\-vmcore=\fI/path/to/vmcore\fR - path to vmcore file for report generation (required) 22 | .RE 23 | .RS 3 24 | [\fB\-\-vmlinux=\fI/path/to/vmlinux\fR] - use this vmlinux instead of already configured value 25 | .RE 26 | .RS 3 27 | [\fB\-\-crash_cmds=\fIcmd1,cmd2,...\fR] - comma seperated crash cmds to be used in report generation instead of configured commands 28 | .RE 29 | .RS 3 30 | [\fB\-\-outfile=\fI/path/to/outfile\fR] - write output to outfile. By default output goes to stdout 31 | .RE 32 | 33 | .TP 34 | \fBconfigure --default\fR - Configure with default values, overwriting the existing values. 35 | .TP 36 | \fBconfigure [--show]\fR - Show the configured values. This is the default when no config-options are given. 37 | 38 | .TP 39 | \fBconfigure [config-options]\fR - other possible options 40 | .RS 3 41 | config-options: 42 | .RE 43 | .RS 3 44 | [\fB\-\-vmlinux_path=\fI/path/to/vmlinux\fR] - set vmliux_path to given value 45 | .RE 46 | .RS 3 47 | [\fB\-\-crash_cmds_file=\fI/path/to/file\fR] - set crash_cmds_file to given value 48 | .RE 49 | .RS 3 50 | [\fB\-\-vmcore=\fIyes/no\fR] - set vmcore to given value. Controls vmcore generation in kexec kernel. 51 | .RE 52 | .RS 3 53 | [\fB\-\-max_out_files=\fI\fR] - set max_out_files to given value 54 | .RE 55 | 56 | .TP 57 | \fBenable_kexec\fR - enable lkce to run user scripts in kexec kernel. 58 | 59 | .TP 60 | \fBdisable_kexec\fR - disable lkce to run user scripts in kexec kernel. 61 | 62 | .TP 63 | \fBstatus\fR - check the status of lkce configuration 64 | 65 | .TP 66 | \fBclean\fR - remove all but last 3 kexec report files. 67 | 68 | .TP 69 | \fBclean --all\fR - remove all the kexec report files. 70 | 71 | .TP 72 | \fBlist\fR - list the kexec report files. 73 | 74 | .SH HOW TO SET UP LKCE TO GENERATE KDUMP REPORT 75 | Make sure kexec-tools pkg is installed and you can generate a vmcore. Then, run these commands to enable lkce: 76 | .RS 0 77 | # yum install oled-tools 78 | .RE 79 | .RS 0 80 | # oled lkce enable_kexec 81 | .RE 82 | 83 | Crash the system manually. Once the machine boots, run 'oled lkce list'. Verify the listed file for output. Keep the system in configured state to generate the report when it crashes. 84 | 85 | .SH EXAMPLES 86 | .RS 0 87 | # oled lkce report --vmcore=/var/crash/127.0.0.1-2020-12-29-17\:17\:45/vmcore 88 | .RE 89 | .RS 0 90 | # oled lkce report --vmcore=/var/crash/127.0.0.1-2020-12-29-17\:17\:45/vmcore --crash_cmds=bt,sys 91 | .RE 92 | .RS 0 93 | # oled lkce report --vmcore=/var/crash/127.0.0.1-2020-12-29-17\:17\:45/vmcore --crash_cmds=bt,sys --outfile=/tmp/outfile.txt 94 | .RE 95 | 96 | .RS 0 97 | # oled lkce configure --default 98 | .RE 99 | .RS 0 100 | # oled lkce configure --show 101 | .RE 102 | .RS 0 103 | # oled lkce configure --vmlinux_path=/usr/lib/debug/lib/modules/4.14.35-1902.3.1.el7uek.x86_64/vmlinux 104 | .RE 105 | 106 | # oled lkce help 107 | .RE 108 | .RS 0 109 | # oled lkce enable_kexec 110 | .RE 111 | .RS 0 112 | # oled lkce disable_kexec 113 | .RE 114 | .RS 0 115 | # oled lkce status 116 | .RE 117 | 118 | .RS 0 119 | # oled lkce clean 120 | .RE 121 | .RS 0 122 | # oled lkce list 123 | .RE 124 | 125 | .SH MISC 126 | .TP 127 | kexec kernel is a secondary kernel used to collect the memory image[vmcore] of the first kernel when system crashes. kexec-tools pkg is used to configure the kexec kernel. 128 | 129 | .TP 130 | LKCE provides an environment inside kexec kernel to run user scripts. One such user script used by lkce to generate the vmcore report when the system crashes. LKCE links with kdump[of kexec-tools] service as kdump_pre hook to run scripts inside kexec kernel. 131 | 132 | .SH FILES 133 | .TP 134 | /etc/oled/lkce/lkce.conf 135 | -------------------------------------------------------------------------------- /scripts/net/arp_origin.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author(s): Rama Nichanamatlu, Praveen Kumar Kannoju. 27 | * Purpose: This script detects and prints the details of all ARP requests, 28 | * replies, and ignored packets on all the network interfaces present on the 29 | * system where this script is being executed. 30 | * Prerequisites: Refer to the file arp_origin_example.txt 31 | * Sample output: Refer to the file arp_origin_example.txt 32 | */ 33 | 34 | #define DT_VERSION_NUMBER_(M, m, u) \ 35 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 36 | 37 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 38 | #pragma D option lockmem=unlimited 39 | #endif 40 | 41 | #define arp_protocol 0x0806 42 | #define arp_request_opcode 1 43 | #define arp_reply_opcode 2 44 | 45 | fbt:vmlinux:__dev_queue_xmit:entry 46 | / 47 | ntohs(((struct sk_buff *)arg0)->protocol) == arp_protocol && 48 | (this->arphdr = ((struct arphdr *)(((struct sk_buff *)arg0)->head + ((struct sk_buff *)arg0)->network_header))) && 49 | ntohs(this->arphdr->ar_op) == arp_request_opcode 50 | / 51 | { 52 | this->dev = ((struct net_device *) ((struct sk_buff *)arg0)->dev); 53 | this->ip_var = (u64)this->arphdr + sizeof(struct arphr) + this->arphdr->ar_hln + sizeof(this->arphdr->ar_op); 54 | this->sip = (ipaddr_t *)this->ip_var; 55 | this->tip = (ipaddr_t *)(this->ip_var + this->arphdr->ar_pln + this->arphdr->ar_hln); 56 | 57 | printf("%Y %s Send arp request: sip: %s, tip: %s\n", 58 | walltimestamp, this->dev->name, inet_ntoa(this->sip), 59 | inet_ntoa(this->tip)); 60 | } 61 | 62 | fbt:vmlinux:arp_rcv:entry 63 | / 64 | (this->arphdr = ((struct arphdr *)(((struct sk_buff *)arg0)->head + ((struct sk_buff *)arg0)->network_header))) && 65 | ntohs(this->arphdr->ar_op) == arp_reply_opcode 66 | / 67 | { 68 | this->dev = ((struct net_device *) ((struct sk_buff *)arg0)->dev); 69 | this->ip_var = (u64)this->arphdr + sizeof(struct arphr) + this->arphdr->ar_hln + sizeof(this->arphdr->ar_op); 70 | this->sip = (ipaddr_t *)this->ip_var; 71 | this->tip = (ipaddr_t *)(this->ip_var + this->arphdr->ar_pln + this->arphdr->ar_hln); 72 | 73 | printf("%Y %s Recv arp reply sip: %s tip: %s\n", 74 | walltimestamp, this->dev->name, inet_ntoa(this->sip), 75 | inet_ntoa(this->tip)); 76 | } 77 | 78 | fbt:vmlinux:arp_send:entry, 79 | fbt:vmlinux:arp_create:entry 80 | / arg0 == arp_request_opcode || arg0 == arp_reply_opcode / 81 | { 82 | this->tip = (__be32 *)alloca(4); 83 | *(this->tip) = arg2; 84 | this->sip = (__be32 *)alloca(4); 85 | *(this->sip) = arg4; 86 | 87 | printf("%Y %s Send arp %s sip: %s, tip: %s\n", 88 | walltimestamp, ((struct net_device *)arg3)->name, 89 | arg0 == 1 ? "request":arg0 == 2 ? "reply":"unknown", 90 | inet_ntoa((ipaddr_t *)this->sip), 91 | inet_ntoa((ipaddr_t *)this->tip)); 92 | 93 | } 94 | 95 | fbt:vmlinux:arp_ignore:entry 96 | { 97 | self->dev = (struct net_device*)((struct in_device *)arg0)->dev; 98 | self->sip = arg1; 99 | self->tip = arg2; 100 | } 101 | 102 | fbt:vmlinux:arp_ignore:return 103 | / self->dev != NULL / 104 | { 105 | this->tip = (__be32 *)alloca(4); 106 | *(this->tip) = self->tip; 107 | this->sip = (__be32 *)alloca(4); 108 | *(this->sip) = self->sip; 109 | 110 | printf("%Y %s Recv arp request %s sip: %s, tip: %s\n", 111 | walltimestamp, self->dev->name, 112 | arg1 == 1 ? "(Ignoring) " : "", 113 | inet_ntoa((ipaddr_t *)this->sip), 114 | inet_ntoa((ipaddr_t *)this->tip)); 115 | 116 | self->sip = 0; 117 | self->tip = 0; 118 | self->dev = 0; 119 | } 120 | -------------------------------------------------------------------------------- /scripts/io/scsi_latency.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -qs 2 | 3 | /* 4 | * Copyright (c) 2024, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | * 24 | * 25 | * Author(s): Rajan Shanmugavelu 26 | * Purpose: to measure SCSI Mid Layer IO latency in milliseconds (ms) and 27 | * to measure QLogic adapter layer IO latency in milliseconds(ms). 28 | * The script requires 1 arguments: 29 | * - the reporting period (in secs) for latency 30 | * The DTrace 'fbt' and 'profile' modules need to be loaded 31 | * ('modprobe -a fbt profile') for UEK5. 32 | * Sample output: Refer to the file scsi_latency_example.txt 33 | */ 34 | 35 | #pragma D option dynvarsize=100m 36 | #pragma D option strsize=25 37 | 38 | string opcode[ushort_t]; 39 | 40 | BEGIN 41 | { 42 | printf("Running %s with %d seconds interval\n", $0, $1); 43 | opcode[0x00] = "TUR"; 44 | opcode[0x08] = "Read(6)"; 45 | opcode[0x0a] = "Write(6)"; 46 | opcode[0x12] = "Inquiry"; 47 | opcode[0x25] = "Read Cap(10)"; 48 | opcode[0x28] = "Read(10)"; 49 | opcode[0x2a] = "Write(10)"; 50 | opcode[0x5e] = "Pers Resv In"; 51 | opcode[0xa0] = "Report Luns"; 52 | opcode[0xa3] = "Maint In"; 53 | 54 | printf("Hit Ctrl-C to quit...\n"); 55 | } 56 | 57 | ::scsi_dispatch_cmd_start 58 | { 59 | this->cmnd = (struct scsi_cmnd *) arg0; 60 | scsistarttime[this->cmnd] = timestamp; 61 | } 62 | 63 | fbt::scsi_done:entry, 64 | fbt::scsi_mq_done:entry 65 | / scsistarttime[(this->scsi_cmnd = (struct scsi_cmnd *) arg0)] / 66 | { 67 | this->opcode = this->scsi_cmnd->cmnd[0]; 68 | this->status = stringof((this->scsi_cmnd->result == 0) ? "Success" : "Failed"); 69 | this->start = (int64_t) scsistarttime[this->scsi_cmnd]; 70 | this->elasped = ((timestamp - this->start) / 1000000); 71 | @cmd_count[opcode[this->opcode], this->opcode, this->status] = count(); 72 | @lat[opcode[this->opcode], this->status] = quantize(this->elasped); 73 | scsistarttime[this->scsi_cmnd] = 0; 74 | } 75 | 76 | fbt::qla2xxx_queuecommand:entry 77 | { 78 | this->qla_cmd = (struct scsi_cmnd *)arg1; 79 | qla_starttime[this->qla_cmd] = timestamp; 80 | } 81 | 82 | fbt::qla2x00_sp_compl:entry 83 | / qla_starttime[(this->qla_comp_cmd = (struct scsi_cmnd *) ((srb_t *)arg0)->u.scmd.cmd)] / 84 | { 85 | this->qla_opcode = this->qla_comp_cmd->cmnd[0]; 86 | this->status = stringof((this->qla_comp_cmd->result == 0) ? "Success" : "Failed"); 87 | this->qla_start = (int64_t) qla_starttime[this->qla_comp_cmd]; 88 | this->qla_elapsed = (timestamp - this->qla_start) / 1000000; 89 | @qla_lat[opcode[this->qla_opcode], this->status] = quantize(this->qla_elapsed); 90 | qla_starttime[this->qla_comp_cmd] = 0; 91 | } 92 | 93 | 94 | END, 95 | tick-$1s 96 | { 97 | printf("\n Sample Time : %-25Y SCSI Mid layer latency\n", walltimestamp); 98 | printf("========================================================\n"); 99 | printa(" %-36s %-5s %@d \t(ms)\n ", @lat); 100 | printf("========================================================\n"); 101 | printf("\n Sample Time : %-25Y\n", walltimestamp); 102 | printf(" =============================================================\n"); 103 | printf(" Command OpCode Status Count\n"); 104 | printf(" =============================================================\n"); 105 | printa(" %-10s 0x%-2x %-10s %@d\n", @cmd_count); 106 | printf(" =============================================================\n"); 107 | printf("\n Sample Time : %-25Y Latency at Qlogic Adapter layer\n", walltimestamp); 108 | printf("========================================================\n"); 109 | printa(" %-36s %-5d %@d \t(ms)\n ", @qla_lat); 110 | printf("========================================================\n"); 111 | clear(@lat); 112 | clear(@cmd_count); 113 | clear(@qla_lat); 114 | } 115 | -------------------------------------------------------------------------------- /scripts/io/nvme_io_comp_example.txt: -------------------------------------------------------------------------------- 1 | Below are sample output, the script gathers nvme IO statistics given a sampling 2 | interval and displays the summary ouptut in histogram format. The script takes 3 | 2 arguments, the first argument is sampling interval in seconds for latency, 4 | the second argument is interval time in seconds for summary of command completion counts. 5 | 6 | The left column of the histogram 'value' output is the latency time in microseconds 7 | and the right 'count' are the number of IO commands completed. 8 | 9 | # ./nvme-io-comp2.d 10 | dtrace: failed to compile script ./nvme-io-comp2.d: line 15: macro argument $2 is not defined 11 | 12 | # ./nvme-io-comp2.d 2 2 13 | Tracing... Hit Ctrl-C to end. 14 | 15 | Sample Time : 2022 Oct 26 12:03:32 16 | ======================================================== 17 | nvme0n1 18 | value ------------- Distribution ------------- count 19 | 4 | 0 20 | 8 |@@@ 1868 21 | 16 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 21266 22 | 32 |@@@@ 2777 23 | 64 |@ 829 24 | 128 |@ 615 25 | 256 | 215 26 | 512 | 17 27 | 1024 | 21 28 | 2048 | 12 29 | 4096 | 0 30 | (us) 31 | ======================================================== 32 | 33 | Sample Time : 2022 Oct 26 12:03:32 34 | ======================================================= 35 | Device Command Status Count 36 | ======================================================= 37 | nvme0n1 FLUSH Success 143 38 | nvme0n1 Success 360 <- NvME admin cmds 39 | nvme0n1 WRITE Success 13548 40 | nvme0n1 READ Success 13569 41 | ======================================================= 42 | 43 | Upon fault Injection. 44 | 45 | Sample Time : 2022 Oct 26 12:04:16 46 | ======================================================== 47 | nvme0n1 48 | value ------------- Distribution ------------- count 49 | 4 | 0 50 | 8 |@@@ 1999 51 | 16 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 19991 52 | 32 |@@@@ 2830 53 | 64 |@ 701 54 | 128 |@ 525 55 | 256 | 193 56 | 512 | 18 57 | 1024 | 30 58 | 2048 | 9 59 | 4096 | 3 60 | 8192 | 0 61 | 16384 | 0 62 | 32768 | 2 63 | 65536 | 0 64 | (us) 65 | ======================================================== 66 | 67 | Sample Time : 2022 Oct 26 12:04:16 68 | ======================================================= 69 | Device Command Status Count 70 | ======================================================= 71 | nvme0n1 DSM Success 0 72 | nvme0n1 RESV_ACQ Success 0 73 | nvme0n1 RESV_REG Success 0 74 | nvme0n1 RESV_REL Success 0 75 | nvme0n1 RESV_RPT Success 0 76 | nvme0n1 VERIFY Success 0 77 | nvme0n1 WRITE_UNCOR Success 0 78 | nvme0n1 WRITE_ZEROS Success 0 79 | nvme0n1 WRITE Invalid Opcode 1 80 | nvme0n1 READ Invalid Opcode 2 81 | nvme0n1 COMPARE Success 5 82 | nvme0n1 FLUSH Success 105 83 | nvme0n1 Success 357 <- NvME admin cmds 84 | nvme0n1 READ Success 12754 85 | nvme0n1 WRITE Success 13077 86 | ======================================================= 87 | 88 | 89 | -------------------------------------------------------------------------------- /tools/memstate/memstate.man: -------------------------------------------------------------------------------- 1 | .TH OLED MEMSTATE 8 "Jul 2021" "Oracle Linux Enhanced Diagnostics" 2 | 3 | .SH NAME 4 | oled memstate - Captures and analyzes various memory usage statistics on the 5 | running system. 6 | 7 | .SH SYNOPSIS 8 | oled memstate [-h] [-p [PID]] [-s [FILE]] [-n [FILE]] [-a] [-v] \ 9 | [-f [INTERVAL]] 10 | 11 | .SH DESCRIPTION 12 | The memstate tool gathers data about memory usage on the system where it's run, 13 | and analyzes the raw data to compute various statistics, including total memory 14 | used by kernel/userspace allocations, a list of top 10 processes consuming most 15 | memory (RSS), the 10 biggest slab caches, the top 10 swap space consumers, etc. It 16 | can also display per-process memory usage (PSS), including how the allocations are 17 | split across the NUMA nodes (if applicable). The tool checks, and prints warnings 18 | for, any abnormalities detected; for instance - excessive slab growth or large 19 | page tables, NUMA imbalance, or invalid vm.min_free_kbytes config. The tool also 20 | supports an offline mode to analyze slab and NUMA stats - see OPTIONS for more 21 | details. 22 | 23 | Note: RSS is not an accurate metric to measure per-process memory usage. For 24 | more accurate values, use the -p option, which extracts per-process PSS values. 25 | 26 | .SH OPTIONS 27 | .TP 28 | -h, --help 29 | Displays a help message with all the valid options that 30 | can be passed to the tool. 31 | 32 | .TP 33 | -p [PID], --pss [PID] 34 | Displays per-process memory usage, for the top 10 process 35 | memory consumers. This gathers data from /proc//smaps_rollup files and 36 | sorts them, to display the most memory intensive processes on the system. It 37 | displays the following metrics per process: PSS, RSS, private, swap-pss. 38 | Use with -v/--verbose to get memory usage data for all processes on the system. 39 | Note that this processing can be quite expensive w.r.t. time taken/CPU cycles, 40 | esp. if run on a heavily-loaded system. Therefore, it is not recommended to 41 | invoke this option (with or without -v/--verbose - that does not matter) too 42 | often. If a process's pid is passed as input, this option will display the 43 | memory mappings and usage summary for that process only, extracted from 44 | /proc//smaps file. 45 | 46 | .TP 47 | -s [FILE], --slab [FILE] 48 | Analyzes/displays slab usage. It takes an optional file 49 | input, which should contain the output of 'cat /proc/slabinfo'. This can be 50 | used to analyze the slab usage on another system offline. To display the 51 | slab usage on the current/running system, simply omit the file input. This 52 | option can also be combined with -v/--verbose, which will display all slab 53 | caches, without which it will display 10 biggest slab caches. This data is 54 | gathered from /proc/slabinfo. 55 | 56 | .TP 57 | -n [FILE], --numa [FILE] 58 | Analyzes/displays NUMA stats. It extracts per-NUMA node 59 | meminfo data from /sys/devices/system/node/nodeX/meminfo. When combined with 60 | -v/--verbose, it also walks /proc//numa_maps for all processes on the 61 | system and computes their total memory allocations on each NUMA node, and 62 | displays them in a sorted fashion. Note that this processing can be quite 63 | expensive w.r.t. time taken/CPU cycles, esp. if run on a heavily-loaded system. 64 | Therefore, it is not recommended to invoke -n/--numa with -v/--verbose too 65 | often. 66 | 67 | It also takes an optional file input (this can be used for offline processing), 68 | which should contain the contents of /proc/*/numa_maps files as well as 69 | /proc/*/comm files. One way to generate this input file is using this shell 70 | script: 71 | 72 | PROCS=/proc/*; for p in $PROCS; do COMM=$(cat $p/comm); \ 73 | MAPS=$(cat $p/numa_maps); echo "$COMM(${p##*/}):"; echo "$MAPS"; done 74 | 75 | and redirecting it to a file. This file can be passed as input, with 76 | -n, to analyze and display per-NUMA node memory allocation statistics for each 77 | process. 78 | 79 | .TP 80 | -a, --all 81 | Displays all data. This option displays a basic memory usage summary, 82 | plus the biggest slab caches, the top swap space consumers along with the NUMA 83 | stats summary. If combined with -v/--verbose, it also checks/displays per-process 84 | memory use, but this can be an expensive operation. See -p/--pss for more 85 | details. 86 | 87 | .TP 88 | -v, --verbose 89 | Verbose data capture; this should be combined with other options. 90 | 91 | .TP 92 | -f [INTERVAL], --frequency [INTERVAL] 93 | Interval at which data should be collected periodically. 94 | The default is 30 seconds; the user can pass in a different interval (in 95 | seconds) as an optional parameter for this option. Note that the lowest valid 96 | interval value is 5 seconds. 97 | 98 | This option can be combined with most of the options above to capture desired 99 | data periodically. The data will be written to the file 100 | /var/oled/memstate/memstate.log which is rotated and compressed, by logrotate. 101 | Each file can grow to a max size of 20 MB, and at most 20 files are retained. 102 | 103 | After compresssion, the size of /var/oled/memstate/ directory on disk will be 104 | around 20 - 40 MB. The tool tries not to fill up the partition, so it checks 105 | before running if there is at least 50 MB available in that file system and if 106 | the disk utilization is <= 85%. If not, it will exit with an error message. 107 | 108 | .SH REPORTING BUGS 109 | .TP 110 | Please contact Oracle Linux Support to report any bugs for OLED tools. 111 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/hugepages.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Helper module to retrieve hugepages info.""" 23 | 24 | from __future__ import print_function 25 | import glob 26 | import os 27 | from memstate_lib import Base 28 | 29 | 30 | class Hugepages(Base): 31 | """ 32 | Extracts the number of hugepages of all sizes (reserved and free) on 33 | all NUMA nodes. 34 | """ 35 | def __init__(self): 36 | self.hp_nr_dict = {} 37 | self.hp_free_dict = {} 38 | 39 | def __read_nr_hugepages(self): 40 | for path in glob.glob("/sys/devices/system/node/node*"): 41 | # Extract node0 from /sys/devices/system/node/node0, then extract 42 | # the number following "node" 43 | current_node = path.split("/")[-1][4:] 44 | hp_dir = os.path.join(path, "hugepages/hugepages*") 45 | for hp_path in glob.glob(hp_dir): 46 | # We're attempting to extract hugepage size from hp_path (for 47 | # 2MB/1GB pages): 48 | # /sys/devices/system/node/node*/hugepages/hugepages-2048kB 49 | # /sys/devices/system/node/node*/hugepages/hugepages-1048576kB 50 | hp_size = hp_path.split("/")[-1][10:] 51 | hp_size_kb = hp_size[:-2] 52 | 53 | # Read nr_hugepages and free_hugepages for each NUMA node 54 | nr_path = os.path.join(hp_path, "nr_hugepages") 55 | free_path = os.path.join(hp_path, "free_hugepages") 56 | nr_val = int(self.read_text_file(nr_path)) 57 | free_val = int(self.read_text_file(free_path)) 58 | key = hp_size_kb + "_" + current_node 59 | self.hp_nr_dict[key] = nr_val 60 | self.hp_free_dict[key] = free_val 61 | 62 | def get_total_hugepages_gb(self): 63 | """Get total amount of memory in hugepages in GBs.""" 64 | self.__read_nr_hugepages() 65 | hp_nr_total_kb = 0 66 | for key, val in self.hp_nr_dict.items(): 67 | (hp_size, _) = key.split("_") 68 | hp_nr_total_kb = \ 69 | hp_nr_total_kb + round(float(val * int(hp_size)), 1) 70 | return round(self.convert_kb_to_gb(hp_nr_total_kb), 1) 71 | 72 | @staticmethod 73 | def __create_matrix(hp_dict): 74 | hp_size_arr = [] 75 | 76 | # Determine number of hugepage sizes 77 | for key in hp_dict: 78 | (size, _) = key.split("_") 79 | hp_size_kb = int(size) 80 | if hp_size_kb not in hp_size_arr: 81 | hp_size_arr.append(hp_size_kb) 82 | 83 | # Create a dictionary of lists, where each list in the dictionary is 84 | # accessed using the hugepage_size as key, and the list has 85 | # nr_hugepages (of that size) for each NUMA node. 86 | # For instance, on a 2-node system with both 2 MB and 1 GB hugepages, 87 | # hp_matrix will be something similar to: 88 | # {'2048': [22212, 22272], '1048576': [99, 100]} 89 | 90 | hp_matrix = {} 91 | for key in hp_dict: 92 | (size, _) = key.split("_") 93 | val = round(hp_dict[key] * int(size)) 94 | hp_matrix.setdefault(size, []).append(int(val)) 95 | 96 | return hp_matrix 97 | 98 | def get_nr_hugepages_matrix_kb(self): 99 | """Get total KBs of hugepages per hugepage size per NUMA node. 100 | 101 | Return a dictionary with following format: 102 | { 103 | : [, , ..., : [, , ..., are the different hugepage sizes (e.g. '2048' for 2MB 109 | hupepages and '1048576' for 1GB pages) and is the amount of 110 | memory, in KBs, of hugepages of the given size in NUMA node N. 111 | """ 112 | self.__read_nr_hugepages() 113 | hp_dict = self.hp_nr_dict 114 | return self.__create_matrix(hp_dict) 115 | 116 | def get_free_hugepages_matrix_kb(self): 117 | """Get total free KBs of hugepages per hupepage size per NUMA node. 118 | 119 | See get_nr_hugepages_matrix_kb() for details of the information 120 | returned. 121 | """ 122 | 123 | if len(self.hp_free_dict) == 0: 124 | self.__read_nr_hugepages() 125 | hp_dict = self.hp_free_dict 126 | return self.__create_matrix(hp_dict) 127 | -------------------------------------------------------------------------------- /scripts/net/ping_lat.d: -------------------------------------------------------------------------------- 1 | #!/usr/sbin/dtrace -Cqs 2 | 3 | /* 4 | * Copyright (c) 2022, Oracle and/or its affiliates. 5 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 | * 7 | * This code is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU General Public License version 2 only, as 9 | * published by the Free Software Foundation. 10 | * 11 | * This code is distributed in the hope that it will be useful, but WITHOUT 12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 | * version 2 for more details (a copy is included in the LICENSE file that 15 | * accompanied this code). 16 | * 17 | * You should have received a copy of the GNU General Public License version 18 | * 2 along with this work; if not, see . 19 | * 20 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 | * or visit www.oracle.com if you need additional information or have any 22 | * questions. 23 | */ 24 | 25 | /* 26 | * Author: Praveen Kumar Kannoju 27 | * Purpose: This script provides latency details of the icmp packet, during 28 | * its journey in the kernel. Output shows the time delta between two 29 | * consequtive kernel functions through which the packet travels. 30 | * Prerequisites: Refer to the file ping_lat_example.txt 31 | * Sample output: Refer to the file ping_lat_example.txt 32 | */ 33 | 34 | #define DT_VERSION_NUMBER_(M, m, u) \ 35 | ((((M) & 0xFF) << 24) | (((m) & 0xFFF) << 12) | ((u) & 0xFFF)) 36 | 37 | #if __SUNW_D_VERSION >= DT_VERSION_NUMBER_(2,0,0) 38 | #pragma D option lockmem=unlimited 39 | #endif 40 | 41 | #define ICMP_PROTOCOL 1 42 | #define ICMP_ECHO 8 43 | #define ICMP_ECHO_REPLY 0 44 | 45 | fbt:vmlinux:ip_send_skb:entry, 46 | fbt:ib_ipoib:ipoib_cm_send:entry 47 | / 48 | (this->nh = ((struct sk_buff *)arg1)->head + ((struct sk_buff *)arg1)->network_header) && 49 | (this->iphdr = (struct iphdr *)this->nh) && 50 | (this->iphdr->protocol == ICMP_PROTOCOL) && 51 | (this->th = ((struct sk_buff *)arg1)->head + ((struct sk_buff *)arg1)->transport_header) && 52 | (this->icmphdr = (struct icmphdr *)this->th) && 53 | (this->icmphdr->type == ICMP_ECHO) 54 | / 55 | { 56 | this->sip = inet_ntoa((ipaddr_t *)(&this->iphdr->saddr)); 57 | this->dip = inet_ntoa((ipaddr_t *)(&this->iphdr->daddr)); 58 | this->icmp_seq = this->icmphdr->un.echo.sequence; 59 | this->icmp_id = ntohs(this->icmphdr->un.echo.id); 60 | 61 | timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, probefunc] = timestamp; 62 | } 63 | 64 | /* 65 | * Same as above, but for functions with sk_buff in arg0 rather than arg1. 66 | */ 67 | fbt:vmlinux:dev_hard_start_xmit:entry, 68 | fbt:ib_ipoib:ipoib_start_xmit:entry 69 | / 70 | (this->nh = ((struct sk_buff *)arg0)->head + ((struct sk_buff *)arg0)->network_header) && 71 | (this->iphdr = (struct iphdr *)this->nh) && 72 | (this->iphdr->protocol == ICMP_PROTOCOL) && 73 | (this->th = ((struct sk_buff *)arg0)->head + ((struct sk_buff *)arg0)->transport_header) && 74 | (this->icmphdr = (struct icmphdr *)this->th) && 75 | (this->icmphdr->type == ICMP_ECHO) 76 | / 77 | { 78 | this->sip = inet_ntoa((ipaddr_t *)(&this->iphdr->saddr)); 79 | this->dip = inet_ntoa((ipaddr_t *)(&this->iphdr->daddr)); 80 | this->icmp_seq = this->icmphdr->un.echo.sequence; 81 | this->icmp_id = ntohs(this->icmphdr->un.echo.id); 82 | 83 | timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, probefunc] = timestamp; 84 | } 85 | 86 | fbt:vmlinux:icmp_rcv:entry 87 | / 88 | (this->nh = ((struct sk_buff *)arg0)->head + ((struct sk_buff *)arg0)->network_header) && 89 | (this->iphdr = (struct iphdr *)this->nh) && 90 | (this->iphdr->protocol == ICMP_PROTOCOL) && 91 | (this->th = ((struct sk_buff *)arg0)->head + ((struct sk_buff *)arg0)->transport_header) && 92 | (this->icmphdr = (struct icmphdr *)this->th) && 93 | (this->icmphdr->type == ICMP_ECHO_REPLY) 94 | / 95 | { 96 | this->sip = inet_ntoa((ipaddr_t *)(&this->iphdr->daddr)); 97 | this->dip = inet_ntoa((ipaddr_t *)(&this->iphdr->saddr)); 98 | this->icmp_seq = this->icmphdr->un.echo.sequence; 99 | this->icmp_id = ntohs(this->icmphdr->un.echo.id); 100 | 101 | this->ip_send_skb_time = timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "ip_send_skb"]; 102 | this->dev_hard_start_xmit_time = timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "dev_hard_start_xmit"]; 103 | this->ipoib_start_xmit_time = timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "ipoib_start_xmit"]; 104 | this->ipoib_cm_send_time = timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "ipoib_cm_send"]; 105 | this->icmp_rcv_time = timestamp; 106 | 107 | printf("icmp id: %d, icmp sequence: %d, src ip: %s, dst ip: %s\n", this->icmp_id, this->icmp_seq, this->sip, this->dip); 108 | printf("\nroutine delta(ns)\n\n"); 109 | printf("ip_send_skb\n"); 110 | printf("dev_hard_start_xmit %15d\n", this->dev_hard_start_xmit_time - this->ip_send_skb_time); 111 | printf("ipoib_start_xmit %15d\n", this->ipoib_start_xmit_time > 0 ? this->ipoib_start_xmit_time - this->dev_hard_start_xmit_time : 0); 112 | printf("ipoib_cm_send %15d\n", this->ipoib_cm_send_time > 0 ? this->ipoib_cm_send_time - this->ipoib_start_xmit_time : 0); 113 | printf("icmp_rcv %15d\n\n", this->icmp_rcv_time - this->ipoib_cm_send_time); 114 | printf("request-reply latency: %15d\n", this->icmp_rcv_time - this->ip_send_skb_time); 115 | printf("------------------------------------------------------------------\n"); 116 | 117 | timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "ip_send_skb"] = 0; 118 | timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "dev_hard_start_xmit"] = 0; 119 | timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "ipoib_start_xmit"] = 0; 120 | timestamp_info[this->sip, this->dip, this->icmp_seq, this->icmp_id, "ipoib_cm_send"] = 0; 121 | 122 | } 123 | 124 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/slabinfo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Helper module to analyze slab info.""" 23 | 24 | from __future__ import print_function 25 | from collections import OrderedDict 26 | from memstate_lib import Base 27 | from memstate_lib import Meminfo 28 | from memstate_lib import constants 29 | from memstate_lib import Hugepages 30 | 31 | 32 | class Slabinfo(Base): 33 | """ Analyzes output from /proc/slabinfo """ 34 | 35 | def __init__(self, infile=None): 36 | self.input_file = False 37 | self.hugepages = Hugepages() 38 | self.data = "" 39 | if infile: 40 | self.input_file = True 41 | self.__read_slabinfo_file(infile) 42 | 43 | def __read_slabinfo_file(self, infile): 44 | data = self.open_file(str(infile), 'r') 45 | if data: 46 | self.data = data.read() 47 | data.close() 48 | else: 49 | self.print_error("Unable to read input file '" + str(infile) + "'") 50 | self.data = "" 51 | 52 | def __read_slabinfo(self): 53 | self.data = self.read_text_file("/proc/slabinfo") 54 | 55 | @staticmethod 56 | def __slabinfo_get_name(line): 57 | return line.split()[0].strip() 58 | 59 | @staticmethod 60 | def __slabinfo_get_num_slabs(line): 61 | return line.split()[14].strip() 62 | 63 | @staticmethod 64 | def __slabinfo_get_pages_per_slab(line): 65 | return line.split()[5].strip() 66 | 67 | def __get_ordered_slab_caches(self): 68 | """ 69 | Run "cat /proc/slabinfo" and compute memory used by each slab cache. 70 | Sort in descending order and return list. 71 | @return: List of all slab caches, sorted in descending order based on 72 | size. Note that the size of the slab caches in the list is in 73 | KB. 74 | """ 75 | if self.data == "": 76 | self.print_error("Slabinfo data unavailable; nothing to analyze.") 77 | return None 78 | slab_list = {} 79 | slab_caches_sorted = {} 80 | page_size_kb = Base.get_page_size() / constants.ONE_KB 81 | for line in self.data.splitlines(): 82 | if len(line.split()) != 16: 83 | continue 84 | c_name = self.__slabinfo_get_name(line) 85 | c_num_slabs = self.__slabinfo_get_num_slabs(line) 86 | c_pages_per_slab = self.__slabinfo_get_pages_per_slab(line) 87 | slab_list[c_name] = ( 88 | int(c_num_slabs) * int(c_pages_per_slab) * page_size_kb) 89 | 90 | slab_caches_sorted = OrderedDict( 91 | sorted(slab_list.items(), key=lambda x: x[1], reverse=True)) 92 | return slab_caches_sorted 93 | 94 | def __display_top_slab_caches(self, num, slabs_list=None): 95 | if slabs_list is None: 96 | self.print_error("Slab caches list unavailable!") 97 | return 98 | num_printed = 0 99 | for slab, size_kb in slabs_list.items(): 100 | if num != constants.NO_LIMIT and num_printed >= num: 101 | break 102 | if size_kb > 0: 103 | self.print_pretty_kb(slab, size_kb) 104 | num_printed += 1 105 | if num == constants.NO_LIMIT: 106 | print("") 107 | print( 108 | "Total memory used by all slab caches: " 109 | f"{self.__get_total_slab_size_gb(slabs_list)} GB") 110 | 111 | def __get_total_slab_size_gb(self, slabs_list=None): 112 | if slabs_list is None: 113 | self.print_error("Slab caches list unavailable!") 114 | return None 115 | slab_size_kb = sum(slabs_list.values()) 116 | 117 | return self.convert_kb_to_gb(slab_size_kb) 118 | 119 | def __check_slab_usage(self, num): 120 | """ 121 | Print a warning if total slab usage is >= SLAB_USE_PERCENT of 122 | (TotalRAM - HugePages). 123 | 124 | Also lists the biggest slab caches. 125 | """ 126 | if not self.input_file: 127 | self.__read_slabinfo() 128 | if self.data == "": 129 | self.print_error("Slabinfo data unavailable!") 130 | return 131 | slab_list = self.__get_ordered_slab_caches() 132 | if not self.input_file: 133 | meminfo = Meminfo() 134 | slab_total_gb = meminfo.get_total_slab_gb() 135 | if (slab_total_gb >= 136 | (constants.SLAB_USE_PERCENT * 137 | (meminfo.get_total_ram_gb() - 138 | self.hugepages.get_total_hugepages_gb()))): 139 | self.print_warn("Large slab caches found on this system!") 140 | self.__display_top_slab_caches(num, slab_list) 141 | 142 | def memstate_check_slab(self, num=constants.NUM_TOP_SLAB_CACHES): 143 | """Check state of slab.""" 144 | if num == constants.NO_LIMIT: 145 | hdr = "SLAB CACHES (in KB):" 146 | else: 147 | hdr = f"TOP {num} SLAB CACHES (in KB):" 148 | print(hdr) 149 | self.__check_slab_usage(num) 150 | print("") 151 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/buddyinfo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Helper module to check buddyinfo.""" 23 | 24 | import re 25 | 26 | from memstate_lib import Base 27 | from memstate_lib import constants 28 | 29 | 30 | class Buddyinfo(Base): 31 | """ Analyzes output from /proc/buddyinfo """ 32 | 33 | def __read_buddyinfo(self): 34 | return self.read_text_file("/proc/buddyinfo", on_error="") 35 | 36 | def __display_pagetypeinfo(self): 37 | data = self.read_text_file("/proc/pagetypeinfo", on_error="") 38 | if data == "": 39 | self.print_error("Unable to read file /proc/pagetypeinfo") 40 | return 41 | 42 | data = data.split("\n\n")[1].splitlines() 43 | pgti_header = data[0] 44 | pgti_body = data[1:] 45 | 46 | print("\nPagetypeinfo:") 47 | print(f" {pgti_header}") 48 | 49 | regex = re.compile("Normal.*(Unmovable|Movable|Reclaimable)") 50 | for line in pgti_body: 51 | if regex.search(line): 52 | print(f" {line}") 53 | 54 | def __display_zoneinfo(self): 55 | interesting = ['Node', 'free', 'min', 'low', 'high'] 56 | data = self.read_text_file("/proc/zoneinfo", on_error="") 57 | if data == "": 58 | self.print_error("Unable to read file /proc/zoneinfo") 59 | return 60 | print("\nZoneinfo:") 61 | for line in data.splitlines(): 62 | if any(word in line.split() for word in interesting): 63 | print(f" {line}") 64 | 65 | def check_fragmentation_status(self, num_numa_nodes=1): 66 | """ 67 | Parses the output of 'cat /proc/buddyinfo' for each 'Normal' zone to 68 | check for fragmentation status. If the amount of memory available in 69 | higher-order chunks is constants.FRAG_LEVEL_LOW_ORDERS: 106 | fragmented = True 107 | if num_numa_nodes > 1: 108 | fragmented_nodes.append(current_node) 109 | 110 | print(f" {line}") 111 | print(f" Total: {total} KB;\t\tLow: {low} KB ({low_percent}%)" 112 | f";\t\tHigh: {high} KB ({high_percent}%)") 113 | if fragmented is True: 114 | print("") 115 | if num_numa_nodes > 1: 116 | node_str = ", ".join(fragmented_nodes) 117 | self.print_warn( 118 | f"Memory on NUMA node(s) ({node_str}) is fragmented; " 119 | "system may run into compaction stalls.") 120 | else: 121 | self.print_warn( 122 | "Memory is fragmented - system may run into compaction " 123 | "stalls.") 124 | self.__display_per_zone_data() 125 | 126 | def __display_per_zone_data(self): 127 | self.__display_pagetypeinfo() 128 | self.__display_zoneinfo() 129 | 130 | def check_vmstat(self): 131 | """ 132 | Read /proc/vmstat and get counters related to compaction stalls, etc. 133 | """ 134 | vmstat = self.read_text_file("/proc/vmstat", on_error="") 135 | if vmstat == "": 136 | self.print_error("Unable to read file /proc/vmstat") 137 | return 138 | interesting = ( 139 | 'compact', 'allocstall_normal', 'kswapd_low_wmark_hit_quickly', 140 | 'kswapd_high_wmark_hit_quickly', 'drop_', 'oom_', 141 | 'zone_reclaim_failed' 142 | ) 143 | print("\nVmstat:") 144 | for line in vmstat.splitlines(): 145 | if any(line.startswith(word) for word in interesting): 146 | print(f" {line}") 147 | print("") 148 | return 149 | -------------------------------------------------------------------------------- /tools/syswatch/syswatch.8.scd: -------------------------------------------------------------------------------- 1 | syswatch(8) 2 | 3 | # NAME 4 | 5 | syswatch - Execute user-provided commands on CPU utilization. 6 | 7 | # SYNOPSIS 8 | 9 | syswatch [-h] [-b] [-C NR_CPUS] -s STAT:PERCENTAGE [-t TARGET_DIR] 10 | \[-M MAX_FS_UTIL] [-I SECONDS] -c CMD [-c CMD ...] 11 | 12 | # DESCRIPTION 13 | 14 | *syswatch* is used to watch CPU utilization and execute user-provided commands 15 | when the configured thresholds are reached. 16 | 17 | The tool samples CPU utilization for intervals of _SECONDS_ (5 seconds by 18 | default), and if the configured thresholds are reached in that interval, the 19 | commands specified with *-c* are executed. 20 | 21 | At least one threshold to watch must be specified with *-s*. If more than one 22 | threshold is specified, i.e. *-s* is specified more than once, the commands 23 | will only trigger if all the thresholds are reached in a given interval. These 24 | values are percentages and must be integer values in the range [1, 100]. The 25 | sum of all thresholds must be <= 100. 26 | 27 | If _NR_CPUS_ is not specified or is <= 0, the cumulative utilization of all 28 | CPUs is used. Otherwise, the commands trigger only if _at least_ _NR_CPUS_ 29 | reach all the configured thresholds in the interval. 30 | 31 | Option *-c* must be specified at least once. _CMD_ must be a valid shell 32 | command. See *USER COMMAND CONSIDERATIONS* for important details. 33 | 34 | By default, *syswatch* will create logs in subdirectories in 35 | /var/oled/syswatch. If *-t* is specified, _TARGET_DIR_ is used as base 36 | directory instead. The base directory will be created if missing (either the 37 | default value or the one configured with *-t*). See *LOGGING* for a 38 | description of the logs dumped by the tool. 39 | 40 | By default, *syswatch* exits once the user-provided commands finish execution 41 | when the CPU utilization thresholds are reached. If *-b* is specified, instead 42 | it will continue running indefinitely, it will restart watching once the user 43 | commands finish. 44 | 45 | *syswatch* monitors the space utilization of the filesystem where logging is 46 | stored (see *LOGGING*). If the configured space usage limit is reached, 47 | *syswatch* terminates. The default limit is 85% utilization (i.e. 15% free 48 | space) but this limit can be changed with *-M*. This limit is checked when 49 | *syswatch* starts running and when the CPU threshold are reached, before 50 | executing the user commands. 51 | 52 | *syswatch* must be run as root. 53 | 54 | # OPTIONS 55 | 56 | *-h* 57 | Display help. 58 | 59 | *-b* 60 | Run *syswatch* indefinitely until it's manually terminated. 61 | 62 | *-C NR_CPUS* 63 | # of CPUs to apply match criteria. A value <= 0 means apply system 64 | wide. (default: 0) 65 | 66 | *-s STAT:PERCENTAGE* 67 | CPU stat utilization to watch. _STAT_ can be any of the following 68 | values: usr, nice, sys, idle, iowait, irq, soft, steal, guest, gnice. 69 | _PERCENTAGE_ must be an integer in the range [1, 100]. 70 | 71 | *-t TARGET_DIR* 72 | _TARGET_DIR_ to store logs (see *LOGGING*). (default: 73 | /var/oled/syswatch). 74 | 75 | *-M MAX_FS_UTIL* 76 | Max filesystem utilization, in percentage. (default: 85) 77 | 78 | *-I SECONDS* 79 | CPU utilization sampling interval, in seconds. (default: 5) 80 | 81 | *-c CMD* 82 | User command to execute when thresholds are reached. 83 | 84 | # USER COMMAND CONSIDERATIONS 85 | 86 | - The user commands are assumed to be valid shell commands, and are executed 87 | within a shell (the equivalent of `sh -c '$CMD'`). 88 | 89 | - If more than one command is specified (i.e. *-c* appears more than once), all 90 | of them are executed in parallel (a separate shell is spawned for each one). 91 | 92 | - Each command is execute in its own, dedicated directory (see *LOGGING*). 93 | 94 | - When syswatch is instructed to run indefinitely (*-b*), CPU utilization 95 | during the time the commands are running will not be accounted; the next 96 | sampling interval will start when the commands finish. This ensures that CPU 97 | spikes caused by the user commands themselves don't trip the tool. 98 | 99 | # LOGGING 100 | 101 | Logs are written in _TARGET_DIR_ /var/oled/syswatch by default unless this is 102 | changed with *-t*. 103 | 104 | All logs of one *syswatch* invocation will be placed in INSTANCE_DIR 105 | $TARGET_DIR/syswatch_\_. Where is the date and time 106 | *syswatch* started running and is the PID of that instance. 107 | 108 | *syswatch*'s logs will be at $INSTANCE_DIR/syswatch.log. If stdout is a TTY, 109 | logging will be written to stdout too. 110 | 111 | When utilization thresholds are reached, a directory $INSTANCE_DIR/ 112 | will be created ( being the datetime at which the thresholds were 113 | reached) and inside this directory, one directory for every command to be 114 | executed (named after the commands themselves; e.g. for command `echo hello` 115 | directory `echo__hello` will be created). The commands will be executed with 116 | their $CWD pointing to this latter directory, and stdout/stderr redirected to 117 | file `output` inside that directory. 118 | 119 | When *syswatch* is ran indefinitely (*-b*), there will be one 120 | $INSTANCE_DIR/ directory for every time the thresholds are reached. 121 | 122 | # DATA COLLECTION 123 | 124 | When collecting data for distribution (e.g. to give to support team), it's 125 | recommended to collect the whole *syswatch*'s INSTANCE_DIR, as described in 126 | *LOGGING* (e.g. as a \*.tar.gz file). If there where several *syswatch* runs, 127 | the whole TARGET_DIR can be collected instead. 128 | 129 | # EXAMPLES 130 | 131 | - Collect perf data once 99% sys utilization is reached system-wide: 132 | 133 | ``` 134 | $ syswatch -s sys:99 -c 'perf record -c -g -F99 -- sleep 15' 135 | ``` 136 | 137 | - Run `cmd1` and `cmd2` in parallel when 70% sys and 30% user util is reached: 138 | 139 | ``` 140 | $ syswatch -s sys:70 -s usr:30 -c 'cmd1' -c 'cmd2' 141 | ``` 142 | 143 | - Run `cmd1` and `cmd2` serially and `cmd3` in parallel on 70% user util: 144 | 145 | ``` 146 | $ syswatch -s usr:70 -c 'cmd1 ; cmd2' -c 'cmd3' 147 | ``` 148 | 149 | - Run `cmd` if at least 3 CPUs reach 70% irq and 30% user within intervals of 150 | 2 seconds and the filesystem where the logs are stored has at most 80% space 151 | used (i.e. 20% free space): 152 | 153 | ``` 154 | $ syswatch -s irq:70 -s usr:30 -C 3 -I 2 -c 'cmd' -M 80 155 | ``` 156 | 157 | - Run syswatch continuously and run `cmd` every time 85% sys util is reached 158 | and use /tmp/syswatch as base dir for the logs: 159 | 160 | ``` 161 | $ syswatch -b -s sys:85 -c 'cmd' -t /tmp/syswatch 162 | ``` 163 | 164 | - Wait until system reached 90% sys util and exit, then run `cmd`: 165 | 166 | ``` 167 | $ syswatch -s sys:90 -c true && cmd 168 | ``` 169 | 170 | # AUTHOR 171 | 172 | Jose Lombera 173 | 174 | ; vim:set tw=79: 175 | -------------------------------------------------------------------------------- /tools/lkce/scripts/kdump_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | 23 | """ 24 | Program that runs inside kdump kernel 25 | Program to run crash utility inside the kdump kernel 26 | """ 27 | import glob 28 | import time 29 | import os 30 | import platform 31 | import re 32 | import shutil 33 | import subprocess # nosec 34 | import sys 35 | 36 | 37 | class KdumpReport: 38 | """Class to include all kdump report related functionality""" 39 | # pylint: disable=too-many-instance-attributes 40 | def __init__(self) -> None: 41 | """Constructor for KdumpReport class""" 42 | self.vmlinux = "" 43 | self.vmcore = "/proc/vmcore" 44 | self.kdump_kernel_ver = platform.uname().release 45 | 46 | self.kdump_report_home = "/etc/oled/lkce" 47 | self.kdump_report_config_file = self.kdump_report_home + "/lkce.conf" 48 | self.kdump_report_crash_cmds_file = self.kdump_report_home + \ 49 | "/crash_cmds_file" 50 | self.kdump_report_out = "/var/oled/lkce" 51 | self.kdump_report_out_file = self.kdump_report_out + \ 52 | "/crash_" + time.strftime("%Y%m%d-%H%M%S") + ".out" 53 | self.timedout_action = "reboot -f" 54 | 55 | # default values 56 | self.kdump_report = "yes" 57 | self.vmlinux_path = "/usr/lib/debug/lib/modules/" + \ 58 | self.kdump_kernel_ver + "/vmlinux" 59 | self.crash_cmds_file = self.kdump_report_crash_cmds_file 60 | self.max_out_files = "50" 61 | 62 | self.read_config(self.kdump_report_config_file) 63 | # def __init__ 64 | 65 | def read_config(self, filename: str) -> int: 66 | """Read config file and update the class variables""" 67 | if not os.path.exists(filename): 68 | sys.exit(1) 69 | 70 | try: 71 | file = open(filename, "r") 72 | except OSError: 73 | print(f"kdump_report: Unable to operate on file: {filename}") 74 | return 1 75 | 76 | for line in file.readlines(): 77 | if re.search("^#", line): # ignore lines starting with '#' 78 | continue 79 | 80 | # trim space/tab/newline from the line 81 | line = re.sub(r"\s+", "", line) 82 | 83 | entry = re.split("=", line) 84 | if "vmlinux_path" in entry[0] and entry[1]: 85 | self.vmlinux_path = entry[1] 86 | 87 | elif "crash_cmds_file" in entry[0] and entry[1]: 88 | self.crash_cmds_file = entry[1] 89 | 90 | elif "enable_kexec" in entry[0] and entry[1]: 91 | self.kdump_report = entry[1] 92 | 93 | elif "max_out_files" in entry[0] and entry[1]: 94 | self.max_out_files = entry[1] 95 | 96 | return 0 97 | # def read_config 98 | 99 | def get_vmlinux(self) -> int: 100 | """Check for vmlinux in config path and then in default location. 101 | 102 | Report error if not found 103 | """ 104 | vmlinux_1 = self.vmlinux_path 105 | vmlinux_2 = "/usr/lib/debug/lib/modules/" + \ 106 | self.kdump_kernel_ver + "/vmlinux" 107 | 108 | if os.path.exists(vmlinux_1) and os.path.isfile(vmlinux_1): 109 | self.vmlinux = vmlinux_1 110 | elif os.path.exists(vmlinux_2) and os.path.isfile(vmlinux_2): 111 | self.vmlinux = vmlinux_2 112 | else: 113 | print("kdump_report: vmlinux not found in following locations.") 114 | print("kdump_report: %s" % vmlinux_1) 115 | print("kdump_report: %s" % vmlinux_2) 116 | sys.exit(1) 117 | 118 | print("kdump_report: vmlinux found at %s" % self.vmlinux) 119 | return 0 120 | # def get_vmlinux 121 | 122 | def run_crash(self) -> int: 123 | """Run crash utility against vmcore""" 124 | crash_path = shutil.which("crash") 125 | 126 | if not crash_path: 127 | print("kdump_report: 'crash' executable not found") 128 | return 1 129 | 130 | self.get_vmlinux() 131 | if not (os.path.exists(self.crash_cmds_file) and 132 | os.path.isfile(self.crash_cmds_file)): 133 | print("kdump_report: %s not found" % self.crash_cmds_file) 134 | return 1 135 | 136 | os.makedirs(self.kdump_report_out, exist_ok=True) 137 | 138 | args = (crash_path, self.vmlinux, self.vmcore, "-i", 139 | self.crash_cmds_file) 140 | print(f"kdump_report: Executing '{' '.join(args)}'; output file " 141 | f"'{self.kdump_report_out_file}'") 142 | 143 | with open(self.kdump_report_out_file, "w") as output_fd: 144 | subprocess.run(args, close_fds=True, stdout=output_fd, 145 | stderr=output_fd, stdin=subprocess.DEVNULL, 146 | shell=False, check=True) # nosec 147 | return 0 148 | # def run_crash 149 | 150 | def clean_up(self) -> None: 151 | """Clean up old crash reports to save space""" 152 | max_files = int(self.max_out_files) 153 | crash_files = glob.glob(f"{self.kdump_report_out}/crash*out") 154 | 155 | if len(crash_files) > max_files: 156 | print(f"kdump_report: found more than {max_files}[max_out_files] " 157 | "out files. Deleting older ones") 158 | 159 | for file in sorted(crash_files, reverse=True)[max_files:]: 160 | try: 161 | os.remove(file) 162 | except OSError: 163 | pass # ignore permissions and missing file errors 164 | # def clean_up 165 | # class KDUMP_REPORT 166 | 167 | 168 | def main() -> int: 169 | """Main routine""" 170 | kdump_report = KdumpReport() 171 | 172 | if kdump_report.kdump_report != "yes": 173 | print("kdump_report: kdump_report is disabled to run") 174 | sys.exit(1) 175 | else: 176 | print("kdump_report: kdump_report is enabled to run") 177 | kdump_report.run_crash() 178 | kdump_report.clean_up() 179 | sys.exit(0) 180 | # def main 181 | 182 | 183 | if __name__ == '__main__': 184 | main() 185 | -------------------------------------------------------------------------------- /tools/vmcore-utils/vmcore_sz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | 23 | # Author: Partha Satapathy 24 | # Co-author: Srikanth C S 25 | 26 | """ 27 | The vmcore_sz takes the dump_level as an argument and estimates the 28 | vmcore size if a kernel dump is obtained at that moment. It displays 29 | the total number of pages; pages need to be excluded depending on 30 | the dump level and the expected VMcore size in bytes. If the dump 31 | level is not specified, the default configured in "/etc/kdump.conf" 32 | will be used. 33 | """ 34 | 35 | import sys 36 | import os 37 | import argparse 38 | import subprocess # nosec 39 | from typing import List 40 | from typing import NamedTuple 41 | 42 | 43 | class MemoryUsage(NamedTuple): 44 | """ 45 | This class saves page count details 46 | """ 47 | zero_pg: int 48 | npvt_pg: int 49 | pctc_pg: int 50 | user_pg: int 51 | free_pg: int 52 | page_sz: int 53 | total_pg: int 54 | 55 | 56 | def get_mem_usage() -> MemoryUsage: 57 | """ 58 | Get memory data from makedumpfile --mem-usage command 59 | """ 60 | mem_op = subprocess.run(["makedumpfile", # nosec 61 | "--mem-usage", "/proc/kcore"], 62 | stdout=subprocess.PIPE, 63 | stderr=subprocess.STDOUT, check=False) 64 | if mem_op.returncode != 0: 65 | print(mem_op.stdout.decode()) 66 | exit_with_msg("Error executing makedumpfile.") 67 | 68 | mem_usg = mem_op.stdout.decode().strip().splitlines() 69 | header = mem_usg.pop(0).split() 70 | typ, pages = header.index("TYPE"), header.index("PAGES") 71 | page_sz = total_pg = 0 72 | zero_pg = npvt_pg = pctc_pg = user_pg = free_pg = 0 73 | 74 | for i in mem_usg: 75 | if "page size:" in i: 76 | page_sz = int(i.split()[-1]) 77 | continue 78 | if "Total pages on system:" in i: 79 | total_pg = int(i.split()[-1]) 80 | continue 81 | arr = i.split() 82 | if not arr: 83 | continue 84 | if arr[typ] == "ZERO": 85 | zero_pg = int(arr[pages]) 86 | elif arr[typ] == "NON_PRI_CACHE": 87 | npvt_pg = int(arr[pages]) 88 | elif arr[typ] == "PRI_CACHE": 89 | pctc_pg = int(arr[pages]) 90 | elif arr[typ] == "USER": 91 | user_pg = int(arr[pages]) 92 | elif arr[typ] == "FREE": 93 | free_pg = int(arr[pages]) 94 | 95 | return MemoryUsage(zero_pg=zero_pg, npvt_pg=npvt_pg, 96 | pctc_pg=pctc_pg, user_pg=user_pg, 97 | free_pg=free_pg, page_sz=page_sz, 98 | total_pg=total_pg) 99 | 100 | 101 | def exit_with_msg(msg: str = "", error: int = 1) -> None: 102 | """" 103 | Error out when something undesired happens 104 | """ 105 | print(msg) 106 | sys.exit(error) 107 | 108 | 109 | def check_if_root() -> None: 110 | """ 111 | Check if the script is run in superuser mode 112 | """ 113 | if os.getuid() != 0: 114 | exit_with_msg("This tool should be run as root.", 1) 115 | 116 | 117 | def parse_args() -> argparse.Namespace: 118 | """ 119 | Parse the CLI arguments 120 | """ 121 | parser = argparse.ArgumentParser( 122 | prog='vmcore_sz', 123 | description='vmcore_sz: Estimating vmcore size before kernel dump') 124 | 125 | parser.add_argument("-d", "--dump_level", help="Dump level, " 126 | "an integer in range 0..31", 127 | type=int) 128 | 129 | args = parser.parse_args() 130 | if args.dump_level is not None: 131 | if not 0 <= args.dump_level <= 31: 132 | parser.error(f"Invalid dump level - {args.dump_level}") 133 | 134 | return args 135 | 136 | 137 | def get_default_dump_level() -> List[int]: 138 | """ 139 | Get the default dump level from /etc/kdump.conf 140 | """ 141 | find_def = [] 142 | default = [] 143 | output = subprocess.getoutput("grep core_collector /etc/kdump.conf " 144 | "| grep makedumpfile" 145 | "| grep -v '#'").strip().split() 146 | 147 | # The file /etc/kdump.conf can contain multiple dump levels. Such as 148 | # core_collector makedumpfile -l --message-level 7 -d 9,17,31 149 | # In cases like above, dump vmcore size estimate for all the dump levels. 150 | while "-d" in output: 151 | find_def.append(output[output.index("-d") + 1]) 152 | output.pop(output.index("-d")) 153 | 154 | for ele in find_def: 155 | default += [int(i) for i in ele.split(',')] 156 | 157 | return default 158 | 159 | 160 | def get_vmcore_size(mem: MemoryUsage, dump_level: int) -> int: 161 | """ 162 | Calculate size of the vmcore based on makedumpfile --mem-usage 163 | """ 164 | ex_page = dump_level 165 | dump_pg = mem.total_pg 166 | 167 | if ex_page & 0x1: 168 | print(f"Exclude zero pages : {mem.zero_pg}") 169 | dump_pg -= mem.zero_pg 170 | 171 | if ex_page & 0x2: 172 | print(f"Exclude non private cache : {mem.npvt_pg}") 173 | dump_pg -= mem.npvt_pg 174 | 175 | if ex_page & 0x4: 176 | print(f"Exclude private cache : {mem.pctc_pg}") 177 | dump_pg -= mem.pctc_pg 178 | 179 | if ex_page & 0x8: 180 | print(f"Exclude user pages : {mem.user_pg}") 181 | dump_pg -= mem.user_pg 182 | 183 | if ex_page & 0x10: 184 | print(f"Exclude free pages : {mem.free_pg}") 185 | dump_pg -= mem.free_pg 186 | 187 | print(f"\nTotal Pages : {mem.total_pg}") 188 | print(f"Pages to be dumped : {dump_pg}") 189 | dump_size = dump_pg * mem.page_sz 190 | 191 | return dump_size 192 | 193 | 194 | def main() -> None: 195 | """ 196 | main function 197 | """ 198 | args = parse_args() 199 | check_if_root() 200 | dump_levels = [] 201 | if args.dump_level is None: 202 | dump_levels = get_default_dump_level() 203 | print("\nDump level is not specified. " 204 | f"Using default/configured - {dump_levels}") 205 | else: 206 | dump_levels.append(args.dump_level) 207 | 208 | mem = get_mem_usage() 209 | 210 | for level in dump_levels: 211 | print("\n----------------------------------------------") 212 | if not 0 <= level <= 31: 213 | print(f"Invalid dump level {level}") 214 | continue 215 | print(f"Dump level = {level}") 216 | dump_size = get_vmcore_size(mem, level) 217 | 218 | print("\n----------------------------------------------") 219 | print(f"Expected vmcore size in bytes : {dump_size}") 220 | print("----------------------------------------------\n") 221 | 222 | 223 | if __name__ == '__main__': 224 | main() 225 | -------------------------------------------------------------------------------- /tools/syswatch/syswatch.8: -------------------------------------------------------------------------------- 1 | .\" Generated by scdoc 1.9.6 2 | .ie \n(.g .ds Aq \(aq 3 | .el .ds Aq ' 4 | .nh 5 | .ad l 6 | .\" Begin generated content: 7 | .TH "syswatch" "8" "2023-07-20" 8 | .P 9 | .SH NAME 10 | .P 11 | syswatch - Execute user-provided commands on CPU utilization. 12 | .P 13 | .SH SYNOPSIS 14 | .P 15 | syswatch [-h] [-b] [-C NR_CPUS] -s STAT:PERCENTAGE [-t TARGET_DIR] 16 | [-M MAX_FS_UTIL] [-I SECONDS] -c CMD [-c CMD ...] 17 | .P 18 | .SH DESCRIPTION 19 | .P 20 | \fBsyswatch\fR is used to watch CPU utilization and execute user-provided commands 21 | when the configured thresholds are reached. 22 | .P 23 | The tool samples CPU utilization for intervals of \fISECONDS\fR (5 seconds by 24 | default), and if the configured thresholds are reached in that interval, the 25 | commands specified with \fB-c\fR are executed. 26 | .P 27 | At least one threshold to watch must be specified with \fB-s\fR. If more than one 28 | threshold is specified, i.e. \fB-s\fR is specified more than once, the commands 29 | will only trigger if all the thresholds are reached in a given interval. These 30 | values are percentages and must be integer values in the range [1, 100]. The 31 | sum of all thresholds must be <= 100. 32 | .P 33 | If \fINR_CPUS\fR is not specified or is <= 0, the cumulative utilization of all 34 | CPUs is used. Otherwise, the commands trigger only if \fIat least\fR \fINR_CPUS\fR 35 | reach all the configured thresholds in the interval. 36 | .P 37 | Option \fB-c\fR must be specified at least once. \fICMD\fR must be a valid shell 38 | command. See \fBUSER COMMAND CONSIDERATIONS\fR for important details. 39 | .P 40 | By default, \fBsyswatch\fR will create logs in subdirectories in 41 | /var/oled/syswatch. If \fB-t\fR is specified, \fITARGET_DIR\fR is used as base 42 | directory instead. The base directory will be created if missing (either the 43 | default value or the one configured with \fB-t\fR). See \fBLOGGING\fR for a 44 | description of the logs dumped by the tool. 45 | .P 46 | By default, \fBsyswatch\fR exits once the user-provided commands finish execution 47 | when the CPU utilization thresholds are reached. If \fB-b\fR is specified, instead 48 | it will continue running indefinitely, it will restart watching once the user 49 | commands finish. 50 | .P 51 | \fBsyswatch\fR monitors the space utilization of the filesystem where logging is 52 | stored (see \fBLOGGING\fR). If the configured space usage limit is reached, 53 | \fBsyswatch\fR terminates. The default limit is 85% utilization (i.e. 15% free 54 | space) but this limit can be changed with \fB-M\fR. This limit is checked when 55 | \fBsyswatch\fR starts running and when the CPU threshold are reached, before 56 | executing the user commands. 57 | .P 58 | \fBsyswatch\fR must be run as root. 59 | .P 60 | .SH OPTIONS 61 | .P 62 | \fB-h\fR 63 | .RS 4 64 | Display help. 65 | .P 66 | .RE 67 | \fB-b\fR 68 | .RS 4 69 | Run \fBsyswatch\fR indefinitely until it's manually terminated. 70 | .P 71 | .RE 72 | \fB-C NR_CPUS\fR 73 | .RS 4 74 | # of CPUs to apply match criteria. A value <= 0 means apply system 75 | wide. (default: 0) 76 | .P 77 | .RE 78 | \fB-s STAT:PERCENTAGE\fR 79 | .RS 4 80 | CPU stat utilization to watch. \fISTAT\fR can be any of the following 81 | values: usr, nice, sys, idle, iowait, irq, soft, steal, guest, gnice. 82 | \fIPERCENTAGE\fR must be an integer in the range [1, 100]. 83 | .P 84 | .RE 85 | \fB-t TARGET_DIR\fR 86 | .RS 4 87 | \fITARGET_DIR\fR to store logs (see \fBLOGGING\fR). (default: 88 | /var/oled/syswatch). 89 | .P 90 | .RE 91 | \fB-M MAX_FS_UTIL\fR 92 | .RS 4 93 | Max filesystem utilization, in percentage. (default: 85) 94 | .P 95 | .RE 96 | \fB-I SECONDS\fR 97 | .RS 4 98 | CPU utilization sampling interval, in seconds. (default: 5) 99 | .P 100 | .RE 101 | \fB-c CMD\fR 102 | .RS 4 103 | User command to execute when thresholds are reached. 104 | .P 105 | .RE 106 | .SH USER COMMAND CONSIDERATIONS 107 | .P 108 | .RS 4 109 | .ie n \{\ 110 | \h'-04'\(bu\h'+03'\c 111 | .\} 112 | .el \{\ 113 | .IP \(bu 4 114 | .\} 115 | The user commands are assumed to be valid shell commands, and are executed 116 | within a shell (the equivalent of `sh -c '$CMD'`). 117 | 118 | .RE 119 | .P 120 | .RS 4 121 | .ie n \{\ 122 | \h'-04'\(bu\h'+03'\c 123 | .\} 124 | .el \{\ 125 | .IP \(bu 4 126 | .\} 127 | If more than one command is specified (i.e. \fB-c\fR appears more than once), all 128 | of them are executed in parallel (a separate shell is spawned for each one). 129 | 130 | .RE 131 | .P 132 | .RS 4 133 | .ie n \{\ 134 | \h'-04'\(bu\h'+03'\c 135 | .\} 136 | .el \{\ 137 | .IP \(bu 4 138 | .\} 139 | Each command is execute in its own, dedicated directory (see \fBLOGGING\fR). 140 | 141 | .RE 142 | .P 143 | .RS 4 144 | .ie n \{\ 145 | \h'-04'\(bu\h'+03'\c 146 | .\} 147 | .el \{\ 148 | .IP \(bu 4 149 | .\} 150 | When syswatch is instructed to run indefinitely (\fB-b\fR), CPU utilization 151 | during the time the commands are running will not be accounted; the next 152 | sampling interval will start when the commands finish. This ensures that CPU 153 | spikes caused by the user commands themselves don't trip the tool. 154 | 155 | .RE 156 | .P 157 | .SH LOGGING 158 | .P 159 | Logs are written in \fITARGET_DIR\fR /var/oled/syswatch by default unless this is 160 | changed with \fB-t\fR. 161 | .P 162 | All logs of one \fBsyswatch\fR invocation will be placed in INSTANCE_DIR 163 | $TARGET_DIR/syswatch__. Where is the date and time 164 | \fBsyswatch\fR started running and is the PID of that instance. 165 | .P 166 | \fBsyswatch\fR's logs will be at $INSTANCE_DIR/syswatch.log. If stdout is a TTY, 167 | logging will be written to stdout too. 168 | .P 169 | When utilization thresholds are reached, a directory $INSTANCE_DIR/ 170 | will be created ( being the datetime at which the thresholds were 171 | reached) and inside this directory, one directory for every command to be 172 | executed (named after the commands themselves; e.g. for command `echo hello` 173 | directory `echo__hello` will be created). The commands will be executed with 174 | their $CWD pointing to this latter directory, and stdout/stderr redirected to 175 | file `output` inside that directory. 176 | .P 177 | When \fBsyswatch\fR is ran indefinitely (\fB-b\fR), there will be one 178 | $INSTANCE_DIR/ directory for every time the thresholds are reached. 179 | .P 180 | .SH DATA COLLECTION 181 | .P 182 | When collecting data for distribution (e.g. to give to support team), it's 183 | recommended to collect the whole \fBsyswatch\fR's INSTANCE_DIR, as described in 184 | \fBLOGGING\fR (e.g. as a *.tar.gz file). If there where several \fBsyswatch\fR runs, 185 | the whole TARGET_DIR can be collected instead. 186 | .P 187 | .SH EXAMPLES 188 | .P 189 | .RS 4 190 | .ie n \{\ 191 | \h'-04'\(bu\h'+03'\c 192 | .\} 193 | .el \{\ 194 | .IP \(bu 4 195 | .\} 196 | Collect perf data once 99% sys utilization is reached system-wide: 197 | 198 | .RE 199 | .P 200 | .nf 201 | .RS 4 202 | $ syswatch -s sys:99 -c 'perf record -c -g -F99 -- sleep 15' 203 | .fi 204 | .RE 205 | .P 206 | .RS 4 207 | .ie n \{\ 208 | \h'-04'\(bu\h'+03'\c 209 | .\} 210 | .el \{\ 211 | .IP \(bu 4 212 | .\} 213 | Run `cmd1` and `cmd2` in parallel when 70% sys and 30% user util is reached: 214 | 215 | .RE 216 | .P 217 | .nf 218 | .RS 4 219 | $ syswatch -s sys:70 -s usr:30 -c 'cmd1' -c 'cmd2' 220 | .fi 221 | .RE 222 | .P 223 | .RS 4 224 | .ie n \{\ 225 | \h'-04'\(bu\h'+03'\c 226 | .\} 227 | .el \{\ 228 | .IP \(bu 4 229 | .\} 230 | Run `cmd1` and `cmd2` serially and `cmd3` in parallel on 70% user util: 231 | 232 | .RE 233 | .P 234 | .nf 235 | .RS 4 236 | $ syswatch -s usr:70 -c 'cmd1 ; cmd2' -c 'cmd3' 237 | .fi 238 | .RE 239 | .P 240 | .RS 4 241 | .ie n \{\ 242 | \h'-04'\(bu\h'+03'\c 243 | .\} 244 | .el \{\ 245 | .IP \(bu 4 246 | .\} 247 | Run `cmd` if at least 3 CPUs reach 70% irq and 30% user within intervals of 248 | 2 seconds and the filesystem where the logs are stored has at most 80% space 249 | used (i.e. 20% free space): 250 | 251 | .RE 252 | .P 253 | .nf 254 | .RS 4 255 | $ syswatch -s irq:70 -s usr:30 -C 3 -I 2 -c 'cmd' -M 80 256 | .fi 257 | .RE 258 | .P 259 | .RS 4 260 | .ie n \{\ 261 | \h'-04'\(bu\h'+03'\c 262 | .\} 263 | .el \{\ 264 | .IP \(bu 4 265 | .\} 266 | Run syswatch continuously and run `cmd` every time 85% sys util is reached 267 | and use /tmp/syswatch as base dir for the logs: 268 | 269 | .RE 270 | .P 271 | .nf 272 | .RS 4 273 | $ syswatch -b -s sys:85 -c 'cmd' -t /tmp/syswatch 274 | .fi 275 | .RE 276 | .P 277 | .RS 4 278 | .ie n \{\ 279 | \h'-04'\(bu\h'+03'\c 280 | .\} 281 | .el \{\ 282 | .IP \(bu 4 283 | .\} 284 | Wait until system reached 90% sys util and exit, then run `cmd`: 285 | 286 | .RE 287 | .P 288 | .nf 289 | .RS 4 290 | $ syswatch -s sys:90 -c true && cmd 291 | .fi 292 | .RE 293 | .P 294 | .SH AUTHOR 295 | .P 296 | Jose Lombera 297 | .P 298 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Common code for memstate modules.""" 23 | 24 | from __future__ import print_function 25 | import os 26 | import subprocess # nosec 27 | import logging 28 | import fcntl 29 | import platform 30 | import socket 31 | import sys 32 | from datetime import datetime 33 | from typing import Set, Optional 34 | 35 | from memstate_lib import constants 36 | 37 | 38 | class Base: 39 | """ Contains general helper functions used by all other modules. """ 40 | # pylint: disable=too-many-public-methods 41 | 42 | logged_messages: Set[str] = set() 43 | debug_log_configured = False 44 | 45 | def create_file_path(self, filename): 46 | """Create the base dir of a file path if missing.""" 47 | end = filename.rfind("/") 48 | parent_dir = filename[0:end] 49 | if not os.path.exists(parent_dir): 50 | try: 51 | os.makedirs(parent_dir, mode=0o700) 52 | except OSError as exn: 53 | self.log_debug( 54 | f"Could not create directory {parent_dir}: {exn}") 55 | 56 | def read_text_file(self, path: str, on_error: Optional[str] = None) -> str: 57 | """Read content of a file as text. 58 | 59 | If an error occurred, if on_error is not None, return that value; 60 | otherwise raise an exception. 61 | """ 62 | try: 63 | with open(path, "r", encoding="utf8") as fdesc: 64 | return fdesc.read() 65 | except Exception as exn: 66 | self.log_debug(f"Unable to read file '{path}': {exn}") 67 | 68 | if on_error is not None: 69 | return on_error 70 | 71 | raise 72 | 73 | def exec_cmd(self, cmd): 74 | """ 75 | Execute the command passed in as the parameter. 76 | @param cmd: The full command to be executed. 77 | """ 78 | output = "" 79 | try: 80 | args = cmd.split() 81 | result = subprocess.run( 82 | args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 83 | shell=False, check=True) # nosec 84 | output = result.stdout.decode('utf-8') 85 | except Exception as exn: # pylint: disable=broad-except 86 | output = "" 87 | self.log_debug(f"Command '{cmd}' failed with error: {exn}") 88 | return output 89 | 90 | @staticmethod 91 | def get_page_size(): 92 | """Get the default page size on the current system.""" 93 | try: 94 | return os.sysconf("SC_PAGE_SIZE") 95 | except ValueError: 96 | print("Unable to read the PAGE_SIZE of this system; exiting.\n") 97 | sys.exit(1) 98 | 99 | @staticmethod 100 | def print_error(warn_str): 101 | """Print an error message.""" 102 | print(f"{'[ERR] ': <8}{warn_str}") 103 | 104 | @staticmethod 105 | def print_warn(err_str): 106 | """Print a warning message.""" 107 | print(f"{'[WARN] ': <8}{err_str}") 108 | 109 | @staticmethod 110 | def print_info(msg_str): 111 | """Print an info message.""" 112 | print(f"{'[OK] ': <8}{msg_str}") 113 | 114 | def create_memstate_debug_log(self): 115 | """Create memstate debug log file.""" 116 | self.create_file_path(constants.MEMSTATE_DEBUG_LOG) 117 | logging.basicConfig( 118 | filename=constants.MEMSTATE_DEBUG_LOG, level=logging.DEBUG, 119 | format='%(asctime)s %(levelname)-8s %(message)s', 120 | datefmt='%Y-%m-%d %H:%M:%S') 121 | 122 | @staticmethod 123 | def log_debug(err_str): 124 | """ 125 | This function logs errors/warnings/etc. raised by the tool, 126 | for tool debug and development. 127 | """ 128 | if not Base.debug_log_configured: 129 | Base().create_memstate_debug_log() 130 | Base.debug_log_configured = True 131 | if err_str not in Base.logged_messages: 132 | Base.logged_messages.add(err_str) 133 | logging.debug("[memstate_debug] %s", err_str) 134 | 135 | def print_cmd_err(self, cmd_str): 136 | """Print an error informing failure when running a command.""" 137 | self.print_error("Unable to run command '" + cmd_str + "'.") 138 | 139 | @staticmethod 140 | def convert_mb_to_gb(val_in_mb): 141 | """Return MBs to GBs.""" 142 | val_in_gb = float(val_in_mb)/constants.ONE_KB 143 | return round(val_in_gb, 1) 144 | 145 | @staticmethod 146 | def convert_kb_to_gb(val_in_kb): 147 | """Return KBs to GBs.""" 148 | val_in_gb = float(val_in_kb)/(constants.ONE_KB**2) 149 | return round(val_in_gb, 1) 150 | 151 | @staticmethod 152 | def convert_bytes_to_gb(val_in_bytes): 153 | """Return number of bytes to GBs.""" 154 | val_in_gb = float(val_in_bytes)/(constants.ONE_KB**3) 155 | return round(val_in_gb, 1) 156 | 157 | @staticmethod 158 | def convert_bytes_to_numpages(val_in_bytes): 159 | """Return number of bytes to number pages.""" 160 | val_in_4k = \ 161 | float(val_in_bytes) / (Base.get_page_size() * constants.ONE_KB) 162 | return round(val_in_4k, 1) 163 | 164 | @staticmethod 165 | def print_pretty_gb(str_msg, int_arg): 166 | """Print GB value in a pretty format.""" 167 | print(f"{str_msg: <30}{int_arg: >12}") 168 | 169 | @staticmethod 170 | def print_pretty_kb(str_msg, int_arg): 171 | """Print KB value in a pretty format.""" 172 | print(f"{str_msg: <30}{int_arg: >12}") 173 | 174 | @staticmethod 175 | def get_kernel_ver(): 176 | """Get kernel version.""" 177 | return platform.uname().release 178 | 179 | @staticmethod 180 | def get_current_time(): 181 | """Get current time formatted as 'm/d/Y H:M:S'. 182 | 183 | If current time could not be determined, return "Unknown". 184 | """ 185 | current_time = datetime.now().strftime("%m/%d/%Y %H:%M:%S") 186 | if not current_time: 187 | current_time = "Unknown" 188 | return current_time.strip() 189 | 190 | @staticmethod 191 | def get_hostname(): 192 | """Get hostname. 193 | 194 | Return "Unknown" if hostname could not be retrieved. 195 | """ 196 | hname = socket.gethostname() 197 | if not hname: 198 | hname = "Unknown" 199 | return hname 200 | 201 | @staticmethod 202 | def open_file(filename, mode): 203 | """Open file with the given mode. 204 | 205 | If an OSError occurs, return an empty string instead.""" 206 | try: 207 | return open(filename, mode, encoding="utf8") 208 | except OSError: 209 | # Ignore if file does not exist 210 | return "" 211 | 212 | @staticmethod 213 | def order_x_in_kb(order): 214 | """ 215 | Translate order-x page into its corresponding size, in KB. 216 | For instance: if order = 3, return 32 (KB). 217 | """ 218 | pages = 2**order 219 | return pages * Base.get_page_size() / constants.ONE_KB 220 | 221 | 222 | class LockFile(Base): 223 | """ Implements a lock file """ 224 | 225 | def __init__(self): 226 | self.lock_fd = None 227 | if os.path.exists(constants.LOCK_FILE_DIR): 228 | parent_dir = os.path.join(constants.LOCK_FILE_DIR, "memstate") 229 | else: 230 | parent_dir = os.path.join(constants.LOCK_FILE_DIR_OL6, "memstate") 231 | if not os.path.exists(parent_dir): 232 | try: 233 | os.makedirs(parent_dir, mode=0o700) 234 | except OSError as exn: 235 | self.log_debug( 236 | f"Could not create directory {parent_dir}: {exn}") 237 | return 238 | self.lock_filename = os.path.join(parent_dir, "lock") 239 | self.locked = False 240 | self.__create_lock() 241 | 242 | def __create_lock(self): 243 | """ Creates the directory and lock file """ 244 | self.create_file_path(self.lock_filename) 245 | 246 | try: 247 | # pylint: disable-next=consider-using-with 248 | self.lock_fd = open(self.lock_filename, "w", encoding="utf8") 249 | # Exclusive lock | non-blocking request 250 | fcntl.flock(self.lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) 251 | self.locked = True 252 | self.log_debug("Locked file " + self.lock_filename) 253 | except IOError as exn: 254 | self.log_debug(f"Failed to lock {self.lock_filename}: {exn}") 255 | print( 256 | "Another instance of this script is running; please kill that " 257 | "instance if you want to restart the script.") 258 | sys.exit(1) 259 | 260 | def __del__(self): 261 | if self.locked is True: 262 | try: 263 | self.log_debug( 264 | f"Cleaning up; deleting lock file {self.lock_filename}") 265 | fcntl.flock(self.lock_fd, fcntl.LOCK_UN) 266 | self.lock_fd.close() 267 | os.remove(self.lock_filename) 268 | except OSError as exp: 269 | self.log_debug( 270 | "Caught exception while cleaning up/deleting lock file: " 271 | f"{exp}") 272 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/pss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Helper module to analyze per-process memory usage.""" 23 | 24 | import time 25 | import os 26 | from collections import OrderedDict 27 | from memstate_lib import Base 28 | from memstate_lib import constants 29 | 30 | 31 | class Pss(Base): 32 | """ Analyzes output from /proc//smaps_rollup and /proc//smaps""" 33 | 34 | def __get_total_pss_gb(self, pss_list=None): 35 | if pss_list is None: 36 | self.print_error("Process smaps data unavailable!") 37 | return None 38 | pss_kb = sum(pss_list.values()) 39 | return self.convert_kb_to_gb(pss_kb) 40 | 41 | def __display_top_proc_pss(self, num): 42 | """ 43 | Add up all the pss values per-process, from /proc//smaps_rollup. 44 | Sort in descending order, and display the top consumers. 45 | Also display rss, private and smap_pss values. 46 | @param num: The number of top memory consumers to print. 47 | """ 48 | # pylint: disable=too-many-locals,too-many-branches,too-many-statements 49 | pss = {} 50 | pss_sorted = {} 51 | rss = {} 52 | priv = {} 53 | swap_pss = {} 54 | num_printed = 0 55 | start_time = time.time() 56 | num_files_scanned = 0 57 | proc_root = '/proc/' 58 | 59 | try: 60 | files_path = os.listdir(proc_root) 61 | for elem in files_path: 62 | abspath = os.path.join(proc_root, elem) 63 | if not os.path.isdir(abspath): 64 | continue 65 | smaps_file = os.path.join(abspath, "smaps_rollup") 66 | if not os.path.exists(smaps_file): 67 | continue 68 | pid = smaps_file.split("/")[2] 69 | if pid.isdigit() is False: 70 | continue 71 | data = "" 72 | try: 73 | with open(smaps_file, "r", encoding="utf8") as smaps_fd: 74 | data = smaps_fd.read() 75 | num_files_scanned = num_files_scanned + 1 76 | for line in data.splitlines(): 77 | if line.startswith("Pss:"): 78 | pss_val = line.split(":")[1].strip().split()[0] 79 | pss[pid] = int(pss_val) 80 | elif line.startswith("Rss:"): 81 | rss[pid] = \ 82 | int(line.split(":")[1].strip().split()[0]) 83 | elif line.startswith("Private_Clean:"): 84 | priv[pid] = priv.get(pid, 0) + \ 85 | int(line.split(":")[1].strip().split()[0]) 86 | elif line.startswith("Private_Dirty:"): 87 | priv[pid] = priv.get(pid, 0) + \ 88 | int(line.split(":")[1].strip().split()[0]) 89 | elif line.startswith("SwapPss:"): 90 | swap_pss[pid] = \ 91 | int(line.split(":")[1].strip().split()[0]) 92 | except OSError: 93 | pass 94 | except OSError: 95 | pass 96 | 97 | end_time = time.time() 98 | if not pss: 99 | print( 100 | "Unable to get process memory usage summary for all " 101 | "processes on this system;\nplease run memstate with " 102 | "`-p ` to get memory usage details for a specific " 103 | "process.") 104 | return 105 | 106 | self.log_debug( 107 | f"Time taken to extract Pss values from {num_files_scanned} " 108 | f"/proc//smaps_rollup files is {round(end_time - start_time)}" 109 | " second(s).") 110 | pss_sorted = OrderedDict( 111 | sorted(pss.items(), key=lambda x: x[1], reverse=True)) 112 | for pid in pss_sorted: 113 | if num != constants.NO_LIMIT and num_printed >= num: 114 | break 115 | comm_str = self.read_text_file( 116 | f"/proc/{pid}/comm", on_error=str(pid)) 117 | comm_str = f"{comm_str.strip()}({pid})" 118 | print( 119 | f"{comm_str: <30}{pss_sorted[pid]: >16}" 120 | f"{rss[pid]: >16}{priv[pid]: >16}" 121 | f"{swap_pss[pid]: >16}") 122 | num_printed += 1 123 | print("") 124 | print( 125 | "Total memory used by all processes: " 126 | f"{self.__get_total_pss_gb(pss_sorted)} GB") 127 | 128 | def __display_single_process_mem(self, pid): 129 | # pylint: disable=too-many-locals,too-many-branches,too-many-statements 130 | proc_mem = {} 131 | proc_root = '/proc/' 132 | 133 | vma_arr = {} 134 | pss_arr = {} 135 | shared_arr = {} 136 | priv_arr = {} 137 | hugetlb_arr = {} 138 | addr = 0 139 | print_vmas = 0 140 | 141 | try: 142 | abspath = os.path.join(proc_root, str(pid)) 143 | if not os.path.isdir(abspath): 144 | print("Could not read /proc/" + pid + "/smaps file.\n") 145 | return 146 | smaps_file = os.path.join(abspath, "smaps") 147 | if not os.path.exists(smaps_file): 148 | print("Could not read /proc/" + pid + "/smaps file.\n") 149 | return 150 | with open(smaps_file, "r", encoding="utf8") as smaps_fd: 151 | for line in smaps_fd: 152 | # Process the VMAs 153 | val = line.split()[0].strip().split("-")[0].strip() 154 | try: 155 | if int(val, 16): 156 | addr = val 157 | vma_arr[addr] = line 158 | except ValueError: 159 | pass 160 | if line.startswith("Pss:"): 161 | val = int(line.split(":")[1].strip().split()[0]) 162 | proc_mem["Pss"] = proc_mem.get("Pss", 0) + val 163 | pss_arr[addr] = val 164 | elif line.startswith("Shared_Clean:"): 165 | val = int(line.split(":")[1].strip().split()[0]) 166 | proc_mem["Shared"] = proc_mem.get("Shared", 0) + val 167 | shared_arr[addr] = shared_arr.get(addr, 0) + val 168 | elif line.startswith("Shared_Dirty:"): 169 | val = int(line.split(":")[1].strip().split()[0]) 170 | proc_mem["Shared"] = proc_mem.get("Shared", 0) + val 171 | shared_arr[addr] = shared_arr.get(addr, 0) + val 172 | elif line.startswith("Private_Clean:"): 173 | val = int(line.split(":")[1].strip().split()[0]) 174 | proc_mem["Private"] = proc_mem.get("Private", 0) + val 175 | priv_arr[addr] = priv_arr.get(addr, 0) + val 176 | elif line.startswith("Private_Dirty:"): 177 | val = int(line.split(":")[1].strip().split()[0]) 178 | proc_mem["Private"] = proc_mem.get("Private", 0) + val 179 | priv_arr[addr] = priv_arr.get(addr, 0) + val 180 | elif line.startswith("Shared_Hugetlb:"): 181 | val = int(line.split(":")[1].strip().split()[0]) 182 | proc_mem["Hugetlb"] = proc_mem.get("Hugetlb", 0) + val 183 | hugetlb_arr[addr] = hugetlb_arr.get(addr, 0) + val 184 | elif line.startswith("Private_Hugetlb:"): 185 | val = int(line.split(":")[1].strip().split()[0]) 186 | proc_mem["Hugetlb"] = proc_mem.get("Hugetlb", 0) + val 187 | hugetlb_arr[addr] = hugetlb_arr.get(addr, 0) + val 188 | except IOError: 189 | pass 190 | 191 | comm_str = self.read_text_file(f"/proc/{pid}/comm", on_error=str(pid)) 192 | print(f"Memory usage summary for process {comm_str.strip()}" 193 | f" (pid: {pid}):") 194 | for key, value in proc_mem.items(): 195 | print(f"{key: <16}{value: >16} KB") 196 | 197 | print_vmas = 0 198 | print("\nDisplaying process VMAs >= 256 KB (numbers are in KB):") 199 | for key, value in vma_arr.items(): 200 | # pylint: disable=too-many-boolean-expressions 201 | if pss_arr[key] >= constants.VMA_KB or \ 202 | priv_arr[key] >= constants.VMA_KB or \ 203 | shared_arr[key] >= constants.VMA_KB or \ 204 | hugetlb_arr[key] >= constants.VMA_KB: 205 | # Print header first 206 | if not print_vmas: 207 | print( 208 | f"{'ADDR': <28}{'PSS': >16}{'SHARED': >16}" 209 | f"{'PRIV': >16}{'HUGETLB': >16}{' ': ^12}" 210 | f"{'MAPPING': <32}") 211 | print_vmas = 1 212 | addr_range = value.split()[0] 213 | if len(value.split()) < 6: 214 | map_str = "[anon]" 215 | elif len(value.split()) == 6: 216 | map_str = value.split()[5] 217 | else: # 7 fields in vma line 218 | map_str = " ".join(value.split()[-2:]) 219 | print( 220 | f"{addr_range: <28}{pss_arr[key]: >16}" 221 | f"{shared_arr[key]: >16}{priv_arr[key]: >16}" 222 | f"{hugetlb_arr[key]: >16}{' ': ^12}{map_str: <32}") 223 | 224 | if print_vmas == 0: 225 | print("None") 226 | 227 | def memstate_check_pss(self, pid, num=constants.NUM_TOP_MEM_USERS): 228 | """Check per-process memory usage (PSS).""" 229 | if pid != constants.DEFAULT_SHOW_PSS_SUMMARY: 230 | self.__display_single_process_mem(pid) 231 | return 232 | if num == constants.NO_LIMIT: 233 | hdr = "PROCESS MEMORY USAGE (in KB, ordered by PSS):" 234 | else: 235 | hdr = f"TOP {num} MEMORY CONSUMERS (in KB, ordered by PSS):" 236 | print(hdr) 237 | print( 238 | f"{'PROCESS(PID)': <30}{'PSS': >16}{'RSS': >16}" 239 | f"{'PRIVATE': >16}{'SWAP': >16}") 240 | self.__display_top_proc_pss(num) 241 | print("") 242 | -------------------------------------------------------------------------------- /tools/memstate/memstate_lib/meminfo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (c) 2023, Oracle and/or its affiliates. 4 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 | # 6 | # This code is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License version 2 only, as 8 | # published by the Free Software Foundation. 9 | # 10 | # This code is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 | # version 2 for more details (a copy is included in the LICENSE file that 14 | # accompanied this code). 15 | # 16 | # You should have received a copy of the GNU General Public License version 17 | # 2 along with this work; if not, see . 18 | # 19 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 | # or visit www.oracle.com if you need additional information or have any 21 | # questions. 22 | """Helper module to report memory information.""" 23 | 24 | from __future__ import print_function 25 | from memstate_lib import Base 26 | from memstate_lib import constants 27 | from memstate_lib import Hugepages 28 | 29 | 30 | class Meminfo(Base): 31 | """ Analyzes output from /proc/meminfo; checks RDS allocation """ 32 | # pylint: disable=too-many-instance-attributes 33 | 34 | def __init__(self): 35 | self.mem_total_gb = 0 36 | self.slab_total_gb = 0 37 | self.pagetables_gb = 0 38 | self.swap_used_kb = 0 39 | self.unaccounted_gb = 0 40 | self.rds_cache_size_gb = 0 41 | self.committed_as_gb = 0 42 | self.hugepages = Hugepages() 43 | 44 | def __read_meminfo(self): 45 | data = self.read_text_file("/proc/meminfo", on_error="") 46 | if data == "": 47 | self.print_error("Unable to read /proc/meminfo") 48 | return data 49 | for line in data.splitlines(): 50 | if line.startswith("MemTotal:"): 51 | self.mem_total_gb = self.__meminfo_get_value_gb(line) 52 | if line.startswith("Slab:"): 53 | self.slab_total_gb = self.__meminfo_get_value_gb(line) 54 | if line.startswith("PageTables:"): 55 | self.pagetables_gb = self.__meminfo_get_value_gb(line) 56 | return data 57 | 58 | def __calc_rds_cache_size(self): 59 | """ 60 | Get memory consumed for all RDS rx frags from the output of rds-info. 61 | """ 62 | rds_output = self.exec_cmd("rds-info -c") 63 | size_gb = 0 64 | for line in rds_output.splitlines(): 65 | if line.strip().startswith("ib_rx_total_frags"): 66 | num_frags = int(line.split()[1]) 67 | size_gb = self.convert_kb_to_gb( 68 | num_frags * constants.FRAG_SIZE_KB) 69 | return round(size_gb, 1) 70 | 71 | def __meminfo_get_value_gb(self, line): 72 | kb_value = line.split(":")[1].strip().split(" ")[0] 73 | return self.convert_kb_to_gb(int(kb_value)) 74 | 75 | def get_total_ram_gb(self): 76 | """Return total system memory in GBs.""" 77 | self.__read_meminfo() 78 | return self.mem_total_gb 79 | 80 | def get_total_ram_kb(self): 81 | """Return total system memory in KBs.""" 82 | self.__read_meminfo() 83 | return round( 84 | self.mem_total_gb * constants.ONE_KB * constants.ONE_KB, 1) 85 | 86 | def get_rds_cache_gb(self): 87 | """Return RDS cache size in GBs.""" 88 | self.__calc_rds_cache_size() 89 | return self.rds_cache_size_gb 90 | 91 | def get_total_slab_gb(self): 92 | """Return total slab memory in GBs.""" 93 | self.__read_meminfo() 94 | return self.slab_total_gb 95 | 96 | def get_pagetables_gb(self): 97 | """Return size of page tables in GBs.""" 98 | self.__read_meminfo() 99 | return self.pagetables_gb 100 | 101 | def get_swap_used_kb(self): 102 | """Return amount of swap used in KBs.""" 103 | self.__read_meminfo() 104 | return self.swap_used_kb 105 | 106 | def get_unaccounted_memory_gb(self): 107 | """Return amount of unaccounted memory in GBs.""" 108 | return self.unaccounted_gb 109 | 110 | def print_pretty_gb_l1(self, str_msg, int_arg): 111 | """Pretty-print a GBs value with 1-level indentation.""" 112 | printstr = (" " * 2) + str_msg 113 | self.print_pretty_gb(printstr, int_arg) 114 | 115 | def print_pretty_gb_l2(self, str_msg, int_arg): 116 | """Pretty-print a GBs value with 2-level indentation.""" 117 | printstr = (" " * 4) + str_msg 118 | self.print_pretty_gb(printstr, int_arg) 119 | 120 | def __print_hugepages_summary(self): 121 | hp_nr = self.hugepages.get_nr_hugepages_matrix_kb() 122 | for key, val in hp_nr.items(): 123 | hp_nr_total = sum(val) 124 | if hp_nr_total: 125 | self.print_pretty_gb_l1( 126 | f"Total Hugepages ({key} KB)", 127 | round(self.convert_kb_to_gb(hp_nr_total), 1)) 128 | hp_free = self.hugepages.get_free_hugepages_matrix_kb() 129 | for key, val in hp_free.items(): 130 | hp_free_total = sum(val) 131 | if hp_free_total: 132 | self.print_pretty_gb_l1( 133 | f"Free Hugepages ({key} KB)", 134 | round(self.convert_kb_to_gb(hp_free_total), 1)) 135 | 136 | def display_usage_summary(self): 137 | """Report several memory metrics.""" 138 | # pylint: disable=too-many-statements,too-many-branches,too-many-locals 139 | 140 | mem_free = 0 141 | cached = 0 142 | buffers = 0 143 | vmalloc = 0 144 | kernelstack = 0 145 | anonpages = 0 146 | shmem = 0 147 | swap_total = 0 148 | swap_free = 0 149 | user_allocs = 0 150 | 151 | data = self.__read_meminfo() 152 | if data == "": 153 | self.print_cmd_err("cat /proc/meminfo") 154 | return 155 | 156 | print("MEMORY USAGE SUMMARY (in GB):") 157 | for line in data.splitlines(): 158 | if line.startswith("MemFree:"): 159 | mem_free = self.__meminfo_get_value_gb(line) 160 | if line.startswith("Cached:"): 161 | cached = self.__meminfo_get_value_gb(line) 162 | if line.startswith("Buffers:"): 163 | buffers = self.__meminfo_get_value_gb(line) 164 | if line.startswith("VmallocUsed:"): 165 | vmalloc = self.__meminfo_get_value_gb(line) 166 | if line.startswith("KernelStack:"): 167 | kernelstack = self.__meminfo_get_value_gb(line) 168 | if line.startswith("AnonPages:"): 169 | anonpages = self.__meminfo_get_value_gb(line) 170 | if line.startswith("Mapped:"): 171 | mapped = self.__meminfo_get_value_gb(line) 172 | if line.startswith("Shmem:"): 173 | shmem = self.__meminfo_get_value_gb(line) 174 | if line.startswith("SwapTotal:"): 175 | swap_total = self.__meminfo_get_value_gb(line) 176 | if line.startswith("SwapFree:"): 177 | swap_free = self.__meminfo_get_value_gb(line) 178 | if line.startswith("Committed_AS:"): 179 | self.committed_as_gb = self.__meminfo_get_value_gb(line) 180 | 181 | self.rds_cache_size_gb = self.__calc_rds_cache_size() 182 | self.swap_used_kb = round(swap_total - swap_free, 1) 183 | 184 | # Userspace allocations include process stack and heap, page cache, I/O 185 | # buffers. The page cache value includes mapped files as well as shared 186 | # memory. 187 | user_allocs = round(anonpages + cached + buffers, 1) 188 | 189 | # Kernel allocations include slabs, vmalloc, pagetables, stack, RDS 190 | # cache, and more. The rest of kernel allocations are "unknown" - these 191 | # typically tend to be kernel memory that are consumed via 192 | # __get_free_pages() or related APIs, which are not tracked anywhere. 193 | # If the "unaccounted" value is large, that's a red flag. 194 | kernel_allocs = round( 195 | self.slab_total_gb + vmalloc + self.pagetables_gb + kernelstack 196 | + self.rds_cache_size_gb, 1) 197 | self.unaccounted_gb = round( 198 | self.mem_total_gb - 199 | (mem_free + kernel_allocs + user_allocs 200 | + self.hugepages.get_total_hugepages_gb()), 1) 201 | kernel_allocs = round(kernel_allocs + self.unaccounted_gb, 1) 202 | 203 | self.print_pretty_gb("Total memory", self.mem_total_gb) 204 | self.print_pretty_gb("Free memory", mem_free) 205 | self.print_pretty_gb( 206 | "Used memory", round(self.mem_total_gb - mem_free, 1)) 207 | self.print_pretty_gb_l1("Userspace", user_allocs) 208 | self.print_pretty_gb_l2("Processes", round(anonpages + mapped, 1)) 209 | self.print_pretty_gb_l2("Page cache", cached) 210 | self.print_pretty_gb_l2("Shared mem", shmem) 211 | self.print_pretty_gb_l1("Kernel", kernel_allocs) 212 | self.print_pretty_gb_l2("Slabs", self.slab_total_gb) 213 | if self.rds_cache_size_gb != 0: 214 | self.print_pretty_gb_l2("RDS", self.rds_cache_size_gb) 215 | self.print_pretty_gb_l2("Unknown", self.unaccounted_gb) 216 | self.__print_hugepages_summary() 217 | self.print_pretty_gb( 218 | "Swap used", self.convert_kb_to_gb(self.swap_used_kb)) 219 | print("") 220 | 221 | def check_pagetables_size(self): 222 | """Emit a warning if pages tables are too large.""" 223 | self.__read_meminfo() 224 | if (0 < self.pagetables_gb >= 225 | (constants.PAGETABLES_USE_PERCENT * 226 | (self.mem_total_gb - 227 | self.hugepages.get_total_hugepages_gb()))): 228 | print("") 229 | self.print_warn( 230 | "Page tables are larger than expected " 231 | f"({self.pagetables_gb} GB); if this is an Exadata system, " 232 | "check if the DB parameter USE_LARGE_PAGES is set to ONLY.") 233 | else: 234 | print("") 235 | self.print_info( 236 | f"Page tables size is: {self.pagetables_gb} GB.") 237 | 238 | def check_unaccounted_memory(self): 239 | """Emit a warning if unaccounted memory is larger than a threshold.""" 240 | if (0 < self.unaccounted_gb >= 241 | (constants.UNACCOUNTED_THRESHOLD * 242 | (self.mem_total_gb - 243 | self.hugepages.get_total_hugepages_gb()))): 244 | print("") 245 | self.print_warn( 246 | "Unaccounted kernel memory use is larger than expected: " 247 | f"{self.unaccounted_gb} GB.") 248 | elif self.unaccounted_gb > 0: 249 | print("") 250 | self.print_info( 251 | f"Unaccounted kernel memory is: {self.unaccounted_gb} GB.") 252 | 253 | def check_committed_as(self): 254 | """ 255 | Emit a warning if committed virtual memory is more than physically 256 | available. 257 | """ 258 | self.__read_meminfo() 259 | if (self.committed_as_gb >= 260 | (self.mem_total_gb - self.hugepages.get_total_hugepages_gb())): 261 | print("") 262 | self.print_warn( 263 | "Max virtual memory allocated is more than available physical" 264 | f" memory (Committed_AS = {self.committed_as_gb} GB).") 265 | --------------------------------------------------------------------------------