├── .gitignore ├── requirements.txt ├── Documentation ├── User Guide │ ├── ProcmonUserGuide.pdf │ └── ProcmonUserGuide.docx └── Testing and Validation │ └── StressTesting.docx ├── security.md ├── Makefile ├── Dockerfile ├── procmon ├── events.yaml ├── NN_detect.py ├── cmon.py └── procmon.py ├── LICENSE ├── .github └── workflows │ └── codeql.yml ├── CONTRIBUTING.md ├── README.md └── CODE_OF_CONDUCT.md /.gitignore: -------------------------------------------------------------------------------- 1 | #skip VSCode directories 2 | .vs* 3 | /dist -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | black 2 | boto3 3 | botocore 4 | flake8 5 | pandas 6 | -------------------------------------------------------------------------------- /Documentation/User Guide/ProcmonUserGuide.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/interferencedetector/HEAD/Documentation/User Guide/ProcmonUserGuide.pdf -------------------------------------------------------------------------------- /Documentation/User Guide/ProcmonUserGuide.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/interferencedetector/HEAD/Documentation/User Guide/ProcmonUserGuide.docx -------------------------------------------------------------------------------- /Documentation/Testing and Validation/StressTesting.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intel/interferencedetector/HEAD/Documentation/Testing and Validation/StressTesting.docx -------------------------------------------------------------------------------- /security.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 3 | 4 | ## Reporting a Vulnerability 5 | Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE_EXTERNAL := procmon.tgz 2 | default: dist 3 | .PHONY: default dist format format_check style_error_check check 4 | 5 | 6 | dist/$(PACKAGE_EXTERNAL): procmon/procmon.py procmon/dockermon.py procmon/events.yaml procmon/NN_detect.py README.md LICENSE requirements.txt 7 | rm -rf dist 8 | rm -rf build 9 | mkdir -p dist 10 | mkdir -p build/procmon 11 | cp procmon/procmon.py procmon/dockermon.py procmon/events.yaml procmon/NN_detect.py README.md LICENSE requirements.txt build/procmon/ 12 | cd build && tar -czf ../dist/procmon.tgz procmon/* 13 | rm -rf build 14 | 15 | format_check: 16 | black --check procmon 17 | 18 | format: 19 | black procmon 20 | 21 | style_error_check: 22 | # ignore long lines and conflicts with black, i.e., black wins 23 | flake8 procmon --ignore=E501,W503,E203 24 | 25 | check: format_check style_error_check 26 | 27 | dist: check dist/$(PACKAGE_EXTERNAL) -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ENV DEBIAN_FRONTEND=noninteractive 3 | WORKDIR / 4 | RUN apt update ; apt-get install apt-transport-https ca-certificates -y ; update-ca-certificates 5 | RUN apt-get update && \ 6 | apt-get upgrade -y && \ 7 | apt-get install --no-install-recommends -y \ 8 | zip bison build-essential cmake flex git libedit-dev \ 9 | libllvm12 llvm-12-dev libclang-12-dev python zlib1g-dev libelf-dev libfl-dev python3-setuptools \ 10 | liblzma-dev arping netperf iperf linux-tools-generic python3-pip && rm -rf /var/lib/apt/lists/* 11 | RUN rm /usr/bin/perf 12 | RUN ln -s /usr/lib/linux-tools/*/perf /usr/bin/perf 13 | RUN git clone https://github.com/iovisor/bcc.git 14 | RUN mkdir bcc/build; cd bcc/build ; cmake .. ; make ; make install ; cmake -DPYTHON_CMD=python3 .. ; cd src/python/ ; make ; make install ; cd ../.. 15 | COPY procmon/ . 16 | COPY requirements.txt . 17 | RUN pip install -r requirements.txt -------------------------------------------------------------------------------- /procmon/events.yaml: -------------------------------------------------------------------------------- 1 | # events format 2 | # Each architecture has a list of event-name : code pairs 3 | events: 4 | cascadelake/skylake: 5 | refs: "0300" 6 | cycles: "003c" 7 | insts: "00c0" 8 | l1imiss: "e424" 9 | l1dmiss: "0151" 10 | l1dhit: "01d1" 11 | l2miss: "1ff1" 12 | l3miss: "20d1" 13 | ocr_ev1: "01b7,3f840007f7" 14 | ocr_ev3: "01b7,3fB80007f7" 15 | 16 | icelake: 17 | refs: "0300" 18 | cycles: "003c" 19 | insts: "00c0" 20 | l1imiss: "e424" 21 | l1dmiss: "0151" 22 | l1dhit: "01d1" 23 | l2miss: "1ff1" 24 | l3miss: "20d1" 25 | ocr_ev1: "01b7,104000477" 26 | ocr_ev2: "01b7,84002380" 27 | ocr_ev3: "01b7,730000477" 28 | ocr_ev4: "01b7,90002380" 29 | 30 | sapphirerapid: 31 | refs: "0300" 32 | cycles: "003c" 33 | insts: "00c0" 34 | l1imiss: "e424" 35 | l1dmiss: "0151" 36 | l1dhit: "01d1" 37 | l2miss: "1f25" 38 | l3miss: "20d1" 39 | ocr_ev1: "01b7,104004477" 40 | ocr_ev2: "01b7,84002380" 41 | ocr_ev3: "01b7,730007477" 42 | ocr_ev4: "01b7,90002380" 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (C) 2023 Intel Corporation 4 |   5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), 7 | to deal in the Software without restriction, including without limitation 8 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | and/or sell copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, subject to the following conditions: 11 |   12 | The above copyright notice and this permission notice shall be included 13 | in all copies or substantial portions of the Software. 14 |   15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL 18 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES 19 | OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 21 | OR OTHER DEALINGS IN THE SOFTWARE. 22 |   23 | SPDX-License-Identifier: MIT 24 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | schedule: 9 | - cron: '44 17 * * 4' 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | analyze: 16 | name: Analyze 17 | runs-on: 'ubuntu-latest' 18 | timeout-minutes: 360 19 | permissions: 20 | actions: read 21 | contents: read 22 | security-events: write 23 | 24 | strategy: 25 | fail-fast: false 26 | matrix: 27 | language: [ 'python' ] 28 | 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 32 | 33 | # Initializes the CodeQL tools for scanning. 34 | - name: Initialize CodeQL 35 | uses: github/codeql-action/init@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12 36 | with: 37 | languages: ${{ matrix.language }} 38 | - name: Autobuild 39 | uses: github/codeql-action/autobuild@4759df8df70c5ebe7042c3029bbace20eee13edd # v2.23.1 40 | - name: Perform CodeQL Analysis 41 | uses: github/codeql-action/analyze@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12 42 | with: 43 | category: "/language:${{matrix.language}}" 44 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ### License 4 | 5 | Interference Detector is licensed under the terms in [LICENSE](./LICENSE). By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms. 6 | 7 | ### Sign your work 8 | 9 | Please use the sign-off line at the end of the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. The rules are pretty simple: if you can certify 10 | the below (from [developercertificate.org](http://developercertificate.org/)): 11 | 12 | ``` 13 | Developer Certificate of Origin 14 | Version 1.1 15 | 16 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 17 | 660 York Street, Suite 102, 18 | San Francisco, CA 94110 USA 19 | 20 | Everyone is permitted to copy and distribute verbatim copies of this 21 | license document, but changing it is not allowed. 22 | 23 | Developer's Certificate of Origin 1.1 24 | 25 | By making a contribution to this project, I certify that: 26 | 27 | (a) The contribution was created in whole or in part by me and I 28 | have the right to submit it under the open source license 29 | indicated in the file; or 30 | 31 | (b) The contribution is based upon previous work that, to the best 32 | of my knowledge, is covered under an appropriate open source 33 | license and I have the right under that license to submit that 34 | work with modifications, whether created in whole or in part 35 | by me, under the same open source license (unless I am 36 | permitted to submit under a different license), as indicated 37 | in the file; or 38 | 39 | (c) The contribution was provided directly to me by some other 40 | person who certified (a), (b) or (c) and I have not modified 41 | it. 42 | 43 | (d) I understand and agree that this project and the contribution 44 | are public and that a record of the contribution (including all 45 | personal information I submit with it, including my sign-off) is 46 | maintained indefinitely and may be redistributed consistent with 47 | this project or the open source license(s) involved. 48 | ``` 49 | 50 | Then you just add a line to every git commit message: 51 | 52 | Signed-off-by: Joe Smith 53 | 54 | Use your real name (sorry, no pseudonyms or anonymous contributions.) 55 | 56 | If you set your `user.name` and `user.email` git configs, you can sign your 57 | commit automatically with `git commit -s`. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PROJECT NOT UNDER ACTIVE MANAGEMENT 2 | 3 | This project will no longer be maintained by Intel. 4 | 5 | Intel has ceased development and contributions including, but not limited to, maintenance, bug fixes, new releases, or updates, to this project. 6 | 7 | Intel no longer accepts patches to this project. 8 | 9 | If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the open source software community, please create your own fork of this project. 10 | 11 | Contact: webadmin@linux.intel.com 12 |
13 | 14 |
15 |
    16 |

    Workload Interference Detector

     17 |
18 |
19 | 20 | ![CodeQL](https://github.com/intel/interferencedetector/actions/workflows/codeql.yml/badge.svg)[![License](https://img.shields.io/badge/License-MIT-blue)](https://github.com/intel/interferencedetector/blob/master/LICENSE) 21 | 22 | [Requirements](#requirements) | [Usage](#usage) | [Demo](#demo) | [Notes](#notes) 23 |
24 | 25 | Workload Interference Detector uses a combination of hardware events and ebpf to capture a wholistic signature of a workload's performance at very low overhead. 26 | 1. instruction efficiency 27 | - cycles 28 | - instructions 29 | - cycles per instruction 30 | 2. disk IO 31 | - local bandwidth (MB/s) 32 | - remote bandwidth (MB/s) 33 | - disk reads (MB/s) 34 | - disk writes (MB/s) 35 | 3. network IO 36 | - network transmitted (MB/s) 37 | - network received (MB/s) 38 | 4. cache 39 | - L1 instrutions misses per instruction 40 | - L1 data hit ratio 41 | - L1 data miss ratio 42 | - L2 miss ratio 43 | - L3 miss ratio 44 | 5. scheduling 45 | - scheduled count 46 | - average queue length 47 | - average queue latency (ms) 48 | 49 | ## Requirements 50 | 1. Linux Perf 51 | 2. [BCC compiled from source.](https://github.com/iovisor/bcc/blob/master/INSTALL.md#source) 52 | 3. `pip install -r requirements.txt` 53 | 4. Access to PMU 54 | - Bare-metal 55 | - VM with vPMU exposed (uncore metrics like disk IO will be zero) 56 | 5. Intel Xeon chip 57 | - Skylake 58 | - Cascade Lake 59 | - Ice Lake 60 | - Sapphire Rapids 61 | 6. Python 62 | 63 | ## Usage 64 | 1. Monitor processes 65 | ``` 66 | sudo python3 procmon.py 67 | ``` 68 | 2. Monitor containers (can also export to cloudwatch) 69 | ``` 70 | sudo python3 cmon.py 71 | ``` 72 | 3. Detect process or container interference. A list of workloads that likely caused the performance degradation is shown. 73 | ``` 74 | # process 75 | sudo python3 NN_detect.py --pid --ref_signature --distance_ratio 0.15 76 | 77 | # container 78 | sudo python3 NN_detect.py --cid --ref_signature --distance_ratio 0.15 79 | ``` 80 | 81 | ## Demo 82 | 83 | ![basic_stats](https://raw.githubusercontent.com/wiki/intel/interferencedetector/NN_demo1.gif) 84 | 85 | ## Notes: 86 | ** Interference Detector was developed using the following as references: 87 | 1. github.com/iovisor/bcc/tools/llcstat.py (Apache 2.0) 88 | 2. github.com/iovisor/bcc/tools/tcptop.py (Apache 2.0) 89 | 3. github.com/iovisor/bcc/blob/master/examples/tracing/disksnoop.py (Apache 2.0) 90 | 4. github.com/iovisor/bcc/blob/master/tools/runqlen.py (Apache 2.0) 91 | 5. github.com/iovisor/bcc/blob/master/tools/runqlat.py (Apache 2.0) -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual 10 | identity and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the overall 26 | community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or advances of 31 | any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email address, 35 | without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | CommunityCodeOfConduct AT intel DOT com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series of 86 | actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or permanent 93 | ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within the 113 | community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at 126 | [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq -------------------------------------------------------------------------------- /procmon/NN_detect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ########################################################################################################### 4 | # Copyright (C) 2023 Intel Corporation 5 | # SPDX-License-Identifier: MIT 6 | ########################################################################################################### 7 | 8 | from subprocess import Popen, PIPE, check_output 9 | import argparse 10 | import sys 11 | import re 12 | import os 13 | from os.path import exists 14 | from enum import Enum 15 | from datetime import datetime 16 | import pandas 17 | 18 | 19 | class bcolors: 20 | HEADER = "\033[95m" 21 | OKGREEN = "\033[92m" 22 | OKBLUE = "\033[96m" 23 | FAIL = "\033[91m" 24 | ENDC = "\033[0m" 25 | 26 | 27 | parser = argparse.ArgumentParser( 28 | description="Detect Noisy Neighbors for a given PID (process-level) or container ID (container-level). ", 29 | formatter_class=argparse.RawDescriptionHelpFormatter, 30 | ) 31 | 32 | parser.add_argument("-p", "--pid", type=str, help="PID (process-level)") 33 | parser.add_argument("-c", "--cid", type=str, help="Container ID (container-level)") 34 | parser.add_argument( 35 | "--outfile", type=str, help="Output file to save live-updated performance data" 36 | ) 37 | group = parser.add_mutually_exclusive_group() 38 | group.add_argument( 39 | "-s", 40 | "--system_wide_signatures_path", 41 | type=str, 42 | help="path to signatures_*.csv CSV file with referernce signatures per container ID, as generated by cmon.", 43 | ) 44 | group.add_argument( 45 | "-r", 46 | "--ref_signature", 47 | type=str, 48 | help="The tool will use this signature as a baseline. Use the output of either procmon or cmon to collect the signature. The first element in the signature is `cycles`. All live updated signatures will be compared to this reference signature. Use a standalone signature (when the process is the only process executing in the system), or any signature collected over a performance-acceptable duration.", 49 | ) 50 | parser.add_argument( 51 | "-t", 52 | "--threshold", 53 | type=float, 54 | default=0.15, 55 | help="Threshold of acceptable distance from reference (default is 15%% from reference). If the distance is higher than this threshold, the monitored workload will be flagged as a noisy neighbor victim.", 56 | ) 57 | parser.add_argument( 58 | "-v", 59 | "--verbose", 60 | action="store_true", 61 | help="Show more analysis details.", 62 | ) 63 | 64 | args = parser.parse_args() 65 | 66 | # For a single reference signature, check that only pid or cid arguments are passed, but not both 67 | if args.ref_signature: 68 | if (args.pid and args.cid) or (not args.pid and not args.cid): 69 | print( 70 | "Please set either -p/--pid flag for process-level monitoring, or -c/--cid flag for container-level moitoring (but not both)." 71 | ) 72 | sys.exit() 73 | 74 | # Check if output file already exists 75 | if args.outfile: 76 | file_exists = exists(os.curdir + "/" + args.outfile) 77 | 78 | if file_exists: 79 | print( 80 | "Output file", 81 | args.outfile, 82 | "already exists. Please change the output file name", 83 | ) 84 | sys.exit() 85 | else: 86 | out_file = open(args.outfile, "w") 87 | 88 | 89 | # Dictionaries that maps a header to its index and vice versa 90 | header_index_dict = {} 91 | index_header_dict = {} 92 | 93 | 94 | class Direction(Enum): 95 | ANY = 0 96 | HIGHER_IS_BETTER = 1 97 | LOWER_IS_BETTER = 2 98 | IGNORE = 3 99 | 100 | 101 | # Mapping of countrers to performance degredation events (i.e. lower the better vs higher the better): 102 | metric_to_perf_direction = { 103 | "cycles": Direction.ANY, 104 | "insts": Direction.ANY, 105 | "cpi": Direction.LOWER_IS_BETTER, 106 | "l1i_mpi": Direction.LOWER_IS_BETTER, 107 | "l1d_hit_ratio": Direction.HIGHER_IS_BETTER, 108 | "l1d_miss_ratio": Direction.LOWER_IS_BETTER, 109 | "l2_miss_ratio": Direction.LOWER_IS_BETTER, 110 | "l3_miss_ratio": Direction.LOWER_IS_BETTER, 111 | "local_bw": Direction.HIGHER_IS_BETTER, 112 | "remote_bw": Direction.HIGHER_IS_BETTER, 113 | "disk_reads": Direction.HIGHER_IS_BETTER, 114 | "disk_writes": Direction.HIGHER_IS_BETTER, 115 | "network_tx": Direction.HIGHER_IS_BETTER, 116 | "network_rx": Direction.HIGHER_IS_BETTER, 117 | "avg_q_len": Direction.IGNORE, 118 | "scheduled_count": Direction.IGNORE, 119 | "avg_q_latency": Direction.IGNORE, 120 | } 121 | 122 | 123 | def get_CPU_to_NUMA_Mapping(): 124 | core_to_NUMA = {} 125 | num_of_threads_per_core = 0 126 | NUMA_indx = 0 127 | cpu_info = check_output(["lscpu"]) 128 | if cpu_info: 129 | cpu_info_list = cpu_info.decode().split("\n") 130 | for cpu_info_line in cpu_info_list: 131 | # if("NUMA node" in cpu_info_line and "CPU(s)" in cpu_info_line): 132 | if re.match(r"NUMA node.* CPU\(s\):", cpu_info_line): 133 | cpus_groups_list = cpu_info_line.split(":") 134 | if len(cpus_groups_list) > 1: 135 | cpu_groups = cpus_groups_list[1].split(",") 136 | for cpus in cpu_groups: 137 | start_end_indxs = cpus.split("-") 138 | start_indx = int(start_end_indxs[0]) 139 | end_indx = int(start_end_indxs[1]) + 1 140 | for index in range(start_indx, end_indx): 141 | core_to_NUMA[str(index)] = NUMA_indx 142 | NUMA_indx += 1 143 | elif re.match(r"Thread\(s\) per core: *", cpu_info_line): 144 | thread_per_core_str = cpu_info_line.split(":") 145 | if len(thread_per_core_str) > 1: 146 | num_of_threads_per_core = int(thread_per_core_str[1]) 147 | 148 | if len(core_to_NUMA) == 0: 149 | print("Failed to read CPU to NUMA mapping. Exiting...") 150 | sys.exit() 151 | 152 | if num_of_threads_per_core == 0: 153 | print("Failed to read Number of threads per core. Exiting...") 154 | sys.exit() 155 | 156 | num_of_physical_cores = len(core_to_NUMA) / num_of_threads_per_core 157 | 158 | return core_to_NUMA, num_of_physical_cores 159 | 160 | 161 | def get_signature_from_str(signature_str, start_index): 162 | signature_str_list = signature_str.split(",")[start_index:] 163 | return [float(elem) for elem in signature_str_list] 164 | 165 | 166 | def get_signatures_from_csv(cvs_signatures_path): 167 | try: 168 | dataframe = pandas.read_csv(cvs_signatures_path) 169 | except FileNotFoundError: 170 | print( 171 | "Signatures file not found. Please provide the path to signatures .csv file" 172 | ) 173 | sys.exit(1) 174 | key_col_name = dataframe.columns[0] 175 | returned_dict = {} 176 | for index in dataframe.index: 177 | singnature_str_list = dataframe.iloc[index]["cycles":].tolist() 178 | key = dataframe.iloc[index][key_col_name] 179 | if "_" in key: 180 | key = key.split("_")[0] 181 | returned_dict[key] = [float(elem) for elem in singnature_str_list] 182 | return returned_dict 183 | 184 | 185 | def get_min_distance_from_neighbor( 186 | impacted_metric_list, 187 | nn_signature, 188 | detected_signature, 189 | same_core=True, 190 | same_numa=True, 191 | ): 192 | # check if both signatures have the same length 193 | assert len(nn_signature) == len(detected_signature) 194 | 195 | min_dist = float("inf") 196 | contention_metric = "[]" 197 | # If the two workloads are running on different cores, exclude on-core counters: cycles insts cpi l1i_mpi l1d_mpi l2_mpi 198 | # If the two workloads are running on different cores and different NUMA, execulde NUMA shared resources: l3_mpi local_bw 199 | assert "cycles" in header_index_dict 200 | 201 | if same_core: 202 | index = 0 203 | elif same_numa: 204 | assert "l2_miss_ratio" in header_index_dict 205 | index = header_index_dict["l2_miss_ratio"] - header_index_dict["cycles"] 206 | else: # Different cores and different NUMA 207 | assert "remote_bw" in header_index_dict 208 | index = header_index_dict["remote_bw"] - header_index_dict["cycles"] 209 | for i in range(index, len(detected_signature)): 210 | metric_index = i + header_index_dict["cycles"] 211 | metric = index_header_dict[metric_index] 212 | perf_direction = metric_to_perf_direction[metric] 213 | if i not in impacted_metric_list or perf_direction == Direction.IGNORE: 214 | continue 215 | if nn_signature[i] == 0 and detected_signature[i] == 0: 216 | dist = 0 217 | else: 218 | dist = abs(nn_signature[i] - detected_signature[i]) / max( 219 | nn_signature[i], detected_signature[i] 220 | ) 221 | 222 | if dist < min_dist: 223 | min_dist = dist 224 | contention_metric = index_header_dict[i + header_index_dict["cycles"]] 225 | 226 | return min_dist, contention_metric 227 | 228 | 229 | def get_impacted_metrics_list(ref_signature, detected_signature, distance_threshold): 230 | # Verify if both signatures have the same length (this should always be the case) 231 | assert len(ref_signature) == len(detected_signature) 232 | 233 | max_dist = -1 234 | impacted_metrics_list = [] 235 | 236 | for i in range(len(detected_signature)): 237 | metric = index_header_dict[i + header_index_dict["cycles"]] 238 | perf_direction = metric_to_perf_direction[metric] 239 | 240 | if perf_direction == Direction.IGNORE: 241 | continue 242 | 243 | elif (perf_direction == Direction.HIGHER_IS_BETTER) and ( 244 | detected_signature[i] > ref_signature[i] 245 | ): 246 | # performance is better than reference, skip 247 | continue 248 | elif (perf_direction == Direction.LOWER_IS_BETTER) and ( 249 | detected_signature[i] < ref_signature[i] 250 | ): 251 | # performance is better than reference, skip 252 | continue 253 | 254 | dist = abs(ref_signature[i] - detected_signature[i]) / max(ref_signature[i], 1) 255 | if dist > distance_threshold: 256 | impacted_metrics_list.append(i) 257 | if dist > max_dist: 258 | max_dist = dist 259 | 260 | return max_dist, impacted_metrics_list 261 | 262 | 263 | def represents_int(s): 264 | try: 265 | int(s) 266 | except ValueError: 267 | return False 268 | else: 269 | return True 270 | 271 | 272 | # Returns True if the two core IDs are hyperthreads on the same physical core 273 | def is_hyperthread(num_of_physical_cores, process_core, neighbor_core): 274 | # Check that both core ids can be represented as int 275 | assert represents_int(process_core) 276 | assert represents_int(neighbor_core) 277 | # Example: A system with 48 physical cores and 2 threads per core has 96 threads in total. Thread 0 will be hyperthreaded with thread 48, 1 with 49, and so on.. 278 | return abs(int(process_core) - int(neighbor_core)) == num_of_physical_cores 279 | 280 | 281 | # Clear console screen 282 | def clear_screen(): 283 | print("\033[H\033[J", end="") 284 | 285 | 286 | def get_impacted_metrics_list_string(metrics, impacted_metric_list): 287 | str_list = [] 288 | for i in impacted_metric_list: 289 | str_list.append(metrics[header_index_dict["cycles"] + i]) 290 | 291 | return ",".join(str_list) 292 | 293 | 294 | def run_NN_detect(id_to_ref_signatures_dict): 295 | # print(id_to_ref_signatures_dict) 296 | CPU_to_NUMA, num_of_physical_cores = get_CPU_to_NUMA_Mapping() 297 | if args.pid: 298 | # run procmon 299 | proc = Popen( 300 | ["python3", "procmon.py"], 301 | stdin=PIPE, 302 | stdout=PIPE, 303 | stderr=PIPE, 304 | ) 305 | else: 306 | # Run cmon 307 | proc = Popen( 308 | ["python3", "cmon.py"], 309 | stdin=PIPE, 310 | stdout=PIPE, 311 | stderr=PIPE, 312 | ) 313 | 314 | id_to_detected_signatures_dict = {} 315 | metrics = [] 316 | header = "" 317 | 318 | while True: 319 | if not proc.stdout: 320 | print("Reading procmon's or cmon's stdout failed. Exiting...") 321 | return 322 | 323 | line = proc.stdout.readline().decode("utf-8").rstrip() 324 | if not line or "Exiting.." in line: 325 | error_message = line 326 | print("Calling procmon or cmon failed. Exiting...", error_message) 327 | return 328 | 329 | parts = line.split(",") 330 | if "Timestamp" in line: 331 | # Read the metrics names from procmon header 332 | metrics = parts 333 | header = line 334 | header_elements = header.split(",") 335 | for index, he in enumerate(header_elements): 336 | header_index_dict[he] = index 337 | index_header_dict[index] = he 338 | 339 | elif ( 340 | "------------" in line 341 | ): # indicates new collection interval in procmon/cmon 342 | # Clear console screen 343 | clear_screen() 344 | # Write to console 345 | print("-----------------------------------------------------------------") 346 | # Write to file 347 | if args.outfile and out_file: 348 | out_file.write( 349 | "-----------------------------------------------------------------" 350 | + "\n" 351 | ) 352 | if args.verbose: 353 | print(bcolors.HEADER, "Header:\t\t\t", header, bcolors.ENDC) 354 | for key in id_to_detected_signatures_dict: 355 | if key not in id_to_ref_signatures_dict: 356 | id_to_detected_signatures_dict[key] = [] 357 | continue 358 | ref_signature = id_to_ref_signatures_dict[key] 359 | print("________") 360 | print(bcolors.OKBLUE, "ContainerID: ", key, bcolors.ENDC) 361 | if args.verbose: 362 | print("Reference Signature:\t\t", ref_signature) 363 | process_signature_line_list = id_to_detected_signatures_dict[key] 364 | neighbors = [] 365 | for key2 in id_to_detected_signatures_dict: 366 | if key != key2: 367 | neighbors += id_to_detected_signatures_dict[key2] 368 | for process_signature_line in process_signature_line_list: 369 | detected_signature = get_signature_from_str( 370 | process_signature_line, header_index_dict["cycles"] 371 | ) 372 | process_core = process_signature_line.split(",")[ 373 | header_index_dict["core"] 374 | ] 375 | 376 | now = datetime.now() # current date and time 377 | date_time = now.strftime("%m/%d/%Y, %H:%M:%S") 378 | detected_str = ( 379 | "At time: " 380 | + date_time 381 | + " detected signature on core " 382 | + process_core 383 | ) 384 | if args.verbose: 385 | detected_str = detected_str + (" :\t" + str(detected_signature)) 386 | # Write to console 387 | print(detected_str) 388 | # Write to file 389 | if args.outfile and out_file: 390 | out_file.write(detected_str + "\n") 391 | 392 | ( 393 | dist_from_reference, 394 | impacted_metric_list, 395 | ) = get_impacted_metrics_list( 396 | ref_signature, detected_signature, args.threshold 397 | ) 398 | 399 | if dist_from_reference < args.threshold: 400 | status_str = ( 401 | "Distance from reference: " 402 | + str(round(dist_from_reference * 100, 2)) 403 | + "%\t" 404 | + " ==> Performance is OK" 405 | ) 406 | # Write to console 407 | print(bcolors.OKGREEN + status_str + bcolors.ENDC) 408 | # Write to file 409 | if args.outfile and out_file: 410 | out_file.write(status_str + "\n") 411 | else: 412 | impacted_metrics_string = get_impacted_metrics_list_string( 413 | metrics, impacted_metric_list 414 | ) 415 | status_str = ( 416 | "Distance from reference: " 417 | + str(round(dist_from_reference * 100, 2)) 418 | + "%\t" 419 | + " ==> Performance may suffer. Imapacted metrics: " 420 | + impacted_metrics_string 421 | ) 422 | # Write to console 423 | print(bcolors.FAIL + status_str + bcolors.ENDC) 424 | # Write to file 425 | if args.outfile and out_file: 426 | out_file.write(status_str + "\n") 427 | 428 | nn_same_core_distance_line_tuple_list = [] 429 | nn_same_numa_distance_line_tuple_list = [] 430 | nn_different_core_distance_line_tuple_list = [] 431 | 432 | for nn in neighbors: 433 | neighbor_signature = get_signature_from_str( 434 | nn, header_index_dict["cycles"] 435 | ) 436 | neighbor_core = nn.split(",")[header_index_dict["core"]] 437 | if process_core == neighbor_core or is_hyperthread( 438 | num_of_physical_cores, process_core, neighbor_core 439 | ): 440 | _same_core = True 441 | _same_numa = True 442 | list_to_append = nn_same_core_distance_line_tuple_list 443 | elif ( 444 | CPU_to_NUMA[process_core] == CPU_to_NUMA[neighbor_core] 445 | ): 446 | _same_core = False 447 | _same_numa = True 448 | list_to_append = nn_same_numa_distance_line_tuple_list 449 | else: 450 | _same_core = False 451 | _same_numa = False 452 | list_to_append = ( 453 | nn_different_core_distance_line_tuple_list 454 | ) 455 | 456 | ( 457 | dist_from_neighbor, 458 | contention_metric, 459 | ) = get_min_distance_from_neighbor( 460 | impacted_metric_list, 461 | neighbor_signature, 462 | detected_signature, 463 | same_core=_same_core, 464 | same_numa=_same_numa, 465 | ) 466 | neighbor_id = nn.split(",")[1] 467 | list_to_append.append( 468 | ( 469 | dist_from_neighbor, 470 | nn, 471 | neighbor_signature, 472 | contention_metric, 473 | neighbor_id, 474 | neighbor_core, 475 | ) 476 | ) 477 | # Sort neighbors ascendingly based on distance (less distance -> more noise) 478 | nn_same_core_distance_line_tuple_list.sort() 479 | nn_same_numa_distance_line_tuple_list.sort() 480 | nn_different_core_distance_line_tuple_list.sort() 481 | # Show Same-core Noisy Neighbors in order, most noisy on top 482 | i = 1 483 | for nn_tup in nn_same_core_distance_line_tuple_list: 484 | # skip processes with less than 10M cycles/sec 485 | if nn_tup[2][0] < 10000: 486 | continue 487 | if nn_tup[0] < args.threshold: 488 | NN_status_str = ( 489 | "[Same-Core/Thread] Noisy Neighbor #" 490 | + str(i) 491 | + " on core #" 492 | + nn_tup[5] 493 | + ":\t" 494 | + nn_tup[4] 495 | ) 496 | if args.verbose: 497 | NN_status_str = NN_status_str + ( 498 | " Full signature:" 499 | + nn_tup[1] 500 | + " Min distance: " 501 | + str(round(nn_tup[0], 2)) 502 | + " Max similarity in: " 503 | + nn_tup[3] 504 | ) 505 | # Write to console 506 | print(bcolors.FAIL + NN_status_str + bcolors.ENDC) 507 | # Write to file 508 | if args.outfile and out_file: 509 | out_file.write(NN_status_str + "\n") 510 | i += 1 511 | 512 | # Show Same-NUMA Noisy Neighbors in order, most noisy on top 513 | i = 0 514 | for nn_tup in nn_same_numa_distance_line_tuple_list: 515 | # skip processes with less than 10M cycles/sec 516 | if nn_tup[2][0] < 10000: 517 | continue 518 | if nn_tup[0] < args.threshold: 519 | NN_status_str = ( 520 | "[Same-NUMA] Noisy Neighbor #" 521 | + str(i) 522 | + " on core #" 523 | + nn_tup[5] 524 | + ":\t" 525 | + nn_tup[4] 526 | ) 527 | if args.verbose: 528 | NN_status_str = NN_status_str + ( 529 | " Full signature:" 530 | + nn_tup[1] 531 | + " Min distance: " 532 | ) 533 | print(bcolors.FAIL + NN_status_str + bcolors.ENDC) 534 | # Write to file 535 | if args.outfile and out_file: 536 | out_file.write(NN_status_str + "\n") 537 | i += 1 538 | 539 | # Show Different-core Noisy Neighbors in order, most noisy on top 540 | i = 0 541 | for nn_tup in nn_different_core_distance_line_tuple_list: 542 | # skip processes with less than 10M cycles/sec 543 | if nn_tup[2][0] < 10000000: 544 | continue 545 | if nn_tup[0] < args.threshold: 546 | NN_status_str = ( 547 | "[Diff-Core] Noisy Neighbor #" 548 | + str(i) 549 | + " on core #" 550 | + nn_tup[5] 551 | + ":\t" 552 | + nn_tup[4] 553 | ) 554 | if args.verbose: 555 | NN_status_str += ( 556 | " Full signature:" 557 | + nn_tup[1] 558 | + " Min distance: " 559 | + str(round(nn_tup[0], 2)) 560 | + " Max similarity in: " 561 | + nn_tup[3] 562 | ) 563 | # Write to console 564 | print(bcolors.FAILl + NN_status_str + bcolors.ENDC) 565 | # Write to file 566 | if args.outfile and out_file: 567 | out_file.write(NN_status_str + "\n") 568 | i += 1 569 | 570 | id_to_detected_signatures_dict[key] = [] 571 | 572 | elif ( 573 | "**Warning**" not in line 574 | and "cycles" not in line 575 | and "Architecture:" not in line 576 | ): # data line 577 | data_parts = line.split(",") 578 | _id = "" 579 | if "containerID" in header_index_dict: 580 | _id = data_parts[header_index_dict["containerID"]] 581 | elif "PID" in header_index_dict: 582 | _id = data_parts[header_index_dict["PID"]] 583 | 584 | if "_" in _id: 585 | _id = _id.split("_")[0] 586 | 587 | if _id != "": 588 | if _id not in id_to_detected_signatures_dict: 589 | id_to_detected_signatures_dict[_id] = [] 590 | id_to_detected_signatures_dict[_id].append(line) 591 | 592 | 593 | if __name__ == "__main__": 594 | try: 595 | if args.ref_signature: # single container 596 | ref_signature = get_signature_from_str(args.ref_signature, 0) 597 | pid_or_cid = args.pid if args.pid else args.cid 598 | run_NN_detect({pid_or_cid: ref_signature}) 599 | elif args.system_wide_signatures_path: 600 | ref_signatures = get_signatures_from_csv(args.system_wide_signatures_path) 601 | run_NN_detect(ref_signatures) 602 | else: 603 | print( 604 | "Please provide one of the following: (1) single container's reference sginature (--ref_signature). (2) Path of reference singatures file (--system_wide_signatures_path)." 605 | ) 606 | sys.exit() 607 | except KeyboardInterrupt: 608 | if args.outfile and out_file: 609 | out_file.close() 610 | print("Interrupted by user. Exiting...") 611 | sys.exit() 612 | -------------------------------------------------------------------------------- /procmon/cmon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ########################################################################################################### 4 | # Copyright (C) 2023 Intel Corporation 5 | # SPDX-License-Identifier: MIT 6 | ########################################################################################################### 7 | 8 | import subprocess 9 | from subprocess import Popen, PIPE, SubprocessError 10 | import argparse 11 | from multiprocessing import Manager, Process, Lock 12 | import time 13 | from datetime import datetime 14 | import sys 15 | from time import sleep 16 | import pandas as pd 17 | import os 18 | import boto3 19 | import botocore 20 | 21 | # List of metrics to average across PIDs running within the same container 22 | metrics_to_average = { 23 | "cpi", 24 | "l1i_mpi", 25 | "l1d_hit_ratio", 26 | "l1d_miss_ratio", 27 | "l2_miss_ratio", 28 | "l3_miss_ratio", 29 | "avg_q_len", 30 | "avg_q_latency", 31 | } 32 | 33 | 34 | def get_procmon_out( 35 | container_to_PID_dict, 36 | lock, 37 | container_to_signature_dict=None, 38 | duration=0, 39 | client=None, 40 | cloudwatch_sampling_duration_in_sec=10, 41 | ): 42 | metrics = [] 43 | # The following dictionary has data for each process and each core 44 | process_data_dict = {} 45 | process_PID_index = 1 46 | process_index = 2 47 | core_index = 4 48 | procmon_command = ["python3", "procmon.py"] 49 | seconds_to_skip = max(cloudwatch_sampling_duration_in_sec - 1, 0) 50 | seconds_counter = 0 51 | if duration > 0: 52 | procmon_command += ["-d", str(duration)] 53 | proc = Popen(procmon_command, stdin=PIPE, stdout=PIPE, stderr=PIPE) 54 | try: 55 | while True: 56 | line = proc.stdout.readline().decode("utf-8").rstrip() 57 | if not line or "Exiting.." in line: 58 | print("No output from procmon. Exiting...", line) 59 | return 60 | if "Timestamp" in line: 61 | metrics = line.split(",") 62 | for i in range(len(metrics)): 63 | if metrics[i] == "PID": 64 | process_PID_index = i 65 | elif metrics[i] == "core": 66 | core_index = i 67 | elif metrics[i] == "process": 68 | process_index = i 69 | 70 | elif "Architecture:" in line: 71 | continue 72 | 73 | elif "------------" in line: 74 | seconds_counter += 1 75 | # print cmon output 76 | sys.stdout.flush() 77 | # waiting for lock on container_to_PID_dict 78 | lock.acquire() 79 | try: 80 | if ( 81 | args.aggregate_on_containerID 82 | ): # show a single record per container 83 | container_aggregate_vec = {} 84 | for PID in process_data_dict: 85 | if PID in container_to_PID_dict: 86 | container_ID = container_to_PID_dict[PID] 87 | container_aggregate_vec[container_ID] = {} 88 | aggregate_vec_dict = container_aggregate_vec[ 89 | container_ID 90 | ] 91 | data_metrics = metrics[core_index + 1 :] 92 | for data_line in process_data_dict[PID]: 93 | line_parts = data_line[core_index + 1 :] 94 | for i, data_point in enumerate(line_parts): 95 | if data_metrics[i] not in aggregate_vec_dict: 96 | aggregate_vec_dict[data_metrics[i]] = [] 97 | 98 | aggregate_vec_dict[data_metrics[i]].append( 99 | float(data_point) 100 | ) 101 | print( 102 | "---------------------------------------------------------------------------------" 103 | ) 104 | header = "Timestamp,containerID," + ",".join( 105 | metrics[core_index + 1 :] 106 | ) 107 | print(header) 108 | if container_to_signature_dict is not None: 109 | if "header" not in container_to_signature_dict: 110 | container_to_signature_dict["header"] = ",".join( 111 | metrics[core_index + 1 :] 112 | ) 113 | if "key" not in container_to_signature_dict: 114 | container_to_signature_dict["key"] = "container_ID" 115 | for container_ID in container_aggregate_vec: 116 | aggregate_vec_dict = container_aggregate_vec[container_ID] 117 | 118 | aggregate_vec = [] 119 | for dm in data_metrics: 120 | if dm in metrics_to_average: 121 | aggregate_vec.append( 122 | str( 123 | round( 124 | sum(aggregate_vec_dict[dm]) 125 | / len(aggregate_vec_dict[dm]), 126 | 2, 127 | ) 128 | ) 129 | ) 130 | else: 131 | aggregate_vec.append( 132 | str(sum(aggregate_vec_dict[dm])) 133 | ) 134 | 135 | t = datetime.now().timestamp() 136 | print( 137 | str(t) 138 | + "," 139 | + container_ID 140 | + "," 141 | + ",".join(aggregate_vec) 142 | ) 143 | if container_to_signature_dict is not None: 144 | if container_ID not in container_to_signature_dict: 145 | container_to_signature_dict[container_ID] = [] 146 | container_to_signature_dict[ 147 | container_ID 148 | ] = container_to_signature_dict[container_ID] + [ 149 | (t, ",".join(aggregate_vec)) 150 | ] 151 | if client is not None: 152 | if seconds_counter >= seconds_to_skip: 153 | send_to_cloud_watch( 154 | client, 155 | ",".join(metrics[core_index + 1 :]), 156 | container_ID, 157 | ",".join(aggregate_vec), 158 | ) 159 | 160 | elif ( 161 | args.aggregate_on_core 162 | ): # Show a single record per ContainerID/Core 163 | container_aggregate_vec = {} 164 | for PID in process_data_dict: 165 | if PID in container_to_PID_dict: 166 | container_ID = container_to_PID_dict[PID] 167 | container_aggregate_vec[container_ID] = {} 168 | aggregate_vec_dict = container_aggregate_vec[ 169 | container_ID 170 | ] 171 | data_metrics = metrics[core_index + 1 :] 172 | for vec in process_data_dict[PID]: 173 | vec_parts = vec[core_index + 1 :] 174 | core = vec[core_index] 175 | if core not in aggregate_vec_dict: 176 | aggregate_vec_dict[core] = {} 177 | 178 | for i, data_point in enumerate(vec_parts): 179 | if ( 180 | data_metrics[i] 181 | not in aggregate_vec_dict[core] 182 | ): 183 | aggregate_vec_dict[core][ 184 | data_metrics[i] 185 | ] = [] 186 | 187 | aggregate_vec_dict[core][ 188 | data_metrics[i] 189 | ].append(float(data_point)) 190 | print( 191 | "---------------------------------------------------------------------------------" 192 | ) 193 | header = "Timestamp,containerID,core," + ",".join( 194 | metrics[core_index + 1 :] 195 | ) 196 | print(header) 197 | if container_to_signature_dict is not None: 198 | if "header" not in container_to_signature_dict: 199 | container_to_signature_dict["header"] = ",".join( 200 | metrics[core_index + 1 :] 201 | ) 202 | if "key" not in container_to_signature_dict: 203 | container_to_signature_dict["key"] = "containerID_core" 204 | for container_ID in container_aggregate_vec: 205 | for core in container_aggregate_vec[container_ID]: 206 | aggregate_vec_dict = container_aggregate_vec[ 207 | container_ID 208 | ][core] 209 | 210 | aggregate_vec = [] 211 | for dm in data_metrics: 212 | if dm in metrics_to_average: 213 | aggregate_vec.append( 214 | str( 215 | sum(aggregate_vec_dict[dm]) 216 | / len(aggregate_vec_dict[dm]) 217 | ) 218 | ) 219 | else: 220 | aggregate_vec.append( 221 | str(sum(aggregate_vec_dict[dm])) 222 | ) 223 | 224 | t = datetime.now().timestamp() 225 | print( 226 | str(t) 227 | + "," 228 | + container_ID 229 | + "," 230 | + str(core) 231 | + "," 232 | + ",".join(aggregate_vec) 233 | ) 234 | key = container_ID + "_" + str(core) 235 | if container_to_signature_dict is not None: 236 | if key not in container_to_signature_dict: 237 | container_to_signature_dict[key] = [] 238 | container_to_signature_dict[ 239 | key 240 | ] = container_to_signature_dict[key] + [ 241 | (t, ",".join(aggregate_vec)) 242 | ] 243 | if client is not None: 244 | if seconds_counter >= seconds_to_skip: 245 | send_to_cloud_watch( 246 | client, 247 | ",".join(metrics[core_index + 1 :]), 248 | key, 249 | ",".join(aggregate_vec), 250 | ) 251 | else: # Show a single record per PID 252 | print( 253 | "---------------------------------------------------------------------------------" 254 | ) 255 | header = "Timestamp,containerID,PID,process," + ",".join( 256 | metrics[process_index + 1 :] 257 | ) 258 | print(header) 259 | if container_to_signature_dict is not None: 260 | if "header" not in container_to_signature_dict: 261 | container_to_signature_dict["header"] = ",".join( 262 | metrics[process_index:] 263 | ) 264 | if "key" not in container_to_signature_dict: 265 | container_to_signature_dict["key"] = "containerID_PID" 266 | 267 | for PID in process_data_dict: 268 | if PID in container_to_PID_dict: 269 | for line in process_data_dict[PID]: 270 | t = datetime.now().timestamp() 271 | print( 272 | str(t) 273 | + "," 274 | + container_to_PID_dict[PID] 275 | + "," 276 | + PID 277 | + "," 278 | + ",".join(line[process_index:]) 279 | ) 280 | key = container_to_PID_dict[PID] + "_" + PID 281 | if container_to_signature_dict is not None: 282 | if key not in container_to_signature_dict: 283 | container_to_signature_dict[key] = [] 284 | container_to_signature_dict[ 285 | key 286 | ] = container_to_signature_dict[key] + [ 287 | (t, ",".join(line[process_index:])) 288 | ] 289 | if client is not None: 290 | if seconds_counter >= seconds_to_skip: 291 | send_to_cloud_watch( 292 | client, 293 | ",".join(metrics[process_index:]), 294 | key, 295 | ",".join(line[process_index:]), 296 | ) 297 | if seconds_counter >= seconds_to_skip: 298 | seconds_counter = 0 299 | finally: 300 | # Releasing the lock 301 | lock.release() 302 | # clear Process data 303 | process_data_dict = {} 304 | 305 | elif line.startswith("**Warning**"): 306 | print(line) 307 | else: 308 | dataVals = line.split(",") 309 | if process_PID_index < len(dataVals): 310 | if dataVals[process_PID_index] not in process_data_dict: 311 | process_data_dict[dataVals[process_PID_index]] = [] 312 | process_data_dict[dataVals[process_PID_index]].append(dataVals) 313 | 314 | except KeyboardInterrupt: 315 | print("Exiting procmon thread") 316 | return 317 | 318 | 319 | def get_process_to_container_mapping(container_to_PID_dict, lock): 320 | while True: 321 | try: 322 | local_container_to_PID_dict = {} 323 | 324 | t0 = time.time() 325 | 326 | p = subprocess.Popen( 327 | ["ps", "-a", "-x", "-o", "pid,cgroup"], 328 | stdout=subprocess.PIPE, 329 | stderr=subprocess.PIPE, 330 | ) 331 | 332 | try: 333 | out, _err = p.communicate() 334 | except SubprocessError as e: 335 | print("Failed to get process to container mapping.", e) 336 | print("Exiting...") 337 | sys.exit() 338 | 339 | t1 = time.time() 340 | diff = t1 - t0 341 | if args.verbose: 342 | print("Sudo ps and grep Latency: ", str(round(diff, 2)), " seconds") 343 | 344 | out_lines = [ 345 | *set( 346 | filter( # remove extraneous lines 347 | lambda x: x != "" 348 | and "CGROUP" not in x 349 | and x != "-" 350 | and ("docker" in x or "containerd" in x or "crio-" in x) 351 | and x.endswith(".scope"), 352 | out.decode("utf-8").split("\n"), 353 | ) 354 | ) 355 | ] 356 | 357 | for line in out_lines: 358 | parts = line.strip().split(" ") 359 | if len(parts) > 1: 360 | cont_short_name = parts[1].split("/")[-1] 361 | local_container_to_PID_dict[parts[0]] = cont_short_name 362 | 363 | t1 = time.time() 364 | diff = t1 - t0 365 | 366 | # Waiting for lock on container_to_PID_dict 367 | lock.acquire() 368 | 369 | try: 370 | # Here we copy the data from the local dictionary to the shared dictionary 371 | container_to_PID_dict.clear() 372 | for k in local_container_to_PID_dict: 373 | container_to_PID_dict[k] = local_container_to_PID_dict[k] 374 | finally: 375 | # Releasing the lock 376 | lock.release() 377 | 378 | sys.stdout.flush() 379 | sleep(1) 380 | if args.verbose: 381 | print("Total API Calls Latency: ", str(round(diff, 2)), " seconds") 382 | # return cont_pids,cont_names 383 | except KeyboardInterrupt: 384 | print("Exiting cid thread") 385 | return 386 | 387 | 388 | def get_args(): 389 | parser = argparse.ArgumentParser( 390 | description="Display procmon data on container level", 391 | formatter_class=argparse.RawDescriptionHelpFormatter, 392 | ) 393 | 394 | parser.add_argument( 395 | "-v", "--verbose", action="store_true", help="show raw verbose logging info." 396 | ) 397 | 398 | parser.add_argument( 399 | "--collect_signatures", 400 | action="store_true", 401 | help="collect signatures of running containers and dump to: signatures_*.csv", 402 | ) 403 | 404 | parser.add_argument( 405 | "-d", 406 | "--duration", 407 | type=int, 408 | default=0, 409 | help="Collection duration in seconds. Default is 0 (indefinitely)", 410 | ) 411 | 412 | group = parser.add_mutually_exclusive_group() 413 | group.add_argument( 414 | "--aggregate_on_core", 415 | action="store_true", 416 | help="Show a single aggregated record for each containerID + core. This option is mutually exclusive with '--aggregate_on_containerID'", 417 | ) 418 | 419 | group.add_argument( 420 | "--aggregate_on_containerID", 421 | action="store_true", 422 | help="Show a single aggregated record for each containerID. This option is mutually exclusive with '--aggregate_on_core'", 423 | ) 424 | 425 | # cloudwatch arguments 426 | parser.add_argument( 427 | "--export_to_cloudwatch", 428 | action="store_true", 429 | help="Export collected data to cloudwatch. Expects the following AWS parameters to be configured in `aws cli`: aws_access_key_id, aws_secret_access_key, aws_region.", 430 | ) 431 | 432 | parser.add_argument( 433 | "--cloudwatch_sampling_duration_in_sec", 434 | type=int, 435 | default=10, 436 | help="Duration between samples of data points sent to cloudwatch. Default is 10 (one sample every 10 seconds). The minimum duration is 1 second. Note: this argument is only effective when --export_to_cloudwatch is set.", 437 | ) 438 | 439 | args = parser.parse_args() 440 | if args.cloudwatch_sampling_duration_in_sec < 1: 441 | parser.error( 442 | "Wrong value error. Minimum value of --cloudwatch_sampling_duration_in_sec is 1 (one sample per second)" 443 | ) 444 | 445 | return args 446 | 447 | 448 | def initialize_cloudwatch_client(): 449 | try: 450 | client = boto3.client("cloudwatch") 451 | except ( 452 | botocore.exceptions.ClientError, 453 | botocore.exceptions.EndpointConnectionError, 454 | ) as err: 455 | print("Failed to export data to cloudwatch. ", str(err)) 456 | sys.exit() 457 | return client 458 | 459 | 460 | def send_to_cloud_watch(client, header, key, line): 461 | header_fields = header.split(",") 462 | values = line.split(",") 463 | metric_data_list = [] 464 | for i, h in enumerate(header_fields): 465 | try: 466 | float_value = float(values[i]) 467 | except ValueError: 468 | continue 469 | key_parts = key.split("_") 470 | metric_data_list.append( 471 | { 472 | "MetricName": str(h), 473 | "Dimensions": [ 474 | {"Name": "ContainerID", "Value": key_parts[0]}, 475 | ], 476 | "Value": float_value, 477 | "Unit": "Count", 478 | } 479 | ) 480 | try: 481 | response = client.put_metric_data( 482 | Namespace="Dockermon", 483 | MetricData=metric_data_list, 484 | ) 485 | except ( 486 | botocore.exceptions.ClientError, 487 | botocore.exceptions.EndpointConnectionError, 488 | ) as err: 489 | print("Failed to export data to cloudwatch. ", str(err)) 490 | sys.exit() 491 | return response 492 | 493 | 494 | def save_signatures_to_CSV(container_to_signature_dict): 495 | if len(container_to_signature_dict) == 2: # header and key only 496 | print("No data to save. Exiting...") 497 | sys.exit() 498 | timestr = time.strftime("%Y-%m-%d_%H-%M-%S") 499 | current_directory = os.getcwd() 500 | folder_name = "cmon_" + timestr 501 | output_directory = os.path.join(current_directory, folder_name) 502 | if os.path.exists(output_directory): 503 | print( 504 | "Cannot create directory", 505 | output_directory, 506 | "directory already exists!\nExiting...", 507 | ) 508 | sys.exit() 509 | try: 510 | os.makedirs(output_directory) 511 | except OSError as error: 512 | print("Output directory can not be created\n", error.message) 513 | sys.exit() 514 | columns = ( 515 | [container_to_signature_dict["key"]] 516 | + ["TimeStamp"] 517 | + container_to_signature_dict["header"].split(",") 518 | ) 519 | data_list = [] 520 | for key in container_to_signature_dict: 521 | if key == "header" or key == "key": 522 | continue 523 | for event_list in container_to_signature_dict[key]: 524 | data_list.append([key] + [event_list[0]] + event_list[1].split(",")) 525 | # prepare dataframe 526 | df = pd.DataFrame(data_list) 527 | df.columns = columns 528 | df = df[ 529 | [container_to_signature_dict["key"]] 530 | + ["TimeStamp"] 531 | + columns[columns.index("cycles") :] 532 | ] 533 | df = df.apply(pd.to_numeric, errors="ignore") 534 | df.round(2).to_csv( 535 | output_directory + "//" + "signatures_time_series.csv", index=False 536 | ) 537 | # prepare mean and max dataframes 538 | df.insert( 539 | loc=1, 540 | column="TimeStamp_Count", 541 | value=df.groupby(container_to_signature_dict["key"])["TimeStamp"].transform( 542 | "nunique" 543 | ), 544 | ) 545 | df = df.drop(["TimeStamp"], axis=1) 546 | mean_df = df.groupby(container_to_signature_dict["key"]).agg("mean") 547 | mean_df.round(2).to_csv(output_directory + "//" + "signatures_mean.csv") 548 | min_df = df.groupby(container_to_signature_dict["key"]).agg("min") 549 | min_df.round(2).to_csv(output_directory + "//" + "signatures_min.csv") 550 | max_df = df.groupby(container_to_signature_dict["key"]).agg("max") 551 | max_df.round(2).to_csv(output_directory + "//" + "signatures_max.csv") 552 | print("Successfully saved signatures to: ", folder_name) 553 | 554 | 555 | if __name__ == "__main__": 556 | args = get_args() 557 | try: 558 | lock = Lock() 559 | manager = Manager() 560 | container_to_PID_dict = manager.dict() 561 | cloudwatch_client = None 562 | if args.export_to_cloudwatch: 563 | cloudwatch_client = initialize_cloudwatch_client() 564 | 565 | if args.collect_signatures: 566 | container_to_signature_dict = manager.dict() 567 | procmon = Process( 568 | target=get_procmon_out, 569 | args=( 570 | container_to_PID_dict, 571 | lock, 572 | container_to_signature_dict, 573 | args.duration, 574 | cloudwatch_client, 575 | args.cloudwatch_sampling_duration_in_sec, 576 | ), 577 | ) 578 | else: 579 | procmon = Process( 580 | target=get_procmon_out, 581 | args=( 582 | container_to_PID_dict, 583 | lock, 584 | None, 585 | args.duration, 586 | cloudwatch_client, 587 | args.cloudwatch_sampling_duration_in_sec, 588 | ), 589 | ) 590 | container = Process( 591 | target=get_process_to_container_mapping, args=(container_to_PID_dict, lock) 592 | ) 593 | 594 | procmon.start() 595 | container.start() 596 | 597 | while procmon.is_alive() and container.is_alive(): 598 | sleep(2) 599 | 600 | # If procmon or container processes are not alive, terminate 601 | procmon.terminate() 602 | container.terminate() 603 | 604 | procmon.join() 605 | container.join() 606 | 607 | global procmon_exit, container_exit 608 | procmon_exit = False 609 | container_exit = False 610 | 611 | except (KeyboardInterrupt, Exception) as e: 612 | print("Exiting Main Thread", e) 613 | if procmon: 614 | procmon.terminate() 615 | if container: 616 | container.terminate() 617 | 618 | if args.collect_signatures and container_to_signature_dict is not None: 619 | save_signatures_to_CSV(container_to_signature_dict) 620 | sys.exit() 621 | -------------------------------------------------------------------------------- /procmon/procmon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ########################################################################################################### 4 | # Copyright (C) 2023 Intel Corporation 5 | # SPDX-License-Identifier: MIT 6 | ########################################################################################################### 7 | 8 | from __future__ import division 9 | from __future__ import print_function 10 | from time import sleep 11 | from datetime import datetime 12 | import argparse 13 | import signal 14 | import os 15 | from struct import unpack 16 | import resource 17 | import sys 18 | import yaml 19 | 20 | try: 21 | from bcc import BPF, Perf, PerfType 22 | except (ModuleNotFoundError, ImportError): 23 | print( 24 | "BCC modules (BPF, Perf, & PerfType) are not installed. Did you compile and build BCC from the source? https://github.com/iovisor/bcc/blob/master/INSTALL.md#source" 25 | ) 26 | sys.exit(1) 27 | 28 | userid = os.geteuid() 29 | if userid != 0: 30 | print( 31 | "Root privileges are needed to run this script.\nPlease try again using 'sudo'. Exiting." 32 | ) 33 | sys.exit(1) 34 | 35 | parser = argparse.ArgumentParser( 36 | description="eBPF based Core metrics by PID", 37 | formatter_class=argparse.RawDescriptionHelpFormatter, 38 | ) 39 | parser.add_argument( 40 | "-f", 41 | "--sample_freq", 42 | type=int, 43 | default=10000000, 44 | help="Sample one in this many number of events", 45 | ) 46 | parser.add_argument("-d", "--duration", type=int, help="duration") 47 | parser.add_argument("-i", "--interval", type=int, default=1, help="interval in seconds") 48 | parser.add_argument( 49 | "--aggregate_cpus", 50 | action="store_true", 51 | help="Aggregate all the counters across CPUs, the cpu field will be set to zero for all PIDs/Containers", 52 | ) 53 | parser.add_argument( 54 | "--aggregate_cgroup", 55 | action="store_true", 56 | help="Aggregate all the counters on cgroup level, every container will then have a single row", 57 | ) 58 | parser.add_argument( 59 | "--acc", 60 | action="store_true", 61 | help="collect events in accumulate mode. If not set, all counter cleared in each round", 62 | ) 63 | parser.add_argument( 64 | "-v", "--verbose", action="store_true", help="show raw counters in every interval" 65 | ) 66 | parser.add_argument("--ebpf", action="store_true", help=argparse.SUPPRESS) 67 | args = parser.parse_args() 68 | 69 | interval = float(args.interval) 70 | duration = args.duration 71 | 72 | # Read events names and codes from events.yaml 73 | events_file_path = "events.yaml" 74 | check_file = os.path.exists(events_file_path) 75 | if not check_file: 76 | print("events.yaml files is not present in the directory. Exiting..") 77 | sys.exit("events.yaml file does not exist!") 78 | 79 | with open(events_file_path) as f: 80 | try: 81 | events = yaml.safe_load(f)["events"] 82 | 83 | except yaml.scanner.ScannerError: 84 | print("Format error in " + events_file_path + ". Exiting..") 85 | sys.exit("Format error") 86 | 87 | 88 | # Increase open file limits 89 | if args.verbose: 90 | print( 91 | "Setting open files limit to 10K. The limit should increase for more cores and/or groups" 92 | ) 93 | 94 | resource.setrlimit(resource.RLIMIT_NOFILE, (10000, 10000)) 95 | 96 | 97 | # Get processor information 98 | def get_cpuinfo(): 99 | cpuinfo = [] 100 | temp_dict = {} 101 | try: 102 | file_getcpuinfo = open("/proc/cpuinfo", "r") 103 | except OSError as err: 104 | print("OS error: {0}".format(err)) 105 | sys.exit("OS error") 106 | else: 107 | for line in file_getcpuinfo: 108 | try: 109 | key, value = list(map(str.strip, line.split(":", 1))) 110 | except ValueError: 111 | cpuinfo.append(temp_dict) 112 | temp_dict = {} 113 | else: 114 | temp_dict[key] = value 115 | file_getcpuinfo.close() 116 | return cpuinfo 117 | 118 | 119 | # Get current architecture and check if it supports OCR counters 120 | def check_OCR_support(): 121 | procinfo_arch = get_cpuinfo() 122 | # check procinfo_arch is not empty and contains model, cpu family, and stepping 123 | assert len(procinfo_arch) > 0 124 | assert "model" in procinfo_arch[0] 125 | assert "cpu family" in procinfo_arch[0] 126 | assert "stepping" in procinfo_arch[0] 127 | 128 | model = int(procinfo_arch[0]["model"].strip()) 129 | cpufamily = int(procinfo_arch[0]["cpu family"].strip()) 130 | stepping = int(procinfo_arch[0]["stepping"].strip()) 131 | if model == 106 and cpufamily == 6 and stepping >= 0: 132 | current_arch = "icelake" 133 | elif model == 143 and cpufamily == 6 and stepping >= 3: 134 | current_arch = "sapphirerapid" 135 | elif model == 85 and cpufamily == 6 and (4 <= stepping < 10): 136 | current_arch = "cascadelake/skylake" 137 | else: 138 | print( 139 | "Current architecture does not support OCR counters. OCR counters are supported in Cascadelake/Icelake/SapphireRapid architectures." 140 | ) 141 | # if system does not support OCR counters, call sys.exit() 142 | sys.exit("System not supported") 143 | 144 | print("Architecture: ", current_arch.upper(), " has OCR support!") 145 | return current_arch, True 146 | 147 | 148 | # load BPF program 149 | bpf_text = """ 150 | #include 151 | #include 152 | #include 153 | #include 154 | #include 155 | #include 156 | #include 157 | #include 158 | #include 159 | 160 | struct key_t { 161 | u64 pid; 162 | char name[TASK_COMM_LEN]; 163 | int cgroupid; 164 | }; 165 | 166 | BPF_PERCPU_HASH(ref_count, struct key_t); 167 | BPF_PERCPU_HASH(cycles_count, struct key_t); 168 | BPF_PERCPU_HASH(insts_count, struct key_t); 169 | BPF_PERCPU_HASH(l1imiss_count, struct key_t); 170 | BPF_PERCPU_HASH(l1dmiss_count, struct key_t); 171 | BPF_PERCPU_HASH(l1dhit_count, struct key_t); 172 | BPF_PERCPU_HASH(l2miss_count, struct key_t); 173 | BPF_PERCPU_HASH(l3miss_count, struct key_t); 174 | BPF_PERCPU_HASH(ocr_ev1_count, struct key_t); 175 | BPF_PERCPU_HASH(ocr_ev2_count, struct key_t); 176 | BPF_PERCPU_HASH(ocr_ev3_count, struct key_t); 177 | BPF_PERCPU_HASH(ocr_ev4_count, struct key_t); 178 | BPF_PERCPU_HASH(disk_io_R_count, struct key_t); 179 | BPF_PERCPU_HASH(disk_io_W_count, struct key_t); 180 | BPF_PERCPU_HASH(ipv4_send_bytes, struct key_t); 181 | BPF_PERCPU_HASH(ipv4_recv_bytes, struct key_t); 182 | BPF_PERCPU_HASH(sock_store, u32, struct sock *); 183 | BPF_PERCPU_HASH(qlen_sum, struct key_t); 184 | BPF_PERCPU_HASH(qlen_count, struct key_t); 185 | BPF_PERCPU_HASH(qlat_accum, struct key_t); 186 | BPF_PERCPU_HASH(mem_sizes, struct key_t); 187 | BPF_PERCPU_HASH(memptrs, u64, u64); 188 | BPF_PERCPU_HASH(start_runq, u64); 189 | 190 | static inline __attribute__((always_inline)) void get_key(struct key_t* key) { 191 | if(!GROUPID_FILTER) 192 | { 193 | key->pid = bpf_get_current_pid_tgid() >> 32; 194 | bpf_get_current_comm(&(key->name), sizeof(key->name)); 195 | } 196 | else{ 197 | key->pid = bpf_get_current_cgroup_id(); 198 | } 199 | } 200 | int on_ref(struct bpf_perf_event_data *ctx) { 201 | struct key_t key = {}; 202 | get_key(&key); 203 | if(key.pid > 0) { 204 | u64 *val; 205 | u64 sample_period = (u64)ctx->sample_period; 206 | val = ref_count.lookup_or_try_init(&key, &sample_period); 207 | if (val) { 208 | *val += sample_period; 209 | } 210 | } 211 | return 0; 212 | } 213 | int on_cycles(struct bpf_perf_event_data *ctx) { 214 | struct key_t key = {}; 215 | get_key(&key); 216 | if(key.pid > 0) { 217 | u64 *val; 218 | u64 sample_period = (u64)ctx->sample_period; 219 | val = cycles_count.lookup_or_try_init(&key, &sample_period); 220 | if (val) { 221 | *val += sample_period; 222 | } 223 | } 224 | return 0; 225 | } 226 | int on_insts(struct bpf_perf_event_data *ctx) { 227 | struct key_t key = {}; 228 | get_key(&key); 229 | if(key.pid > 0) { 230 | u64 *val; 231 | u64 sample_period = (u64)ctx->sample_period; 232 | val = insts_count.lookup_or_try_init(&key, &sample_period); 233 | if (val) { 234 | *val += sample_period; 235 | } 236 | } 237 | return 0; 238 | } 239 | int on_l1imiss(struct bpf_perf_event_data *ctx) { 240 | struct key_t key = {}; 241 | get_key(&key); 242 | if(key.pid > 0) { 243 | u64 *val; 244 | u64 sample_period = (u64)ctx->sample_period; 245 | val = l1imiss_count.lookup_or_try_init(&key, &sample_period); 246 | if (val) { 247 | *val += sample_period; 248 | } 249 | } 250 | return 0; 251 | } 252 | int on_l1dmiss(struct bpf_perf_event_data *ctx) { 253 | struct key_t key = {}; 254 | get_key(&key); 255 | if(key.pid > 0) { 256 | u64 *val; 257 | u64 sample_period = (u64)ctx->sample_period; 258 | val = l1dmiss_count.lookup_or_try_init(&key, &sample_period); 259 | if (val) { 260 | *val += sample_period; 261 | } 262 | } 263 | return 0; 264 | } 265 | int on_l1dhit(struct bpf_perf_event_data *ctx) { 266 | struct key_t key = {}; 267 | get_key(&key); 268 | if(key.pid > 0) { 269 | u64 *val; 270 | u64 sample_period = (u64)ctx->sample_period; 271 | val = l1dhit_count.lookup_or_try_init(&key, &sample_period); 272 | if (val) { 273 | *val += sample_period; 274 | } 275 | } 276 | return 0; 277 | } 278 | int on_l2miss(struct bpf_perf_event_data *ctx) { 279 | struct key_t key = {}; 280 | get_key(&key); 281 | if(key.pid > 0) { 282 | u64 *val; 283 | u64 sample_period = (u64)ctx->sample_period; 284 | val = l2miss_count.lookup_or_try_init(&key, &sample_period); 285 | if (val) { 286 | *val += sample_period; 287 | } 288 | } 289 | return 0; 290 | } 291 | int on_l3miss(struct bpf_perf_event_data *ctx) { 292 | struct key_t key = {}; 293 | get_key(&key); 294 | if(key.pid > 0) { 295 | u64 *val; 296 | u64 sample_period = (u64)ctx->sample_period; 297 | val = l3miss_count.lookup_or_try_init(&key, &sample_period); 298 | if (val) { 299 | *val += sample_period; 300 | } 301 | } 302 | return 0; 303 | } 304 | int on_ocr_ev1(struct bpf_perf_event_data *ctx) { 305 | struct key_t key = {}; 306 | get_key(&key); 307 | if(key.pid > 0) { 308 | u64 *val; 309 | u64 sample_period = (u64)ctx->sample_period; 310 | val = ocr_ev1_count.lookup_or_try_init(&key, &sample_period); 311 | if (val) { 312 | *val += sample_period; 313 | } 314 | } 315 | return 0; 316 | } 317 | int on_ocr_ev2(struct bpf_perf_event_data *ctx) { 318 | struct key_t key = {}; 319 | get_key(&key); 320 | if(key.pid > 0) { 321 | u64 *val; 322 | u64 sample_period = (u64)ctx->sample_period; 323 | val = ocr_ev2_count.lookup_or_try_init(&key, &sample_period); 324 | if (val) { 325 | *val += sample_period; 326 | } 327 | } 328 | return 0; 329 | } 330 | int on_ocr_ev3(struct bpf_perf_event_data *ctx) { 331 | struct key_t key = {}; 332 | get_key(&key); 333 | if(key.pid > 0) { 334 | u64 *val; 335 | u64 sample_period = (u64)ctx->sample_period; 336 | val = ocr_ev3_count.lookup_or_try_init(&key, &sample_period); 337 | if (val) { 338 | *val += sample_period; 339 | } 340 | } 341 | return 0; 342 | } 343 | int on_ocr_ev4(struct bpf_perf_event_data *ctx) { 344 | struct key_t key = {}; 345 | get_key(&key); 346 | if(key.pid > 0) { 347 | u64 *val; 348 | u64 sample_period = (u64)ctx->sample_period; 349 | val = ocr_ev4_count.lookup_or_try_init(&key, &sample_period); 350 | if (val) { 351 | *val += sample_period; 352 | } 353 | } 354 | return 0; 355 | } 356 | // Trace point for block io issue events 357 | // We can break this down further by the device using args->device parameter 358 | TRACEPOINT_PROBE(block, block_rq_issue) { 359 | struct key_t key = {}; 360 | get_key(&key); 361 | char R_W = args->rwbs[0]; 362 | u64 *val; 363 | if(R_W == 'R'){ 364 | u64 bytes = (u64)args->bytes; 365 | val = disk_io_R_count.lookup_or_try_init(&key, &bytes); 366 | if (val) { 367 | *val += args->bytes; 368 | } 369 | } 370 | else { 371 | u64 bytes = (u64)args->bytes; 372 | val = disk_io_W_count.lookup_or_try_init(&key, &bytes); 373 | if (val) { 374 | *val += args->bytes; 375 | } 376 | } 377 | return 0; 378 | } 379 | // Code for tracing network throughput: Adopted from https://github.com/iovisor/bcc/blob/master/tools/tcptop.py 380 | static int tcp_sendstat(int size) 381 | { 382 | u32 pid = bpf_get_current_pid_tgid() >> 32; 383 | u32 tid = bpf_get_current_pid_tgid(); 384 | struct sock **sockpp; 385 | sockpp = sock_store.lookup(&tid); 386 | if (sockpp == 0) { 387 | return 0; //miss the entry 388 | } 389 | struct sock *sk = *sockpp; 390 | u16 dport = 0, family; 391 | bpf_probe_read_kernel(&family, sizeof(family), 392 | &sk->__sk_common.skc_family); 393 | 394 | if (family == AF_INET || family == AF_INET6) { 395 | struct key_t key = {}; 396 | get_key(&key) ; 397 | u64 *val; 398 | u64 size_u64 = (u64)size; 399 | val = ipv4_send_bytes.lookup_or_try_init(&key, &size_u64); 400 | if (val) { 401 | *val += size; 402 | } 403 | } 404 | 405 | sock_store.delete(&tid); 406 | 407 | return 0; 408 | } 409 | int kretprobe__tcp_sendmsg(struct pt_regs *ctx) 410 | { 411 | int size = PT_REGS_RC(ctx); 412 | if (size > 0) 413 | return tcp_sendstat(size); 414 | else 415 | return 0; 416 | } 417 | int kretprobe__tcp_sendpage(struct pt_regs *ctx) 418 | { 419 | int size = PT_REGS_RC(ctx); 420 | if (size > 0) 421 | return tcp_sendstat(size); 422 | else 423 | return 0; 424 | } 425 | static int tcp_send_entry(struct sock *sk) 426 | { 427 | u32 pid = bpf_get_current_pid_tgid() >> 32; 428 | //FILTER_PID 429 | u32 tid = bpf_get_current_pid_tgid(); 430 | u16 family = sk->__sk_common.skc_family; 431 | struct sock **val; 432 | val = sock_store.lookup_or_try_init(&tid, &sk); 433 | if (val) { 434 | *val = sk; 435 | } 436 | return 0; 437 | } 438 | int kprobe__tcp_sendmsg(struct pt_regs *ctx, struct sock *sk, 439 | struct msghdr *msg, size_t size) 440 | { 441 | return tcp_send_entry(sk); 442 | } 443 | int kprobe__tcp_sendpage(struct pt_regs *ctx, struct sock *sk, 444 | struct page *page, int offset, size_t size) 445 | { 446 | return tcp_send_entry(sk); 447 | } 448 | /* 449 | * tcp_recvmsg() would be obvious to trace, but is less suitable because: 450 | * - we'd need to trace both entry and return, to have both sock and size 451 | * - misses tcp_read_sock() traffic 452 | * we'd much prefer tracepoints once they are available. 453 | */ 454 | int kprobe__tcp_cleanup_rbuf(struct pt_regs *ctx, struct sock *sk, int copied) 455 | { 456 | u32 pid = bpf_get_current_pid_tgid() >> 32; 457 | u16 dport = 0, family = sk->__sk_common.skc_family; 458 | u64 *val, zero = 0; 459 | if (copied <= 0) 460 | return 0; 461 | 462 | if (family == AF_INET || family == AF_INET6) { 463 | struct key_t key = {}; 464 | get_key(&key); 465 | u64 *val; 466 | u64 copied_u64 = (u64)copied; 467 | val = ipv4_recv_bytes.lookup_or_try_init(&key, &copied_u64); 468 | if (val) { 469 | *val += copied; 470 | } 471 | } 472 | return 0; 473 | } 474 | 475 | /// code for tracing queue latency and length per PID 476 | static int trace_enqueue(struct key_t key) 477 | { 478 | u64 ts = bpf_ktime_get_ns(); 479 | ts /= 1000000; //msec 480 | u64 *val; 481 | val = start_runq.lookup_or_try_init(&key.pid, &ts); 482 | if (val) { 483 | *val = ts; 484 | } 485 | return 0; 486 | } 487 | RAW_TRACEPOINT_PROBE(sched_wakeup) 488 | { 489 | struct key_t key = {}; 490 | get_key(&key); 491 | return trace_enqueue(key); 492 | } 493 | RAW_TRACEPOINT_PROBE(sched_wakeup_new) 494 | { 495 | struct key_t key = {}; 496 | get_key(&key); 497 | return trace_enqueue(key); 498 | } 499 | struct cfs_rq_partial { 500 | struct load_weight load; 501 | unsigned int nr_running, h_nr_running; 502 | }; 503 | RAW_TRACEPOINT_PROBE(sched_switch) 504 | { 505 | struct task_struct *prev = (struct task_struct *)ctx->args[1]; 506 | struct task_struct *next= (struct task_struct *)ctx->args[2]; 507 | s32 prev_tgid, next_tgid; 508 | 509 | bpf_probe_read_kernel(&prev_tgid, sizeof(prev->tgid), &prev->tgid); 510 | bpf_probe_read_kernel(&next_tgid, sizeof(next->tgid), &next->tgid); 511 | 512 | u32 prev_pid = prev_tgid; 513 | u32 next_pid = next_tgid; 514 | // skip swapper or sched (pid=0) 515 | if (prev_pid == 0 || next_pid == 0) 516 | { 517 | return 0; 518 | } 519 | struct key_t key = {}; 520 | get_key(&key); 521 | 522 | // ivcsw: treat like an enqueue event and store timestamp 523 | if (prev->STATE_FIELD == TASK_RUNNING) { 524 | if (!(prev_pid == 0)) { 525 | key.pid = prev_pid; 526 | trace_enqueue(key); 527 | } 528 | } 529 | struct key_t key_next = {}; 530 | get_key(&key_next); 531 | key_next.pid = next_pid; 532 | 533 | u64 *tsp, delta; 534 | u64 prev_tsp; 535 | u64 ts_zero = 0; 536 | tsp = start_runq.lookup_or_try_init(&key_next.pid, &ts_zero); 537 | if(tsp) { 538 | if(*tsp ==0){ 539 | return 0; 540 | } 541 | prev_tsp = *tsp; 542 | *tsp = ts_zero; 543 | } 544 | else{ 545 | return 0; //missed enqueue 546 | } 547 | struct cfs_rq_partial *my_q = NULL; 548 | my_q = (struct cfs_rq_partial *)next->se.cfs_rq; 549 | 550 | unsigned int len = 0; 551 | len = my_q->nr_running; 552 | 553 | delta = bpf_ktime_get_ns(); 554 | delta /= 1000000; // msecs 555 | delta = delta - prev_tsp; 556 | 557 | u64 *val; 558 | val = qlat_accum.lookup_or_try_init(&key, &ts_zero); 559 | if (val) { 560 | *val += delta; 561 | } 562 | 563 | u64 len_u64 = (u64)len; 564 | val = qlen_sum.lookup_or_try_init(&key, &len_u64); 565 | if (val) { 566 | *val += len; 567 | } 568 | u64 step = 1; 569 | val = qlen_count.lookup_or_try_init(&key, &step); 570 | if (val) { 571 | *val += 1; 572 | } 573 | 574 | return 0; 575 | } 576 | """ 577 | # eBPF program code substitutions 578 | if BPF.kernel_struct_has_field(b"task_struct", b"__state") == 1: 579 | bpf_text = bpf_text.replace("STATE_FIELD", "__state") 580 | else: 581 | bpf_text = bpf_text.replace("STATE_FIELD", "state") 582 | 583 | if args.aggregate_cgroup: 584 | bpf_text = bpf_text.replace("GROUPID_FILTER", "1") 585 | else: 586 | bpf_text = bpf_text.replace("GROUPID_FILTER", "0") 587 | 588 | if args.ebpf: 589 | print(bpf_text) 590 | sys.exit("print bpf text") 591 | 592 | # Create BPF object 593 | try: 594 | b = BPF(text=bpf_text) 595 | except Exception as e: 596 | print("Failed to load bpf program.", e) 597 | sys.exit(1) 598 | 599 | 600 | def create_group_and_attach(events_list, gname, sample_freq): 601 | leader = None 602 | count = 0 603 | try: 604 | for hexcode, fn_name, config1 in events_list: 605 | count += 1 606 | attr = Perf.perf_event_attr() 607 | attr.type = PerfType.RAW 608 | attr.config = int(hexcode, 16) 609 | attr.sample_period = sample_freq 610 | attr.sample_type = int("10000", 16) 611 | attr.read_format = int( 612 | "f", 16 613 | ) # TOTAL_TIME_ENABLED|TOTAL_TIME_RUNNING|ID|GROUP 614 | if len(config1) > 0: 615 | attr._bp_addr_union.config1 = int(config1, 16) 616 | if count == 1: 617 | attr.disabled = 1 618 | b.attach_perf_event_raw(attr=attr, fn_name=fn_name, pid=-1, cpu=-1) 619 | leader = b.open_perf_events[(attr.type, attr.config)] 620 | else: 621 | attr.disabled = 0 622 | for c in leader.keys(): 623 | b.attach_perf_event_raw( 624 | attr=attr, 625 | fn_name=fn_name, 626 | pid=-1, 627 | cpu=c, 628 | group_fd=leader[c], 629 | ) 630 | except Exception as e: 631 | print( 632 | "**Warning**: Failed to attach to events in " 633 | + gname 634 | + ". Is this a virtual machine?", 635 | e, 636 | " -> Skipping...", 637 | ) 638 | 639 | if args.verbose: 640 | print("Events in " + gname + " created and attached successfully") 641 | 642 | return leader, len(events_list) 643 | 644 | 645 | # Check if OCR counters are supported by the current SKU 646 | current_arch, OCR_support = check_OCR_support() 647 | 648 | # Create 1st group and attach 649 | fo_s = [] # file objects list 650 | group_sizes = [] 651 | fd, g_s = create_group_and_attach( 652 | [ 653 | (events[current_arch]["refs"], "on_ref", ""), 654 | (events[current_arch]["cycles"], "on_cycles", ""), 655 | (events[current_arch]["insts"], "on_insts", ""), 656 | (events[current_arch]["l1imiss"], "on_l1imiss", ""), 657 | (events[current_arch]["l1dmiss"], "on_l1dmiss", ""), 658 | (events[current_arch]["l1dhit"], "on_l1dhit", ""), 659 | ], 660 | "L1_cache_group", 661 | args.sample_freq, 662 | ) 663 | 664 | # Open fd and add file object to file objects list 665 | fd_id = fd[0] 666 | fo_s.append(os.fdopen(fd_id, "rb", encoding=None)) 667 | group_sizes.append(g_s) 668 | 669 | 670 | # Create 2nd group and attach 671 | fd, g_s = create_group_and_attach( 672 | [ 673 | (events[current_arch]["refs"], "on_ref", ""), 674 | (events[current_arch]["cycles"], "on_cycles", ""), 675 | (events[current_arch]["insts"], "on_insts", ""), 676 | (events[current_arch]["l2miss"], "on_l2miss", ""), 677 | (events[current_arch]["l3miss"], "on_l3miss", ""), 678 | ], 679 | "L2_L3_cache_group", 680 | args.sample_freq, 681 | ) 682 | 683 | # Open fd and add file object to file objects list 684 | fd_id = fd[0] 685 | fo_s.append(os.fdopen(fd_id, "rb", encoding=None)) 686 | group_sizes.append(g_s) 687 | 688 | if OCR_support: 689 | OCR_Local = [ 690 | (events[current_arch]["refs"], "on_ref", ""), 691 | (events[current_arch]["cycles"], "on_cycles", ""), 692 | (events[current_arch]["insts"], "on_insts", ""), 693 | ] 694 | 695 | OCR_Remote = OCR_Local.copy() 696 | 697 | if "ocr_ev1" in events[current_arch]: 698 | event_config_pair = events[current_arch]["ocr_ev1"].split(",") 699 | OCR_Local.append((event_config_pair[0], "on_ocr_ev1", event_config_pair[1])) 700 | 701 | if "ocr_ev2" in events[current_arch]: 702 | event_config_pair = events[current_arch]["ocr_ev2"].split(",") 703 | OCR_Local.append((event_config_pair[0], "on_ocr_ev2", event_config_pair[1])) 704 | 705 | if "ocr_ev3" in events[current_arch]: 706 | event_config_pair = events[current_arch]["ocr_ev3"].split(",") 707 | OCR_Remote.append((event_config_pair[0], "on_ocr_ev3", event_config_pair[1])) 708 | 709 | if "ocr_ev4" in events[current_arch]: 710 | event_config_pair = events[current_arch]["ocr_ev4"].split(",") 711 | OCR_Remote.append((event_config_pair[0], "on_ocr_ev4", event_config_pair[1])) 712 | 713 | # Attach OCR_L1 events 714 | fd, g_s = create_group_and_attach(OCR_Local, "ocr_local_group", args.sample_freq) 715 | 716 | fd_id = fd[0] 717 | # Open fd and add file object to file objects list 718 | fo_s.append(os.fdopen(fd_id, "rb", encoding=None)) 719 | group_sizes.append(g_s) 720 | 721 | # Attach OCR_L2 events 722 | fd, g_s = create_group_and_attach(OCR_Remote, "ocr_remote_group", args.sample_freq) 723 | 724 | fd_id = fd[0] 725 | # Open fd and add file object to file objects list 726 | fo_s.append( 727 | os.fdopen(fd_id, "rb", encoding=None) 728 | ) # open fd and add file object to file objects list 729 | group_sizes.append(g_s) 730 | 731 | 732 | # Collect events counters 733 | def group1_collect(ebpf_counters): 734 | # Calculate scale factor 735 | # We extract the scale factor from group 0, we assume the scale factor is identical across groups 736 | scale = 1 737 | if len(fo_s) > 0 and len(group_sizes) > 0: 738 | group_size = group_sizes[0] 739 | num_of_fields = ( 740 | 3 + group_size * 2 741 | ) # we have three u64 (8 Bytes) fields (nr, time_enabled, time_running) + two u64 fields for each counter (value, id) 742 | rf_data = fo_s[0].read(8 * num_of_fields) # Each field is of type u64 (8 Bytes) 743 | tup = unpack( 744 | "q" * num_of_fields, rf_data 745 | ) # q represents c type long long (https://docs.python.org/3/library/struct.html) 746 | # tup[0]: The number of events 747 | # tup[1]: has time_enabled 748 | # tup[2]: has time_running 749 | if args.verbose: 750 | print(tup) 751 | if tup[1] == tup[2] or tup[2] == 0: 752 | scale = 1 753 | else: 754 | scale = round(tup[1] / (tup[2]), 2) 755 | if args.verbose: 756 | print("SCALE: " + str(scale)) 757 | # *** Notice that we don't scale ref, cycles, or insts as they are counted in both groups 758 | 759 | global_dict = {} 760 | for i in range(len(ebpf_counters)): 761 | if ebpf_counters[i] not in global_dict: 762 | global_dict[ebpf_counters[i]] = {} 763 | 764 | if args.acc: 765 | for k, per_cpu_array in b[ebpf_counters[i]].items(): 766 | for cpu_id, value in enumerate(per_cpu_array): 767 | if ebpf_counters[i] in [ 768 | "l1dmiss_count", 769 | "l1imiss_count", 770 | "l2miss_count", 771 | "l3miss_count", 772 | "ocr_ev1_count", 773 | "ocr_ev2_count", 774 | "ocr_ev3_count", 775 | "ocr_ev4_count", 776 | ]: 777 | value = value * scale 778 | global_dict[ebpf_counters[i]][ 779 | (k.pid, k.name, cpu_id, k.cgroupid) 780 | ] = value 781 | else: 782 | for k, per_cpu_array in b[ebpf_counters[i]].items(): 783 | for cpu_id, value in enumerate(per_cpu_array): 784 | if ebpf_counters[i] in [ 785 | "l1dmiss_count", 786 | "l1imiss_count", 787 | "l2miss_count", 788 | "l3miss_count", 789 | "ocr_ev1_count", 790 | "ocr_ev2_count", 791 | "ocr_ev3_count", 792 | "ocr_ev4_count", 793 | ]: 794 | value = value * scale 795 | global_dict[ebpf_counters[i]][ 796 | (k.pid, k.name, cpu_id, k.cgroupid) 797 | ] = value 798 | b[ebpf_counters[i]].clear() 799 | if args.verbose: 800 | print("CYCLES:", len(global_dict["cycles_count"]), global_dict["cycles_count"]) 801 | print( 802 | "INSTRUCTIONS:", len(global_dict["insts_count"]), global_dict["insts_count"] 803 | ) 804 | print("REFEREENCES:", len(global_dict["ref_count"]), global_dict["ref_count"]) 805 | print( 806 | "L1DMiss:", len(global_dict["l1dmiss_count"]), global_dict["l1dmiss_count"] 807 | ) 808 | print("L1DHit:", len(global_dict["l1dhit_count"]), global_dict["l1dhit_count"]) 809 | print( 810 | "L1IMiss:", len(global_dict["l1imiss_count"]), global_dict["l1imiss_count"] 811 | ) 812 | print("L2Miss:", len(global_dict["l2miss_count"]), global_dict["l2miss_count"]) 813 | print("L3Miss:", len(global_dict["l3miss_count"]), global_dict["l3miss_count"]) 814 | print( 815 | "OCR_EV1:", len(global_dict["ocr_ev1_count"]), global_dict["ocr_ev1_count"] 816 | ) 817 | print( 818 | "OCR_EV2:", len(global_dict["ocr_ev2_count"]), global_dict["ocr_ev2_count"] 819 | ) 820 | print( 821 | "OCR_EV3:", len(global_dict["ocr_ev3_count"]), global_dict["ocr_ev3_count"] 822 | ) 823 | print( 824 | "OCR_EV4:", len(global_dict["ocr_ev4_count"]), global_dict["ocr_ev4_count"] 825 | ) 826 | print( 827 | "DISK_READS:", 828 | len(global_dict["disk_io_R_count"]), 829 | global_dict["disk_io_R_count"], 830 | ) 831 | print( 832 | "DISK_WRITES:", 833 | len(global_dict["disk_io_W_count"]), 834 | global_dict["disk_io_W_count"], 835 | ) 836 | print( 837 | "NETWORK_TX:", 838 | len(global_dict["ipv4_send_bytes"]), 839 | global_dict["ipv4_send_bytes"], 840 | ) 841 | print( 842 | "NETWORK_RX:", 843 | len(global_dict["ipv4_recv_bytes"]), 844 | global_dict["ipv4_recv_bytes"], 845 | ) 846 | print("MEMORY_SIZES:", len(global_dict["mem_sizes"]), global_dict["mem_sizes"]) 847 | print("___________________________________________") 848 | 849 | print( 850 | "Timestamp,PID,process,cgroupID,core,cycles,insts,cpi,l1i_mpi,l1d_hit_ratio,l1d_miss_ratio,l2_miss_ratio,l3_miss_ratio,local_bw,remote_bw,disk_reads,disk_writes,network_tx,network_rx,scheduled_count,avg_q_len,avg_q_latency" 851 | ) 852 | inst = 0 853 | cycles = 0 854 | l1d_miss = 0 855 | l1d_hit = 0 856 | l1i_miss = 0 857 | l2_miss = 0 858 | l3_miss = 0 859 | ocr_ev1 = 0 860 | ocr_ev2 = 0 861 | ocr_ev3 = 0 862 | ocr_ev4 = 0 863 | disk_reads = 0 864 | disk_writes = 0 865 | qlen_sum = 0 866 | qlen_count = 0 867 | qlat_accum = 0 868 | # *** Notice that we scale the counters to account for multiplixing: https://perf.wiki.kernel.org/index.php/Tutorial#multiplexing_and_scaling_events (see multiplexing and scaling events section) 869 | # *** We don't scale ref, cycles, or insts as they are counted in all groups 870 | # *** We also don't scale disk or network counters as they are counted in kernel tracepoints (no multiplixing) 871 | for k, v in global_dict["insts_count"].items(): 872 | if "insts_count" in global_dict: 873 | inst = global_dict["insts_count"].get(k, 0) 874 | if inst > 0: 875 | if "cycles_count" in global_dict: 876 | cycles = global_dict["cycles_count"].get(k, 0) 877 | if "l1imiss_count" in global_dict: 878 | l1i_miss = global_dict["l1imiss_count"].get(k, 0) 879 | if "l1dmiss_count" in global_dict: 880 | l1d_miss = global_dict["l1dmiss_count"].get(k, 0) 881 | if "l1dhit_count" in global_dict: 882 | l1d_hit = global_dict["l1dhit_count"].get(k, 0) 883 | if "l2miss_count" in global_dict: 884 | l2_miss = global_dict["l2miss_count"].get(k, 0) 885 | if "l3miss_count" in global_dict: 886 | l3_miss = global_dict["l3miss_count"].get(k, 0) 887 | if "ocr_ev1_count" in global_dict: 888 | ocr_ev1 = global_dict["ocr_ev1_count"].get(k, 0) 889 | if "ocr_ev2_count" in global_dict: 890 | ocr_ev2 = global_dict["ocr_ev2_count"].get(k, 0) 891 | if "ocr_ev3_count" in global_dict: 892 | ocr_ev3 = global_dict["ocr_ev3_count"].get(k, 0) 893 | if "ocr_ev4_count" in global_dict: 894 | ocr_ev4 = global_dict["ocr_ev4_count"].get(k, 0) 895 | if "disk_io_R_count" in global_dict: 896 | disk_reads = ( 897 | global_dict["disk_io_R_count"].get(k, 0) / 1000000 898 | ) # MB/sec 899 | if "disk_io_W_count" in global_dict: 900 | disk_writes = ( 901 | global_dict["disk_io_W_count"].get(k, 0) / 1000000 902 | ) # MB/sec 903 | if "ipv4_send_bytes" in global_dict: 904 | network_TX = ( 905 | global_dict["ipv4_send_bytes"].get(k, 0) / 1000000 906 | ) # MB/sec 907 | if "ipv4_recv_bytes" in global_dict: 908 | network_RX = ( 909 | global_dict["ipv4_recv_bytes"].get(k, 0) / 1000000 910 | ) # MB/sec 911 | if "qlen_sum" in global_dict: 912 | qlen_sum = global_dict["qlen_sum"].get(k, 0) 913 | if "qlen_count" in global_dict: 914 | qlen_count = global_dict["qlen_count"].get(k, 0) 915 | if "qlat_accum" in global_dict: 916 | qlat_accum = global_dict["qlat_accum"].get(k, 0) 917 | # we add 1 to the denominator for every metric below to avoid division by zero 918 | cpi = cycles / (inst + 1) 919 | l1i_mpi = l1i_miss / (inst + 1) 920 | l1d_miss_ratio = min(1, l1d_miss / (l1d_miss + l1d_hit + 1)) 921 | l1d_hit_ratio = min(1, l1d_hit / (l1d_miss + l1d_hit + 1)) 922 | l2_miss_ratio = min(1, l2_miss / (l1d_miss + 1)) 923 | l3_miss_ratio = min(1, l3_miss / (l2_miss + 1)) 924 | local_bw = (ocr_ev1 + ocr_ev2) * 64 / 1000000 # MB/sec 925 | remote_bw = (ocr_ev3 + ocr_ev4) * 64 / 1000000 # MB/sec 926 | 927 | if qlen_count == 0: 928 | qlen_avg = 0 929 | else: 930 | qlen_avg = int(qlen_sum / qlen_count) 931 | # if inst = 0, set all counters to 0 932 | else: 933 | cpi = 0 934 | l1i_mpi = 0 935 | l1d_miss_ratio = 0 936 | l1d_hit_ratio = 0 937 | l2_miss_ratio = 0 938 | l3_miss_ratio = 0 939 | local_bw = 0 940 | remote_bw = 0 941 | qlen_avg = 0 942 | qlat_accum = 0 943 | t = datetime.now().timestamp() 944 | if cycles > 0 and inst > 0: 945 | print( 946 | "{},{:d},{:s},{:d},{:d},{:d},{:d},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f},{:0.2f}".format( 947 | t, 948 | k[0], 949 | k[1].decode("utf-8", "replace"), 950 | k[3], 951 | k[2], 952 | cycles, 953 | inst, 954 | cpi, 955 | l1i_mpi, 956 | l1d_hit_ratio, 957 | l1d_miss_ratio, 958 | l2_miss_ratio, 959 | l3_miss_ratio, 960 | local_bw, 961 | remote_bw, 962 | disk_reads, 963 | disk_writes, 964 | network_TX, 965 | network_RX, 966 | qlen_count, 967 | qlen_avg, 968 | qlat_accum, 969 | ) 970 | ) 971 | 972 | 973 | # Function below is not needed when items_lookup_and_delete_batch() is used, since it reads the items and clears the content in one syscall 974 | def clear_nonbatch_counters(): 975 | b["start_runq"].clear() 976 | 977 | 978 | ebpf_counters = [ 979 | "insts_count", 980 | "cycles_count", 981 | "ref_count", 982 | "l1imiss_count", 983 | "l1dmiss_count", 984 | "l1dhit_count", 985 | "l2miss_count", 986 | "l3miss_count", 987 | "ocr_ev1_count", 988 | "ocr_ev2_count", 989 | "ocr_ev3_count", 990 | "ocr_ev4_count", 991 | "disk_io_R_count", 992 | "disk_io_W_count", 993 | "ipv4_send_bytes", 994 | "ipv4_recv_bytes", 995 | "qlen_sum", 996 | "qlen_count", 997 | "qlat_accum", 998 | "mem_sizes", 999 | ] 1000 | 1001 | exiting = 0 1002 | seconds = 0 1003 | while 1: 1004 | try: 1005 | sys.stdout.flush() 1006 | sleep(interval) 1007 | seconds += interval 1008 | group1_collect(ebpf_counters) 1009 | 1010 | if duration and seconds >= duration: 1011 | exiting = 1 1012 | 1013 | print( 1014 | "---------------------------------------------------------------------------------" 1015 | ) 1016 | clear_nonbatch_counters() 1017 | except KeyboardInterrupt: 1018 | exiting = 1 1019 | signal.signal(signal.SIGINT, lambda signal, frame: print()) 1020 | if exiting: 1021 | print("Done. Detaching and exiting...") 1022 | sys.exit(0) 1023 | --------------------------------------------------------------------------------