├── requirements.txt ├── screen.png ├── .gitignore ├── .github ├── FUNDING.yml └── workflows │ ├── publish-to-pypi.yml │ ├── publish-to-test-pypi.yml │ ├── main.yml │ └── codeql-analysis.yml ├── test ├── FAKE_PS ├── FAKE_PS_LONG_PIDS ├── DESIRED_STDOUT_NO_PROCESSES ├── FAKE_STDIN_NO_PROCESSES ├── DESIRED_STDOUT_NO_PROCESSES_DOCKER ├── FAKE_STDIN_NO_PROCESSES_DOCKER ├── DESIRED_STDOUT_NEW_FORMAT_USERS ├── DESIRED_STDOUT_NEW_FORMAT ├── DESIRED_STDOUT_NEW_FORMAT_METER ├── DESIRED_STDOUT_LONG_PIDS ├── DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS ├── DESIRED_STDOUT_NEW_FORMAT_COLOR ├── DESIRED_STDOUT_NEW_FORMAT_METER_COLOR ├── FAKE_STDIN_NEW_FORMAT ├── FAKE_STDIN_LONG_PIDS ├── DESIRED_STDOUT_NEW_FORMAT_METER_L ├── DESIRED_STDOUT_NEW_FORMAT_METER_L150 ├── test_main.py ├── DESIRED_STDOUT ├── DESIRED_STDOUT_COLOR ├── FAKE_STDIN ├── DESIRED_STDOUT_L └── DESIRED_STDOUT_L150 ├── CHANGELOG.md ├── LICENSE ├── setup.py ├── README.md └── nvidia-htop.py /requirements.txt: -------------------------------------------------------------------------------- 1 | termcolor -------------------------------------------------------------------------------- /screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/peci1/nvidia-htop/HEAD/screen.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.cache 2 | /build/ 3 | /dist/ 4 | /*.egg-info/ 5 | /*.egg 6 | /*.py[cod] 7 | /__pycache__/ 8 | /*.so 9 | /*~ 10 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [peci1] 4 | #custom: ["bitcoin:1CQEGaeP8k6TftvJjEwo59iSktBefeVArc?label=peci1&message=Github%20Sponsors%20nvidia-htop"] 5 | custom: ["https://btc.com/1CQEGaeP8k6TftvJjEwo59iSktBefeVArc"] -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPi. 2 | 3 | on: 4 | release: 5 | types: [ published ] 6 | 7 | jobs: 8 | build-n-publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: Install wheel 13 | run: python3 -m pip install wheel 14 | - name: Build a binary wheel and a source tarball 15 | run: python3 setup.py sdist bdist_wheel 16 | - name: Publish distribution 📦 to PyPI 17 | uses: pypa/gh-action-pypi-publish@release/v1 18 | with: 19 | password: ${{ secrets.PYPI_API_TOKEN }} 20 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-test-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to test PyPi. 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | jobs: 8 | build-n-publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: Install wheel 13 | run: python3 -m pip install wheel 14 | - name: Build a binary wheel and a source tarball 15 | run: python3 setup.py sdist bdist_wheel 16 | - name: Publish distribution 📦 to Test PyPI 17 | uses: pypa/gh-action-pypi-publish@release/v1 18 | with: 19 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 20 | repository-url: https://test.pypi.org/legacy/ 21 | skip-existing: true 22 | verbose: true 23 | -------------------------------------------------------------------------------- /test/FAKE_PS: -------------------------------------------------------------------------------- 1 | PID USER %CPU %MEM ELAPSED COMMAND 2 | 1032 root 10.0 5.0 11:42:17 python 0.py 3 | 11021 admin 1.0 4.0 12:42:17 python3 1.py 4 | 25544 test 5.0 3.0 13:42:17 python 2.py 5 | 4755 user1 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m 6 | 14518 root 8.0 1.0 15:42:17 python3 3.py 7 | 13956 root 2.0 0.0 16:42:17 python3 4.py 8 | 14293 root 9.0 6.0 17:42:17 python3 5.py 9 | 14294 user2 50.0 7.0 18:42:17 python3 6.py 10 | 14295 user3 99.0 8.0 19:42:17 python3 7.py 11 | 17547 root 230.0 9.0 30:42:17 python3 8.py 12 | 11527 nobody 5.0 10.0 01:42:17 python 9.py 13 | 27159 hacker 0.0 20.0 00:00:00 python3.5 10.py 14 | 9908 nvidia 1.0 50.0 89-03:41:29 python very_long_very_long_very_long_very_long_very_long_very_long_very_long.py -------------------------------------------------------------------------------- /test/FAKE_PS_LONG_PIDS: -------------------------------------------------------------------------------- 1 | PID USER %CPU %MEM ELAPSED COMMAND 2 | 1032 root 10.0 5.0 11:42:17 python 0.py 3 | 1111021 admin 1.0 4.0 12:42:17 python3 1.py 4 | 25544 test 5.0 3.0 13:42:17 python 2.py 5 | 4755 user1 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m 6 | 1114518 root 8.0 1.0 15:42:17 python3 3.py 7 | 13956 root 2.0 0.0 16:42:17 python3 4.py 8 | 14293 root 9.0 6.0 17:42:17 python3 5.py 9 | 14294 user2 50.0 7.0 18:42:17 python3 6.py 10 | 14295 user3 99.0 8.0 19:42:17 python3 7.py 11 | 17547 root 230.0 9.0 30:42:17 python3 8.py 12 | 11527 nobody 5.0 10.0 01:42:17 python 9.py 13 | 27159 hacker 0.0 20.0 00:00:00 python3.5 10.py 14 | 9908 nvidia 1.0 50.0 89-03:41:29 python very_long_very_long_very_long_very_long_very_long_very_long_very_long.py 15 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: CI 4 | 5 | # Controls when the action will run. Triggers the workflow on push or pull request 6 | # events but only for the master branch 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 14 | jobs: 15 | # This workflow contains a single job called "build" 16 | build: 17 | # The type of runner that the job will run on 18 | runs-on: ubuntu-latest 19 | 20 | # Steps represent a sequence of tasks that will be executed as part of the job 21 | steps: 22 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 23 | - uses: actions/checkout@v2 24 | 25 | - name: Pytest 26 | uses: fylein/python-pytest-github-action@v2 27 | with: 28 | args: pip3 install -r requirements.txt && cd "test" && pytest 29 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NO_PROCESSES: -------------------------------------------------------------------------------- 1 | Fri Mar 31 17:22:46 2017 2 | +------------------------------------------------------+ 3 | | NVIDIA-SMI 346.46 Driver Version: 346.46 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GRID K520 Off | 0000:00:03.0 Off | N/A | 9 | | N/A 37C P0 35W / 125W | 10MiB / 4095MiB | 0% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | 12 | +-----------------------------------------------------------------------------+ 13 | | No running processes found | 14 | +-----------------------------------------------------------------------------+ 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.2.0 4 | - Added CLI argument `--meter` to show graphical utilization meters. 5 | 6 | ## 1.1.0 7 | 8 | - Added CLI argument `--id` to filter the processed GPUs (it is passed to `nvidia-smi`). 9 | - Changed some error text to be printed to stderr instead of on stdout. 10 | - The program now exits with the same error code as `nvidia-smi` if its call failed. 11 | 12 | ## 1.0.7 13 | 14 | - Added CLI argument `--user` to filter processes by selected users. 15 | 16 | ## 1.0.6 17 | 18 | - Fixed coloring output. Thanks @tomix1024 ! 19 | 20 | ## 1.0.5 21 | 22 | - Maintenance release. 23 | 24 | ## 1.0.4 25 | 26 | - Fixed output when there are processed with PIDs >= 1M. 27 | 28 | ## 1.0.3 29 | 30 | - Fixed outputting commands longer than 100 characters. 31 | 32 | ## 1.0.2 33 | 34 | - No change, just re-release. 35 | 36 | ## 1.0.1 37 | 38 | - Released on Jan 17 2021. 39 | - Released on PyPi. 40 | 41 | ## 1.0 42 | 43 | - Basic functionality, supports reading nvidia-smi from stdin, or calling it internally if not stdin is provided. 44 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [master, ] 6 | pull_request: 7 | # The branches below must be a subset of the branches above 8 | branches: [master] 9 | schedule: 10 | - cron: '0 18 * * 5' 11 | 12 | jobs: 13 | analyze: 14 | name: Analyze 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout repository 19 | uses: actions/checkout@v2 20 | with: 21 | # We must fetch at least the immediate parents so that if this is 22 | # a pull request then we can checkout the head. 23 | fetch-depth: 2 24 | 25 | # If this run was triggered by a pull request event, then checkout 26 | # the head of the pull request instead of the merge commit. 27 | - run: git checkout HEAD^2 28 | if: ${{ github.event_name == 'pull_request' }} 29 | 30 | # Initializes the CodeQL tools for scanning. 31 | - name: Initialize CodeQL 32 | uses: github/codeql-action/init@v1 33 | with: 34 | languages: python 35 | 36 | - name: Perform CodeQL Analysis 37 | uses: github/codeql-action/analyze@v1 38 | -------------------------------------------------------------------------------- /test/FAKE_STDIN_NO_PROCESSES: -------------------------------------------------------------------------------- 1 | Fri Mar 31 17:22:46 2017 2 | +------------------------------------------------------+ 3 | | NVIDIA-SMI 346.46 Driver Version: 346.46 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GRID K520 Off | 0000:00:03.0 Off | N/A | 9 | | N/A 37C P0 35W / 125W | 10MiB / 4095MiB | 0% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | 12 | +-----------------------------------------------------------------------------+ 13 | | Processes: GPU Memory | 14 | | GPU PID Type Process name Usage | 15 | |=============================================================================| 16 | | No running processes found | 17 | +-----------------------------------------------------------------------------+ -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NO_PROCESSES_DOCKER: -------------------------------------------------------------------------------- 1 | Thu May 14 22:09:09 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 440.82 Driver Version: 440.82 CUDA Version: 10.2 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:01:00.0 Off | N/A | 9 | | 38% 68C P2 73W / 250W | 10916MiB / 11175MiB | 59% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:02:00.0 Off | N/A | 12 | | 23% 29C P8 8W / 250W | 149MiB / 11178MiB | 0% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | 15 | +-----------------------------------------------------------------------------+ 16 | | No running processes found | 17 | | If you're running in a container, you'll only see processes running inside. | 18 | +-----------------------------------------------------------------------------+ 19 | 20 | -------------------------------------------------------------------------------- /test/FAKE_STDIN_NO_PROCESSES_DOCKER: -------------------------------------------------------------------------------- 1 | Thu May 14 22:09:09 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 440.82 Driver Version: 440.82 CUDA Version: 10.2 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:01:00.0 Off | N/A | 9 | | 38% 68C P2 73W / 250W | 10916MiB / 11175MiB | 59% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:02:00.0 Off | N/A | 12 | | 23% 29C P8 8W / 250W | 149MiB / 11178MiB | 0% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | 15 | +-----------------------------------------------------------------------------+ 16 | | Processes: GPU Memory | 17 | | GPU PID Type Process name Usage | 18 | |=============================================================================| 19 | +-----------------------------------------------------------------------------+ 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, Martin Pecka 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from setuptools import setup 4 | import pathlib 5 | 6 | here = pathlib.Path(__file__).parent.resolve() 7 | 8 | # Get the long description from the README file 9 | long_description = (here / 'README.md').read_text(encoding='utf-8') 10 | 11 | setup(name='nvidia-htop', 12 | version='1.2.0', 13 | description='A tool for enriching the output of nvidia-smi', 14 | long_description=long_description, 15 | long_description_content_type='text/markdown', 16 | url='https://github.com/peci1/nvidia-htop', 17 | author='Martin Pecka', 18 | author_email='peci1@seznam.cz', 19 | scripts=['nvidia-htop.py'], 20 | install_requires=[ 21 | "termcolor" 22 | ], 23 | python_requires='>=3.5, <4', 24 | classifiers=[ 25 | 'Development Status :: 5 - Production/Stable', 26 | 27 | 'Environment :: Console', 28 | 'Environment :: GPU', 29 | 30 | 'Intended Audience :: Developers', 31 | 'Intended Audience :: Science/Research', 32 | 'Intended Audience :: System Administrators', 33 | 34 | 'Topic :: System :: Monitoring', 35 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 36 | 'Topic :: Utilities', 37 | 38 | 'License :: OSI Approved :: BSD License', 39 | 40 | 'Operating System :: POSIX :: Linux', 41 | 'Programming Language :: Python :: 3', 42 | 'Programming Language :: Python :: 3 :: Only', 43 | ], 44 | keywords='nvidia, nvidia-smi, GPU, htop, top', 45 | project_urls={ 46 | 'Bug Reports': 'https://github.com/peci1/nvidia-htop/issues', 47 | 'Source': 'https://github.com/peci1/nvidia-htop', 48 | }, 49 | ) 50 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_USERS: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | | N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | | N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 22 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 23 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 24 | +-----------------------------------------------------------------------------+ 25 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | | N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | | N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 24 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-----------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_METER: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | [|| ] | [ ] N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | [|||||||| ] | [ ] N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 24 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-----------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_LONG_PIDS: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | | N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | | N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-------------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 1111021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 24 | | 1 1114518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-------------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | [|| ] | [ ] N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | [|||||||| ] | [ ] N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-------------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 1111021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 24 | | 1 1114518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-------------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_COLOR: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | | N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | | N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 24 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-----------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_METER_COLOR: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | [|| ] | [ ] N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | [|||||||| ] | [ ] N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 24 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-----------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/FAKE_STDIN_NEW_FORMAT: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | | N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | | N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | Processes: | 20 | | GPU GI CI PID Type Process name GPU Memory | 21 | | ID ID Usage | 22 | |=============================================================================| 23 | | 0 N/A N/A 1032 C python 745MiB | 24 | | 1 N/A N/A 11021 G python3 244MiB | 25 | | 1 N/A N/A 25544 G python 139MiB | 26 | | 1 N/A N/A 4755 G ...matlab94/bin/glnxa64/MATLAB 3MiB | 27 | | 1 N/A N/A 14518 G python3 1MiB | 28 | | 1 N/A N/A 13956 G python3 472MiB | 29 | +-----------------------------------------------------------------------------+ -------------------------------------------------------------------------------- /test/FAKE_STDIN_LONG_PIDS: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | | N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | | N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-----------------------------------------------------------------------------+ 19 | | Processes: | 20 | | GPU GI CI PID Type Process name GPU Memory | 21 | | ID ID Usage | 22 | |=============================================================================| 23 | | 0 N/A N/A 1032 C python 745MiB | 24 | | 1 N/A N/A 1111021 G python3 244MiB | 25 | | 1 N/A N/A 25544 G python 139MiB | 26 | | 1 N/A N/A 4755 G ...matlab94/bin/glnxa64/MATLAB 3MiB | 27 | | 1 N/A N/A 1114518 G python3 1MiB | 28 | | 1 N/A N/A 13956 G python3 472MiB | 29 | +-----------------------------------------------------------------------------+ 30 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_METER_L: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | [|| ] | [ ] N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | [|||||||| ] | [ ] N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +-------------------------------------------------------------------------------------------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m | 24 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +-------------------------------------------------------------------------------------------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_NEW_FORMAT_METER_L150: -------------------------------------------------------------------------------- 1 | Sun Aug 2 13:44:21 2020 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | | | | MIG M. | 8 | |===============================+======================+======================| 9 | | 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A | 10 | | 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default | 11 | | | [|| ] | [ ] N/A | 12 | +-------------------------------+----------------------+----------------------+ 13 | | 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A | 14 | | 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default | 15 | | | [|||||||| ] | [ ] N/A | 16 | +-------------------------------+----------------------+----------------------+ 17 | 18 | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 19 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 20 | | 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py | 21 | | 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py | 22 | | 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py | 23 | | 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m | 24 | | 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py | 25 | | 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py | 26 | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 27 | -------------------------------------------------------------------------------- /test/test_main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import unittest 4 | 5 | 6 | class TestNvidiaHtop(unittest.TestCase): 7 | def do_test(self, stdin, stdout, fake_ps='FAKE_PS', call_args=None): 8 | if call_args is None: 9 | call_args = list() 10 | with open(stdin, 'r') as fake_stdin: 11 | env = os.environ.copy() 12 | env['FORCE_COLOR'] = '1' 13 | test_call = subprocess.run(["../nvidia-htop.py", "--fake-ps", fake_ps] + call_args, stdin=fake_stdin, stdout=subprocess.PIPE, env=env) 14 | self.assertEqual(test_call.returncode, 0) 15 | with open(stdout, 'r') as desired_stdout: 16 | out = desired_stdout.read() 17 | self.assertEqual(out, test_call.stdout.decode()) 18 | 19 | def test_with_processes(self): 20 | self.do_test('FAKE_STDIN', 'DESIRED_STDOUT') 21 | 22 | def test_new_format(self): 23 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT') 24 | 25 | def test_new_format_users(self): 26 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_USERS', call_args=["-u", "root,test"]) 27 | 28 | # The --id option cannot be tested in this way. So we just check that the option is considered valid. 29 | def test_new_format_filter_ids(self): 30 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT', call_args=["-i", "1,2"]) 31 | 32 | def test_long_pids(self): 33 | self.do_test('FAKE_STDIN_LONG_PIDS', 'DESIRED_STDOUT_LONG_PIDS', fake_ps='FAKE_PS_LONG_PIDS') 34 | 35 | def test_with_processes_color(self): 36 | self.do_test('FAKE_STDIN', 'DESIRED_STDOUT_COLOR', call_args=["-c"]) 37 | 38 | def test_new_format_with_processes_color(self): 39 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_COLOR', call_args=["-c"]) 40 | 41 | def test_with_processes_long(self): 42 | self.do_test('FAKE_STDIN', 'DESIRED_STDOUT_L', call_args=["-l"]) 43 | 44 | def test_with_processes_very_long(self): 45 | self.do_test('FAKE_STDIN', 'DESIRED_STDOUT_L150', call_args=["-l", "150"]) 46 | 47 | def test_no_processes(self): 48 | self.do_test('FAKE_STDIN_NO_PROCESSES', 'DESIRED_STDOUT_NO_PROCESSES') 49 | 50 | def test_no_processes_docker(self): 51 | self.do_test('FAKE_STDIN_NO_PROCESSES_DOCKER', 'DESIRED_STDOUT_NO_PROCESSES_DOCKER') 52 | 53 | def test_with_meters(self): 54 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER', call_args=["-m"]) 55 | 56 | def test_with_meters_color(self): 57 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_COLOR', call_args=["-m", "-c"]) 58 | 59 | def test_with_meters_long_pids(self): 60 | self.do_test('FAKE_STDIN_LONG_PIDS', 'DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS', call_args=["-m"], fake_ps='FAKE_PS_LONG_PIDS') 61 | 62 | def test_with_meters_long(self): 63 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L', call_args=["-m", "-l"]) 64 | 65 | def test_with_meters_very_long(self): 66 | self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L150', call_args=["-m", "-l", "150"]) 67 | 68 | 69 | 70 | if __name__ == '__main__': 71 | unittest.main() 72 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT: -------------------------------------------------------------------------------- 1 | Mon May 21 14:07:29 2018 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 390.25 Driver Version: 390.25 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:04:00.0 Off | N/A | 9 | | 53% 75C P2 223W / 250W | 10807MiB / 11178MiB | 98% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:05:00.0 Off | N/A | 12 | | 66% 83C P2 237W / 250W | 10783MiB / 11178MiB | 100% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | | 2 GeForce GTX 108... Off | 00000000:08:00.0 Off | N/A | 15 | | 21% 40C P2 74W / 250W | 10793MiB / 11178MiB | 61% Default | 16 | +-------------------------------+----------------------+----------------------+ 17 | | 3 GeForce GTX 108... Off | 00000000:09:00.0 Off | N/A | 18 | | 65% 82C P2 217W / 250W | 7996MiB / 11178MiB | 100% Default | 19 | +-------------------------------+----------------------+----------------------+ 20 | | 4 GeForce GTX 108... Off | 00000000:84:00.0 Off | N/A | 21 | | 39% 61C P2 149W / 250W | 5964MiB / 11178MiB | 56% Default | 22 | +-------------------------------+----------------------+----------------------+ 23 | | 5 GeForce GTX 108... Off | 00000000:85:00.0 Off | N/A | 24 | | 64% 82C P2 226W / 250W | 10785MiB / 11178MiB | 100% Default | 25 | +-------------------------------+----------------------+----------------------+ 26 | | 6 GeForce GTX 108... Off | 00000000:88:00.0 Off | N/A | 27 | | 44% 65C P2 139W / 250W | 10823MiB / 11178MiB | 23% Default | 28 | +-------------------------------+----------------------+----------------------+ 29 | | 7 GeForce GTX 108... Off | 00000000:89:00.0 Off | N/A | 30 | | 46% 69C P2 249W / 250W | 7773MiB / 11178MiB | 47% Default | 31 | +-------------------------------+----------------------+----------------------+ 32 | 33 | +-----------------------------------------------------------------------------+ 34 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 35 | | 0 1032 root 10781MiB 10.0 5.0 11:42:17 python 0.py | 36 | | 1 11021 admin 10765MiB 1.0 4.0 12:42:17 python3 1.py | 37 | | 2 25544 test 10775MiB 5.0 3.0 13:42:17 python 2.py | 38 | | 3 4755 user1 389MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 39 | | 3 14518 root 7589MiB 8.0 1.0 15:42:17 python3 3.py | 40 | | 4 13956 root 5480MiB 2.0 0.0 16:42:17 python3 4.py | 41 | | 4 14293 root 145MiB 9.0 6.0 17:42:17 python3 5.py | 42 | | 4 14294 user2 145MiB 50.0 7.0 18:42:17 python3 6.py | 43 | | 4 14295 user3 145MiB 99.0 8.0 19:42:17 python3 7.py | 44 | | 5 17547 root 10765MiB 230.0 9.0 30:42:17 python3 8.py | 45 | | 6 11527 nobody 10221MiB 5.0 10.0 01:42:17 python 9.py | 46 | | 6 27159 hacker 577MiB 0.0 20.0 00:00:00 python3.5 10.py | 47 | | 7 9908 nvidia 7755MiB 1.0 50.0 89 days python very_long_ver | 48 | +-----------------------------------------------------------------------------+ 49 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_COLOR: -------------------------------------------------------------------------------- 1 | Mon May 21 14:07:29 2018 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 390.25 Driver Version: 390.25 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:04:00.0 Off | N/A | 9 | | 53% 75C P2 223W / 250W | 10807MiB / 11178MiB | 98% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:05:00.0 Off | N/A | 12 | | 66% 83C P2 237W / 250W | 10783MiB / 11178MiB | 100% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | | 2 GeForce GTX 108... Off | 00000000:08:00.0 Off | N/A | 15 | | 21% 40C P2 74W / 250W | 10793MiB / 11178MiB | 61% Default | 16 | +-------------------------------+----------------------+----------------------+ 17 | | 3 GeForce GTX 108... Off | 00000000:09:00.0 Off | N/A | 18 | | 65% 82C P2 217W / 250W | 7996MiB / 11178MiB | 100% Default | 19 | +-------------------------------+----------------------+----------------------+ 20 | | 4 GeForce GTX 108... Off | 00000000:84:00.0 Off | N/A | 21 | | 39% 61C P2 149W / 250W | 5964MiB / 11178MiB | 56% Default | 22 | +-------------------------------+----------------------+----------------------+ 23 | | 5 GeForce GTX 108... Off | 00000000:85:00.0 Off | N/A | 24 | | 64% 82C P2 226W / 250W | 10785MiB / 11178MiB | 100% Default | 25 | +-------------------------------+----------------------+----------------------+ 26 | | 6 GeForce GTX 108... Off | 00000000:88:00.0 Off | N/A | 27 | | 44% 65C P2 139W / 250W | 10823MiB / 11178MiB | 23% Default | 28 | +-------------------------------+----------------------+----------------------+ 29 | | 7 GeForce GTX 108... Off | 00000000:89:00.0 Off | N/A | 30 | | 46% 69C P2 249W / 250W | 7773MiB / 11178MiB | 47% Default | 31 | +-------------------------------+----------------------+----------------------+ 32 | 33 | +-----------------------------------------------------------------------------+ 34 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 35 | | 0 1032 root 10781MiB 10.0 5.0 11:42:17 python 0.py | 36 | | 1 11021 admin 10765MiB 1.0 4.0 12:42:17 python3 1.py | 37 | | 2 25544 test 10775MiB 5.0 3.0 13:42:17 python 2.py | 38 | | 3 4755 user1 389MiB 3.0 2.0 14:42:17 /opt/software/MATLAB | 39 | | 3 14518 root 7589MiB 8.0 1.0 15:42:17 python3 3.py | 40 | | 4 13956 root 5480MiB 2.0 0.0 16:42:17 python3 4.py | 41 | | 4 14293 root 145MiB 9.0 6.0 17:42:17 python3 5.py | 42 | | 4 14294 user2 145MiB 50.0 7.0 18:42:17 python3 6.py | 43 | | 4 14295 user3 145MiB 99.0 8.0 19:42:17 python3 7.py | 44 | | 5 17547 root 10765MiB 230.0 9.0 30:42:17 python3 8.py | 45 | | 6 11527 nobody 10221MiB 5.0 10.0 01:42:17 python 9.py | 46 | | 6 27159 hacker 577MiB 0.0 20.0 00:00:00 python3.5 10.py | 47 | | 7 9908 nvidia 7755MiB 1.0 50.0 89 days python very_long_ver | 48 | +-----------------------------------------------------------------------------+ 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nvidia-htop 2 | 3 | A tool for enriching the output of `nvidia-smi`. 4 | 5 | ![CI](https://github.com/peci1/nvidia-htop/workflows/CI/badge.svg) 6 | [![PyPI version](https://badge.fury.io/py/nvidia-htop.svg)](https://badge.fury.io/py/nvidia-htop) 7 | 8 | # Install 9 | `pip3 install nvidia-htop` 10 | 11 | **Yes, this tool has been on PyPi since 2021! Enjoy the super-easy way to install it.** 12 | 13 | 14 | 15 | ## Usage 16 | 17 | nvidia-htop.py [-l [length]] 18 | print GPU utilization with usernames and CPU stats for each GPU-utilizing process 19 | 20 | -l|--command-length [length] Print longer part of the commandline. If `length' 21 | is provided, use it as the commandline length, 22 | otherwise print first 100 characters. 23 | -c|--color Colorize the output (green - free GPU, yellow - 24 | moderately used GPU, red - fully used GPU) 25 | -u|--user USER[,USER] Limit the list of processes to selected users 26 | (comma-separated). 27 | -i|--id ID[,ID[,ID]] Limit the command to selected GPU IDs (comma-separated). 28 | -m|--meter Shows meters for GPU and VRAM utilization. 29 | 30 | Note: for backward compatibility, `nvidia-smi | nvidia-htop.py [-l [length]]` is also supported. 31 | 32 | Note: running inside a container (docker, singularity, ...), `nvidia-smi` can only see processes running in the container. 33 | 34 | Note: To periodically check the output of nvidia-htop, use the `watch` utility: `watch nvidia-htop.py`. To get colored output, you have to pass option `-c` to both `watch` and `nvidia-htop`, e.g. `watch -c nvidia-htop.py -c`. 35 | 36 | ## Example output 37 | 38 | $ nvidia-htop.py -l 39 | Mon May 21 15:06:35 2018 40 | +-----------------------------------------------------------------------------+ 41 | | NVIDIA-SMI 390.25 Driver Version: 390.25 | 42 | |-------------------------------+----------------------+----------------------+ 43 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 44 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 45 | |===============================+======================+======================| 46 | | 0 GeForce GTX 108... Off | 00000000:04:00.0 Off | N/A | 47 | | 53% 75C P2 174W / 250W | 10807MiB / 11178MiB | 97% Default | 48 | +-------------------------------+----------------------+----------------------+ 49 | | 1 GeForce GTX 108... Off | 00000000:05:00.0 Off | N/A | 50 | | 66% 82C P2 220W / 250W | 10783MiB / 11178MiB | 100% Default | 51 | +-------------------------------+----------------------+----------------------+ 52 | | 2 GeForce GTX 108... Off | 00000000:08:00.0 Off | N/A | 53 | | 45% 67C P2 85W / 250W | 10793MiB / 11178MiB | 51% Default | 54 | +-------------------------------+----------------------+----------------------+ 55 | 56 | +-----------------------------------------------------------------------------+ 57 | | GPU PID USER GPU MEM %MEM %CPU COMMAND | 58 | | 0 1032 anonymou 10781MiB 308 3.7 python train_image_classifier.py --train_dir=/mnt/xxxxxxxx/xxxxxxxx/xxxxxxxx/xxxxxxx/xxxxxxxxxxxxxxx | 59 | | 1 11021 cannotte 10765MiB 114 1.5 python3 ./train.py --flagfile /xxxxxxxx/xxxxxxxx/xxxxxxxx/xxxxxxxxx/xx/xxxxxxxxxxxxxxx | 60 | | 2 25544 nevermin 10775MiB 108 2.0 python -m xxxxxxxxxxxxxxxxxxxxxxxxxxxxx | 61 | +-----------------------------------------------------------------------------+ 62 | 63 | ## Screenshot with output coloring 64 | 65 | ![Screenshot](screen.png) 66 | -------------------------------------------------------------------------------- /test/FAKE_STDIN: -------------------------------------------------------------------------------- 1 | Mon May 21 14:07:29 2018 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 390.25 Driver Version: 390.25 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:04:00.0 Off | N/A | 9 | | 53% 75C P2 223W / 250W | 10807MiB / 11178MiB | 98% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:05:00.0 Off | N/A | 12 | | 66% 83C P2 237W / 250W | 10783MiB / 11178MiB | 100% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | | 2 GeForce GTX 108... Off | 00000000:08:00.0 Off | N/A | 15 | | 21% 40C P2 74W / 250W | 10793MiB / 11178MiB | 61% Default | 16 | +-------------------------------+----------------------+----------------------+ 17 | | 3 GeForce GTX 108... Off | 00000000:09:00.0 Off | N/A | 18 | | 65% 82C P2 217W / 250W | 7996MiB / 11178MiB | 100% Default | 19 | +-------------------------------+----------------------+----------------------+ 20 | | 4 GeForce GTX 108... Off | 00000000:84:00.0 Off | N/A | 21 | | 39% 61C P2 149W / 250W | 5964MiB / 11178MiB | 56% Default | 22 | +-------------------------------+----------------------+----------------------+ 23 | | 5 GeForce GTX 108... Off | 00000000:85:00.0 Off | N/A | 24 | | 64% 82C P2 226W / 250W | 10785MiB / 11178MiB | 100% Default | 25 | +-------------------------------+----------------------+----------------------+ 26 | | 6 GeForce GTX 108... Off | 00000000:88:00.0 Off | N/A | 27 | | 44% 65C P2 139W / 250W | 10823MiB / 11178MiB | 23% Default | 28 | +-------------------------------+----------------------+----------------------+ 29 | | 7 GeForce GTX 108... Off | 00000000:89:00.0 Off | N/A | 30 | | 46% 69C P2 249W / 250W | 7773MiB / 11178MiB | 47% Default | 31 | +-------------------------------+----------------------+----------------------+ 32 | 33 | +-----------------------------------------------------------------------------+ 34 | | Processes: GPU Memory | 35 | | GPU PID Type Process name Usage | 36 | |=============================================================================| 37 | | 0 1032 C python 10781MiB | 38 | | 1 11021 C python3 10765MiB | 39 | | 2 25544 C python 10775MiB | 40 | | 3 4755 C ...ware/MATLAB/matlab94/bin/glnxa64/MATLAB 389MiB | 41 | | 3 14518 C python3 7589MiB | 42 | | 4 13956 C python3 5480MiB | 43 | | 4 14293 C python3 145MiB | 44 | | 4 14294 C python3 145MiB | 45 | | 4 14295 C python3 145MiB | 46 | | 5 17547 C python3 10765MiB | 47 | | 6 11527 C python 10221MiB | 48 | | 6 27159 C python3.5 577MiB | 49 | | 7 9908 C python 7755MiB | 50 | +-----------------------------------------------------------------------------+ -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_L: -------------------------------------------------------------------------------- 1 | Mon May 21 14:07:29 2018 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 390.25 Driver Version: 390.25 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:04:00.0 Off | N/A | 9 | | 53% 75C P2 223W / 250W | 10807MiB / 11178MiB | 98% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:05:00.0 Off | N/A | 12 | | 66% 83C P2 237W / 250W | 10783MiB / 11178MiB | 100% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | | 2 GeForce GTX 108... Off | 00000000:08:00.0 Off | N/A | 15 | | 21% 40C P2 74W / 250W | 10793MiB / 11178MiB | 61% Default | 16 | +-------------------------------+----------------------+----------------------+ 17 | | 3 GeForce GTX 108... Off | 00000000:09:00.0 Off | N/A | 18 | | 65% 82C P2 217W / 250W | 7996MiB / 11178MiB | 100% Default | 19 | +-------------------------------+----------------------+----------------------+ 20 | | 4 GeForce GTX 108... Off | 00000000:84:00.0 Off | N/A | 21 | | 39% 61C P2 149W / 250W | 5964MiB / 11178MiB | 56% Default | 22 | +-------------------------------+----------------------+----------------------+ 23 | | 5 GeForce GTX 108... Off | 00000000:85:00.0 Off | N/A | 24 | | 64% 82C P2 226W / 250W | 10785MiB / 11178MiB | 100% Default | 25 | +-------------------------------+----------------------+----------------------+ 26 | | 6 GeForce GTX 108... Off | 00000000:88:00.0 Off | N/A | 27 | | 44% 65C P2 139W / 250W | 10823MiB / 11178MiB | 23% Default | 28 | +-------------------------------+----------------------+----------------------+ 29 | | 7 GeForce GTX 108... Off | 00000000:89:00.0 Off | N/A | 30 | | 46% 69C P2 249W / 250W | 7773MiB / 11178MiB | 47% Default | 31 | +-------------------------------+----------------------+----------------------+ 32 | 33 | +-------------------------------------------------------------------------------------------------------------------------------------------------------------+ 34 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 35 | | 0 1032 root 10781MiB 10.0 5.0 11:42:17 python 0.py | 36 | | 1 11021 admin 10765MiB 1.0 4.0 12:42:17 python3 1.py | 37 | | 2 25544 test 10775MiB 5.0 3.0 13:42:17 python 2.py | 38 | | 3 4755 user1 389MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m | 39 | | 3 14518 root 7589MiB 8.0 1.0 15:42:17 python3 3.py | 40 | | 4 13956 root 5480MiB 2.0 0.0 16:42:17 python3 4.py | 41 | | 4 14293 root 145MiB 9.0 6.0 17:42:17 python3 5.py | 42 | | 4 14294 user2 145MiB 50.0 7.0 18:42:17 python3 6.py | 43 | | 4 14295 user3 145MiB 99.0 8.0 19:42:17 python3 7.py | 44 | | 5 17547 root 10765MiB 230.0 9.0 30:42:17 python3 8.py | 45 | | 6 11527 nobody 10221MiB 5.0 10.0 01:42:17 python 9.py | 46 | | 6 27159 hacker 577MiB 0.0 20.0 00:00:00 python3.5 10.py | 47 | | 7 9908 nvidia 7755MiB 1.0 50.0 89 days python very_long_very_long_very_long_very_long_very_long_very_long_very_long.py | 48 | +-------------------------------------------------------------------------------------------------------------------------------------------------------------+ 49 | -------------------------------------------------------------------------------- /test/DESIRED_STDOUT_L150: -------------------------------------------------------------------------------- 1 | Mon May 21 14:07:29 2018 2 | +-----------------------------------------------------------------------------+ 3 | | NVIDIA-SMI 390.25 Driver Version: 390.25 | 4 | |-------------------------------+----------------------+----------------------+ 5 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 6 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 7 | |===============================+======================+======================| 8 | | 0 GeForce GTX 108... Off | 00000000:04:00.0 Off | N/A | 9 | | 53% 75C P2 223W / 250W | 10807MiB / 11178MiB | 98% Default | 10 | +-------------------------------+----------------------+----------------------+ 11 | | 1 GeForce GTX 108... Off | 00000000:05:00.0 Off | N/A | 12 | | 66% 83C P2 237W / 250W | 10783MiB / 11178MiB | 100% Default | 13 | +-------------------------------+----------------------+----------------------+ 14 | | 2 GeForce GTX 108... Off | 00000000:08:00.0 Off | N/A | 15 | | 21% 40C P2 74W / 250W | 10793MiB / 11178MiB | 61% Default | 16 | +-------------------------------+----------------------+----------------------+ 17 | | 3 GeForce GTX 108... Off | 00000000:09:00.0 Off | N/A | 18 | | 65% 82C P2 217W / 250W | 7996MiB / 11178MiB | 100% Default | 19 | +-------------------------------+----------------------+----------------------+ 20 | | 4 GeForce GTX 108... Off | 00000000:84:00.0 Off | N/A | 21 | | 39% 61C P2 149W / 250W | 5964MiB / 11178MiB | 56% Default | 22 | +-------------------------------+----------------------+----------------------+ 23 | | 5 GeForce GTX 108... Off | 00000000:85:00.0 Off | N/A | 24 | | 64% 82C P2 226W / 250W | 10785MiB / 11178MiB | 100% Default | 25 | +-------------------------------+----------------------+----------------------+ 26 | | 6 GeForce GTX 108... Off | 00000000:88:00.0 Off | N/A | 27 | | 44% 65C P2 139W / 250W | 10823MiB / 11178MiB | 23% Default | 28 | +-------------------------------+----------------------+----------------------+ 29 | | 7 GeForce GTX 108... Off | 00000000:89:00.0 Off | N/A | 30 | | 46% 69C P2 249W / 250W | 7773MiB / 11178MiB | 47% Default | 31 | +-------------------------------+----------------------+----------------------+ 32 | 33 | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 34 | | GPU PID USER GPU MEM %CPU %MEM TIME COMMAND | 35 | | 0 1032 root 10781MiB 10.0 5.0 11:42:17 python 0.py | 36 | | 1 11021 admin 10765MiB 1.0 4.0 12:42:17 python3 1.py | 37 | | 2 25544 test 10775MiB 5.0 3.0 13:42:17 python 2.py | 38 | | 3 4755 user1 389MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m | 39 | | 3 14518 root 7589MiB 8.0 1.0 15:42:17 python3 3.py | 40 | | 4 13956 root 5480MiB 2.0 0.0 16:42:17 python3 4.py | 41 | | 4 14293 root 145MiB 9.0 6.0 17:42:17 python3 5.py | 42 | | 4 14294 user2 145MiB 50.0 7.0 18:42:17 python3 6.py | 43 | | 4 14295 user3 145MiB 99.0 8.0 19:42:17 python3 7.py | 44 | | 5 17547 root 10765MiB 230.0 9.0 30:42:17 python3 8.py | 45 | | 6 11527 nobody 10221MiB 5.0 10.0 01:42:17 python 9.py | 46 | | 6 27159 hacker 577MiB 0.0 20.0 00:00:00 python3.5 10.py | 47 | | 7 9908 nvidia 7755MiB 1.0 50.0 89 days python very_long_very_long_very_long_very_long_very_long_very_long_very_long.py | 48 | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 49 | -------------------------------------------------------------------------------- /nvidia-htop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ####### 4 | # USAGE 5 | # 6 | # [nvidia-smi | ] nvidia-htop.py [-l [length]] [-i ID] 7 | # print GPU utilization with usernames and CPU stats for each GPU-utilizing process 8 | # 9 | # -l|--command-length [length] Print longer part of the commandline. If `length' 10 | # is provided, use it as the commandline length, 11 | # otherwise print first 100 characters. 12 | # -c|--color Colorize the output (green - free GPU, yellow - 13 | # moderately used GPU, red - fully used GPU). 14 | # -i|--id ID[,ID[,ID...]] Limit the command to selected GPU IDs (comma-separated). 15 | ###### 16 | 17 | import sys 18 | import os 19 | import re 20 | import subprocess 21 | import select 22 | import argparse 23 | from itertools import takewhile 24 | from termcolor import colored 25 | 26 | MEMORY_FREE_RATIO = 0.05 27 | MEMORY_MODERATE_RATIO = 0.9 28 | GPU_FREE_RATIO = 0.05 29 | GPU_MODERATE_RATIO = 0.75 30 | 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument('-l', '--command-length', default=20, const=100, type=int, nargs='?') 33 | parser.add_argument('-c', '--color', action='store_true') 34 | parser.add_argument('-m', '--meter', action='store_true', help="Shows meters for GPU and VRAM utilization") 35 | parser.add_argument('-u', '--user', default='', help="Limit the list of processes to selected users (comma-separated)") 36 | parser.add_argument('-i', '--id', default='', help="Limit the command to selected GPU IDs (comma-separated)") 37 | # only for testing 38 | parser.add_argument('-p', '--fake-ps', help="The list of processes to use instead of real output of `ps`") 39 | 40 | args = parser.parse_args() 41 | 42 | # parse the command length argument 43 | command_length = args.command_length 44 | color = args.color 45 | meter = args.meter 46 | fake_ps = args.fake_ps 47 | users = set(args.user.split(',')) if len(args.user) > 0 else None 48 | 49 | # for testing, the stdin can be provided in a file 50 | fake_stdin_path = os.getenv("FAKE_STDIN_PATH", None) 51 | stdin_lines = [] 52 | if sys.stdin in select.select([sys.stdin], [], [], 0)[0]: 53 | stdin_lines = sys.stdin.readlines() 54 | 55 | if fake_stdin_path is not None: 56 | with open(fake_stdin_path, 'rt') as f: 57 | lines = f.readlines() 58 | elif stdin_lines: 59 | lines = stdin_lines 60 | if len(args.id) > 0: 61 | print('nvidia-htop argument -i/--id cannot be used when nvidia-smi output is being piped into it. To filter the' 62 | ' shown GPUs, pass the -i argument to the nvidia-smi call instead.', file=sys.stderr) 63 | else: 64 | nvidiasmi_args = [] 65 | if len(args.id) > 0: 66 | nvidiasmi_args = ['-i', args.id] 67 | ps_call = subprocess.run(['nvidia-smi'] + nvidiasmi_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 68 | if ps_call.returncode != 0: 69 | print('nvidia-smi exited with error code {}:'.format(ps_call.returncode), file=sys.stderr) 70 | print(ps_call.stdout.decode() + ps_call.stderr.decode(), file=sys.stderr) 71 | sys.exit(ps_call.returncode) 72 | lines_proc = ps_call.stdout.decode().split("\n") 73 | lines = [line + '\n' for line in lines_proc[:-1]] 74 | lines += lines_proc[-1] 75 | 76 | def add_meters(_lines): 77 | pattern = re.compile(r"\| (?:N/A|..%)\s+[0-9]{2,3}C.*\s([0-9]+)MiB\s+/\s+([0-9]+)MiB.*\s([0-9]+)%") 78 | gpu_util_lines = list(filter(lambda tup: pattern.match(tup[1]), enumerate(_lines))) 79 | for i, line in gpu_util_lines: 80 | m = pattern.match(line) 81 | used_mem = int(m.group(1)) 82 | total_mem = int(m.group(2)) 83 | gpu_util = int(m.group(3)) / 100.0 84 | mem_util = used_mem / float(total_mem) 85 | # Uses empty space underneath gpu & mem util stats for meter placement. 86 | meter_space = re.split(r"\|(?!$)", _lines[i+1]) 87 | gpu_util_space, mig, _ = re.split(r"([^ |]+ \|)", meter_space[-1], maxsplit=1) 88 | gpu_meter_space = len(gpu_util_space)-4 89 | mem_meter_space = len(meter_space[2])-4 90 | # Fill gpu and mem util meters proportional to reported utilization 91 | gpu_meter = "|"*round(gpu_util*gpu_meter_space) + " "*round((1-gpu_util)*gpu_meter_space) 92 | mem_meter = "|"*round(mem_util*mem_meter_space) + " "*round((1-mem_util)*mem_meter_space) 93 | meter_space[-1] = f" [{gpu_meter}] {mig}" 94 | meter_space[2] = f" [{mem_meter}] " 95 | _lines[i+1] = '|'.join(meter_space) 96 | 97 | return _lines 98 | 99 | def colorize(_lines): 100 | _lines = add_meters(_lines) if meter else _lines 101 | # Index of the first content line of the current cell in the nvidia-smi output. 102 | start_idx = 0 103 | for j in range(len(_lines)): 104 | line = _lines[j] 105 | m = re.match(r"\| (?:N/A|..%)\s+[0-9]{2,3}C.*\s([0-9]+)MiB\s+/\s+([0-9]+)MiB.*\s([0-9]+)%", line) 106 | if m is not None: 107 | used_mem = int(m.group(1)) 108 | total_mem = int(m.group(2)) 109 | gpu_util = int(m.group(3)) / 100.0 110 | mem_util = used_mem / float(total_mem) 111 | 112 | is_moderate = False 113 | is_high = gpu_util >= GPU_MODERATE_RATIO or mem_util >= MEMORY_MODERATE_RATIO 114 | if not is_high: 115 | is_moderate = gpu_util >= GPU_FREE_RATIO or mem_util >= MEMORY_FREE_RATIO 116 | 117 | c = 'red' if is_high else ('yellow' if is_moderate else 'green') 118 | # Color all lines from the `start_idx`, until we find a separator. 119 | for k in takewhile(lambda k: not lines[k].startswith("+----"), range(start_idx, len(_lines))): 120 | _lines[k] = colored(_lines[k], c) 121 | elif line.startswith("|====") or line.startswith("+----"): 122 | # If we find a separator, either the end of the header, or a separator between two GPUs, update the `start_idx`. 123 | start_idx = j+1 124 | 125 | return _lines 126 | 127 | 128 | lines_to_print = [] 129 | is_new_format = False 130 | # Copy the utilization upper part verbatim 131 | for i in range(len(lines)): 132 | if not lines[i].startswith("| Processes:"): 133 | lines_to_print.append(lines[i].rstrip()) 134 | else: 135 | while not lines[i].startswith("|===="): 136 | m = re.search(r'GPU\s*GI\s*CI', lines[i]) 137 | if m is not None: 138 | is_new_format = True 139 | i += 1 140 | i += 1 141 | break 142 | 143 | if color: 144 | lines_to_print = colorize(lines_to_print) 145 | 146 | elif meter: 147 | lines_to_print = add_meters(lines_to_print) 148 | 149 | # we print all but the last line which is the +---+ separator 150 | for line in lines_to_print[:-1]: 151 | print(line) 152 | 153 | no_running_process = "No running processes found" 154 | if no_running_process in lines[i] or lines[i].startswith("+--"): 155 | print(lines[-1].strip()) 156 | print("| " + no_running_process + " " * (73 - len(no_running_process)) + " |") 157 | # Issue #9, running inside docker and seeing no processes 158 | if lines[i].startswith("+--"): 159 | print("| If you're running in a container, you'll only see processes running inside. |") 160 | print(lines[-1]) 161 | sys.exit() 162 | 163 | # Parse the PIDs from the lower part 164 | gpu_num = [] 165 | pid = [] 166 | gpu_mem = [] 167 | user = [] 168 | cpu = [] 169 | mem = [] 170 | time = [] 171 | command = [] 172 | 173 | fields = ( 174 | gpu_num, 175 | pid, 176 | gpu_mem, 177 | user, 178 | cpu, 179 | mem, 180 | time, 181 | command, 182 | ) 183 | 184 | gpu_num_idx = 1 185 | pid_idx = 2 if not is_new_format else 4 186 | gpu_mem_idx = -3 187 | 188 | while not lines[i].startswith("+--"): 189 | if "Not Supported" in lines[i]: 190 | i += 1 191 | continue 192 | line = lines[i] 193 | line = re.split(r'\s+', line) 194 | gpu_num.append(line[gpu_num_idx]) 195 | pid.append(line[pid_idx]) 196 | gpu_mem.append(line[gpu_mem_idx]) 197 | user.append("") 198 | cpu.append("") 199 | mem.append("") 200 | time.append("") 201 | command.append("") 202 | i += 1 203 | 204 | if fake_ps is None: 205 | # Query the PIDs using ps 206 | ps_format = "pid,user,%cpu,%mem,etime,command" 207 | ps_call = subprocess.run(["ps", "-o", ps_format, "-p", ",".join(pid)], stdout=subprocess.PIPE) 208 | processes = ps_call.stdout.decode().split("\n") 209 | else: 210 | with open(fake_ps, 'r') as f: 211 | processes = f.readlines() 212 | 213 | # Parse ps output 214 | for line in processes: 215 | if line.strip().startswith("PID") or len(line) == 0: 216 | continue 217 | parts = re.split(r'\s+', line.strip(), 5) 218 | # idx = pid.index(parts[0]) 219 | to_delete = [] # If the command is limited to selected users, we need to delete the other lines 220 | for idx in filter(lambda p: pid[p] == parts[0], range(len(pid))): 221 | if users is not None and parts[1] not in users: 222 | to_delete.append(idx) 223 | continue 224 | user[idx] = parts[1] 225 | cpu[idx] = parts[2] 226 | mem[idx] = parts[3] 227 | time[idx] = parts[4] if "-" not in parts[4] else parts[4].split("-")[0] + " days" 228 | command[idx] = parts[5] 229 | # Delete lines not corresponding to the selected users (if some are selected) 230 | for idx in reversed(sorted(to_delete)): 231 | for field in fields: 232 | del field[idx] 233 | 234 | if len(pid) == 0: 235 | print("| " + no_running_process + " " * (73 - len(no_running_process)) + " |") 236 | sys.exit() 237 | 238 | max_pid_length = max(5, max([len(x) for x in pid])) 239 | format = ("| %3s %" + str(max_pid_length) + "s %8s %8s %5s %5s %9s %-" + str(command_length) + "." + str(command_length) + "s |") 240 | 241 | line = format % ( 242 | "GPU", "PID", "USER", "GPU MEM", "%CPU", "%MEM", "TIME", "COMMAND" 243 | ) 244 | 245 | print("+" + ("-" * (len(line) - 2)) + "+") 246 | 247 | print(line) 248 | 249 | for i in range(len(pid)): 250 | print(format % ( 251 | gpu_num[i], 252 | pid[i], 253 | user[i], 254 | gpu_mem[i], 255 | cpu[i], 256 | mem[i], 257 | time[i], 258 | command[i] 259 | )) 260 | 261 | print("+" + ("-" * (len(line) - 2)) + "+") 262 | --------------------------------------------------------------------------------