├── .util ├── requirements.txt ├── setup.sh ├── setup.ps1 ├── start.py ├── common.ps1 └── common.sh ├── smf_explorer └── .gitkeep ├── .pre-commit-search-and-replace.yaml ├── Notebooks ├── Tutorial │ ├── Tutorial 3 Screenshot 1.png │ ├── README.ipynb │ ├── Skeleton Notebook.ipynb │ ├── Tutorial 5 - Interactive Notebooks.ipynb │ ├── Tutorial 4 - Visualizing.ipynb │ └── Tutorial 1 - Basics.ipynb └── Reports │ └── LPAR Topology Report.ipynb ├── setup.bat ├── start.bat ├── .github └── ISSUE_TEMPLATE │ └── bug_report.md ├── .pre-commit-config.yaml ├── README.md ├── start ├── setup ├── setup.ps1 ├── start.ps1 ├── .gitignore └── LICENSE /.util/requirements.txt: -------------------------------------------------------------------------------- 1 | smfexplorer[jupyter,jupyter-lab]>=1.0.0 2 | -------------------------------------------------------------------------------- /smf_explorer/.gitkeep: -------------------------------------------------------------------------------- 1 | Directory for downloaded SMF Explorer wheels (.whl files). 2 | -------------------------------------------------------------------------------- /.pre-commit-search-and-replace.yaml: -------------------------------------------------------------------------------- 1 | - search: '/DATASET\s*=\s*\\".*\\"/' 2 | replacement: 'DATASET = \"YOUR.SMF.DATA\"' 3 | extended: true 4 | -------------------------------------------------------------------------------- /Notebooks/Tutorial/Tutorial 3 Screenshot 1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/IBM-SMF-Explorer/HEAD/Notebooks/Tutorial/Tutorial 3 Screenshot 1.png -------------------------------------------------------------------------------- /setup.bat: -------------------------------------------------------------------------------- 1 | :: 2 | :: Copyright 2022 IBM Corporation 3 | :: 4 | :: Licensed under the Apache License, Version 2.0 (the "License"); 5 | :: you may not use this file except in compliance with the License. 6 | :: You may obtain a copy of the License at 7 | :: 8 | :: http://www.apache.org/licenses/LICENSE-2.0 9 | :: 10 | :: Unless required by applicable law or agreed to in writing, software 11 | :: distributed under the License is distributed on an "AS IS" BASIS, 12 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | :: See the License for the specific language governing permissions and 14 | :: limitations under the License. 15 | :: 16 | 17 | @ECHO OFF 18 | PowerShell -NoProfile -ExecutionPolicy Bypass -Command "& './setup.ps1' %*" 19 | -------------------------------------------------------------------------------- /start.bat: -------------------------------------------------------------------------------- 1 | :: 2 | :: Copyright 2022 IBM Corporation 3 | :: 4 | :: Licensed under the Apache License, Version 2.0 (the "License"); 5 | :: you may not use this file except in compliance with the License. 6 | :: You may obtain a copy of the License at 7 | :: 8 | :: http://www.apache.org/licenses/LICENSE-2.0 9 | :: 10 | :: Unless required by applicable law or agreed to in writing, software 11 | :: distributed under the License is distributed on an "AS IS" BASIS, 12 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | :: See the License for the specific language governing permissions and 14 | :: limitations under the License. 15 | :: 16 | 17 | @ECHO OFF 18 | PowerShell -NoProfile -ExecutionPolicy Bypass -Command "& './start.ps1' %*" 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS and Version: [e.g. Windows 10] 28 | - Python Version: [e.g. 3.8] 29 | 30 | **SMF Explorer Info** 31 | ``` 32 | # Output of print(smfexplorer.info()) 33 | ``` 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.3.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - id: check-executables-have-shebangs 9 | - id: end-of-file-fixer 10 | - id: requirements-txt-fixer 11 | - id: no-commit-to-branch 12 | args: [--branch, main] 13 | - id: check-case-conflict 14 | - id: check-merge-conflict 15 | - repo: https://github.com/psf/black 16 | rev: 22.8.0 17 | hooks: 18 | - id: black-jupyter 19 | - repo: https://github.com/datarootsio/databooks 20 | rev: 1.0.5 21 | hooks: 22 | - id: databooks-meta 23 | args: 24 | [ 25 | --overwrite, 26 | --rm-outs, 27 | --nb-meta-keep, 28 | kernelspec, 29 | --cell-meta-keep, 30 | autorun, 31 | --cell-meta-keep, 32 | jupyter, 33 | ] 34 | - repo: https://github.com/mattlqx/pre-commit-search-and-replace 35 | rev: v1.0.5 36 | hooks: 37 | - id: search-and-replace 38 | types: [jupyter] 39 | -------------------------------------------------------------------------------- /.util/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Copyright 2022 IBM Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -eou pipefail 20 | 21 | pip_default_opts="--no-cache-dir --no-input --disable-pip-version-check --find-links ./smf_explorer/" 22 | 23 | # Upgrade pip because of issues in MacOS BigSur 24 | # python -m pip install $pip_default_opts --upgrade pip setuptools wheel 25 | 26 | # Base install 27 | pip install $pip_default_opts --upgrade -r $REQUIREMENTS_FILE --upgrade-strategy eager 28 | 29 | if test -f requirements.txt;then 30 | pip install $pip_default_opts --upgrade -r requirements.txt --upgrade-strategy eager 31 | fi 32 | -------------------------------------------------------------------------------- /.util/setup.ps1: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Copyright 2022 IBM Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | $ErrorActionPreference = "Stop" 19 | 20 | $pip_default_opts = @("--no-cache-dir", "--disable-pip-version-check", "--no-input", "--find-links", "./smf_explorer/") 21 | 22 | # Upgrade pip because of issues in MacOS BigSur 23 | python -m pip install $pip_default_opts --upgrade pip setuptools wheel 24 | 25 | # Base install 26 | pip install $pip_default_opts --upgrade -r $REQUIREMENTS_FILE --upgrade-strategy "eager" 27 | 28 | if (Test-Path -Path "requirements.txt") { 29 | pip install $pip_default_opts --upgrade -r requirements.txt --upgrade-strategy "eager" 30 | } 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IBM SMF Explorer 2 | 3 | IBM® SMF Explorer is a Python framework to fetch SMF data into [Pandas](https://pandas.pydata.org/) DataFrames. 4 | It uses the z/OS® Data Gatherer Rest API Services to access data directly from your SMF data sets. 5 | 6 | ## Getting Started 7 | 8 | ### Requirements 9 | 10 | - z/OS Data Gatherer Rest API Services 11 | - Python 3.8 or later installed on your workstation 12 | 13 | To get started with IBM SMF Explorer follow the official [documentation](https://ibm.github.io/IBM-SMF-Explorer/) 14 | When you have done the installation, take a look at the Tutorial notebooks in the `Tutorial` directory. 15 | 16 | 17 | ## Reporting issues and asking questions 18 | 19 | If you want to report a bug, feel free to open an [issue](https://github.com/IBM/IBM-SMF-Explorer/issues). 20 | If you have questions, ideas, or want to share what you did, you can use the GitHub [Discussions](https://github.com/IBM/IBM-SMF-Explorer/discussions) feature. 21 | 22 | ## Contributing 23 | 24 | We want to encourage you to contribute Notebooks or ideas. 25 | We will create a complete Contribution Guide in the future to get you started. 26 | If you want to contribute code in the meantime, be aware that you are required to sign of your contributions to declare you accept the [DCO](https://developercertificate.org/). 27 | -------------------------------------------------------------------------------- /start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Copyright 2022 IBM Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -eou pipefail 20 | 21 | # Ensure script runs in correct directory 22 | cd $(dirname $0) 23 | 24 | # Load common setup 25 | source ./.util/common.sh 26 | 27 | parse_args $* 28 | 29 | # Switch to windows script 30 | if [[ "$MACHINE" = "Windows" ]]; then 31 | exec ./start.bat $($ARG_RESET && echo -Reset) 32 | fi 33 | 34 | # Pre flight checks 35 | if test ! -f ".util/.setup.sha1"; then 36 | echo -e "${RED}It looks like there is no valid setup. Run './setup' to install an environment${NC}" 37 | exit 1 38 | fi 39 | if ! check_checksums; then 40 | echo -e "${ORANGE}There are updates to some configuration files. Run './setup' again to reinstall the environment${NC}" 41 | if ! prompt_user_yes_or_no "Do you want to continue without updating" "N"; then 42 | exit 43 | fi 44 | fi 45 | 46 | if test ! -d .setup_venv;then 47 | echo -e "${RED}No environment found. Run './setup' to install the environment${NC}" 48 | exit 1 49 | fi 50 | 51 | activate_environment 52 | 53 | python .util/start.py $* 54 | -------------------------------------------------------------------------------- /setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Copyright 2022 IBM Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | set -eou pipefail 20 | 21 | # Ensure script runs in correct directory 22 | cd $(dirname $0) 23 | 24 | # Load common setup 25 | source ./.util/common.sh 26 | 27 | parse_args $* 28 | 29 | # Switch to windows script 30 | if [[ "$MACHINE" = "Windows" ]]; then 31 | exec ./setup.bat $($ARG_RESET && echo -Reset) 32 | fi 33 | 34 | find_python_version 35 | 36 | # Run setup 37 | 38 | if $ARG_RESET; then 39 | echo -e "${BLUE}Recreating virtual environment${NC}..." 40 | $PY -m venv --copies --clear --upgrade-deps $ENVIRONMENT_PATH 41 | elif [[ -d "$ENVIRONMENT_PATH" ]];then 42 | echo -e "${BLUE}Updating virtual environment${NC}..." 43 | $PY -m venv --copies --upgrade --upgrade-deps $ENVIRONMENT_PATH 44 | else 45 | echo -e "${BLUE}Creating virtual environment${NC}..." 46 | $PY -m venv --copies --upgrade-deps $ENVIRONMENT_PATH 47 | fi 48 | 49 | activate_environment 50 | 51 | echo -e "${BLUE}Running basic installation${NC}..." 52 | .util/setup.sh 53 | 54 | # Store Checksums for update warning 55 | write_checksums 56 | 57 | echo -e "${BLUE}Done${NC}" 58 | -------------------------------------------------------------------------------- /setup.ps1: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pwsh 2 | param( 3 | [Switch]$Reset 4 | ) 5 | 6 | # 7 | # Copyright 2022 IBM Corporation 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | # Setup Environment 23 | $ErrorActionPreference = "Stop" 24 | 25 | # Changing into working directory 26 | Set-Location -Path $PSScriptRoot 27 | 28 | # Load Common Functions 29 | . ./.util/common.ps1 30 | 31 | FindPythonVersion 32 | 33 | # Run setup 34 | if ($ARG_RESET) { 35 | Write-Host "Recreating virtual environment..." -ForegroundColor Blue 36 | if (-Not(Invoke-Expression $PY" -m venv --copies --clear --upgrade-deps "$ENVIRONMENT_PATH';$?')) { 37 | exit 1 38 | } 39 | } 40 | elseif (Test-Path -Path $ENVIRONMENT_PATH -PathType Container) { 41 | Write-Host "Updating virtual environment..." -ForegroundColor Blue 42 | if (-Not(Invoke-Expression $PY" -m venv --copies --upgrade --upgrade-deps "$ENVIRONMENT_PATH';$?')) { 43 | exit 1 44 | } 45 | } 46 | else { 47 | Write-Host "Creating virtual environment..." -ForegroundColor Blue 48 | if (-Not(Invoke-Expression $PY" -m venv --copies --upgrade-deps "$ENVIRONMENT_PATH';$?')) { 49 | exit 1 50 | } 51 | } 52 | 53 | ActivateEnvironment 54 | 55 | try { 56 | Write-Host "Running basic installation..." -ForegroundColor Blue 57 | .\.util\setup.ps1 58 | } 59 | finally { 60 | deactivate 61 | } 62 | 63 | WriteChecksums 64 | -------------------------------------------------------------------------------- /start.ps1: -------------------------------------------------------------------------------- 1 | param( 2 | [Switch]$Reset 3 | ) 4 | 5 | # 6 | # Copyright 2022 IBM Corporation 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | # Setup Environment 22 | $ErrorActionPreference = "Stop" 23 | 24 | #Changing into working directory 25 | Set-Location -Path $PSScriptRoot 26 | 27 | # Load Common Functions 28 | . ./.util/common.ps1 29 | 30 | # Pre flight checks 31 | if (!(Test-Path -Path ".\.util\.setup_sha.xml")) { 32 | Write-Host "It looks like there is no valid setup. Run './setup.bat' to install an environment" -ForegroundColor Red 33 | exit 1 34 | } 35 | if (!(CheckChecksums)) { 36 | Write-Host "There are updates to some configuration files. Run './setup.bat' again to reinstall the environment" -ForegroundColor Yellow 37 | if (!(PrompUserYesOrNo -Prompt "Do you want to continue without updating" -Default "N")) { 38 | exit 39 | } 40 | } 41 | 42 | 43 | if ( -Not (TestCommand python)) { 44 | Write-Host "Python not found! Make sure that Python is installed and your PATH is set up correctly" -ForegroundColor Red 45 | exit 1 46 | } 47 | 48 | if (!(Test-Path -Path $ENVIRONMENT_PATH -PathType Container)) { 49 | Write-Host "No environment found. Run './setup.bat' to install environment" -ForegroundColor Red 50 | exit 1 51 | } 52 | 53 | ActivateEnvironment 54 | 55 | try { 56 | $arguments = $args 57 | python .\.util\start.py $arguments 58 | } 59 | finally { 60 | deactivate 61 | } 62 | -------------------------------------------------------------------------------- /Notebooks/Tutorial/README.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 5, 4 | "metadata": { 5 | "kernelspec": { 6 | "display_name": "Python 3 (ipykernel)", 7 | "language": "python", 8 | "name": "python3" 9 | } 10 | }, 11 | "cells": [ 12 | { 13 | "metadata": {}, 14 | "source": [ 15 | "# Welcome to the *IBM SMF Explorer* Tutorials!\n", 16 | "\n", 17 | "The Tutorials are split into different sections each with its own notebook.\n", 18 | "\n", 19 | "- **Tutorial 1 - Basics:** SMF data fetching and core package functionality. \n", 20 | "- **Tutorial 2 - Filtering and Sorting:** data reduction, filtering, and sorting using the build-in expression language.\n", 21 | "- **Tutorial 3 - Working with the data:** data processing with *IBM SMF Explorer* and Pandas.\n", 22 | "- **Tutorial 4 - Visualization:** data visualization using plotly.\n", 23 | "- **Tutorial 5 - Interactive Notebooks (Advanced):** creating interactive Notebooks.\n", 24 | "\n", 25 | "All tutorials have cells containing code in them.\n", 26 | "When you come across such a code cell you should execute it by selecting it and pressing `Ctrl`+`Enter`.\n", 27 | "If you ever face issues you can reset the current Notebook with the _'Reset the kernel'_ button in the task bar above.\n", 28 | "\n", 29 | "### What SMF data to use?\n", 30 | "\n", 31 | "In the tutorials you will often find the code line `DATASET = \"YOUR.SMF.DATA\"`.\n", 32 | "You should replace `YOUR.SMF.DATA` with a data set you have access to. The tutorials make use of SMF types 70 and 72.\n", 33 | "Note that, escpecially for **Tutorial 2**, you will find operations that are dependent on the data you use.\n", 34 | "Feel free to change the code, but keep in mind that other parts of the tutorial might be affected.\n", 35 | "\n", 36 | "If you have never worked with Jupyter Notebooks or Python, before exploring the Tutorials, have a look into the following resources:\n", 37 | "- [The Jupyter Notebook Introduction](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html)\n", 38 | "- [JupyterLab Documentation](https://jupyterlab.readthedocs.io/en/stable/)\n", 39 | "- [Python](https://www.learnpython.org)\n", 40 | "\n", 41 | "### Resources for data analysis\n", 42 | "- [pandas documentation](https://pandas.pydata.org/docs/)\n", 43 | "- [Pandas Tutorial: DataFrames in Python](https://www.datacamp.com/community/tutorials/pandas-tutorial-dataframe-python)\n", 44 | "- [numpy documentation](https://numpy.org/doc/stable/)\n", 45 | "\n" 46 | ], 47 | "cell_type": "markdown" 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Excluding all notebooks. Shared notebooks need to be force added. 3 | /Notebooks/ 4 | 5 | # Exclude SMF Explorer .whl files 6 | /smf_explorer/ 7 | 8 | # Exclude setup checksums 9 | .util/.setup_sha.xml 10 | .util/.setup.sha1 11 | 12 | # Exclude Setup virt env 13 | /.setup_venv 14 | 15 | # Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,jupyternotebooks,windows,linux,macos 16 | # Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,jupyternotebooks,windows,linux,macos 17 | 18 | ### JupyterNotebooks ### 19 | # gitignore template for Jupyter Notebooks 20 | # website: http://jupyter.org/ 21 | 22 | .ipynb_checkpoints 23 | */.ipynb_checkpoints/* 24 | 25 | # IPython 26 | profile_default/ 27 | ipython_config.py 28 | 29 | # Remove previous ipynb_checkpoints 30 | # git rm -r .ipynb_checkpoints/ 31 | 32 | ### Linux ### 33 | *~ 34 | 35 | # temporary files which can be created if a process still has a handle open of a deleted file 36 | .fuse_hidden* 37 | 38 | # KDE directory preferences 39 | .directory 40 | 41 | # Linux trash folder which might appear on any partition or disk 42 | .Trash-* 43 | 44 | # .nfs files are created when an open file is removed but is still being accessed 45 | .nfs* 46 | 47 | ### macOS ### 48 | # General 49 | .DS_Store 50 | .AppleDouble 51 | .LSOverride 52 | 53 | # Icon must end with two \r 54 | Icon 55 | 56 | 57 | # Thumbnails 58 | ._* 59 | 60 | # Files that might appear in the root of a volume 61 | .DocumentRevisions-V100 62 | .fseventsd 63 | .Spotlight-V100 64 | .TemporaryItems 65 | .Trashes 66 | .VolumeIcon.icns 67 | .com.apple.timemachine.donotpresent 68 | 69 | # Directories potentially created on remote AFP share 70 | .AppleDB 71 | .AppleDesktop 72 | Network Trash Folder 73 | Temporary Items 74 | .apdisk 75 | 76 | ### macOS Patch ### 77 | # iCloud generated files 78 | *.icloud 79 | 80 | ### VisualStudioCode ### 81 | .vscode/* 82 | !.vscode/settings.json 83 | !.vscode/tasks.json 84 | !.vscode/launch.json 85 | !.vscode/extensions.json 86 | !.vscode/*.code-snippets 87 | 88 | # Local History for Visual Studio Code 89 | .history/ 90 | 91 | # Built Visual Studio Code Extensions 92 | *.vsix 93 | 94 | ### VisualStudioCode Patch ### 95 | # Ignore all local history of files 96 | .history 97 | .ionide 98 | 99 | # Support for Project snippet scope 100 | .vscode/*.code-snippets 101 | 102 | # Ignore code-workspaces 103 | *.code-workspace 104 | 105 | ### Windows ### 106 | # Windows thumbnail cache files 107 | Thumbs.db 108 | Thumbs.db:encryptable 109 | ehthumbs.db 110 | ehthumbs_vista.db 111 | 112 | # Dump file 113 | *.stackdump 114 | 115 | # Folder config file 116 | [Dd]esktop.ini 117 | 118 | # Recycle Bin used on file shares 119 | $RECYCLE.BIN/ 120 | 121 | # Windows Installer files 122 | *.cab 123 | *.msi 124 | *.msix 125 | *.msm 126 | *.msp 127 | 128 | # Windows shortcuts 129 | *.lnk 130 | 131 | # End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,jupyternotebooks,windows,linux,macos 132 | -------------------------------------------------------------------------------- /.util/start.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022 IBM Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import subprocess 18 | import os 19 | import signal 20 | import re 21 | import argparse 22 | import getpass 23 | 24 | os_var_connection_string = "SMFEXPLORER_CONNECTION_STRING" 25 | os_var_user_name = "SMFEXPLORER_USERNAME" 26 | os_var_user_password = "SMFEXPLORER_PASSWORD" 27 | 28 | 29 | def select_server(no_verify_tls: bool = False): 30 | if os_var_connection_string in os.environ: 31 | print("Connection string found in system environment") 32 | connection_string = os.environ[os_var_connection_string] 33 | else: 34 | connection_string = input(f"What is your connection string?: ") 35 | connection_string = connection_string.strip() 36 | 37 | if os_var_user_name in os.environ: 38 | print("User name found in system environment") 39 | user_name = os.environ[os_var_user_name] 40 | else: 41 | user_name = input("Enter your username: ") 42 | user_name = user_name.strip() 43 | 44 | if os_var_user_password in os.environ: 45 | print("User password found in system environment") 46 | user_password = os.environ[os_var_user_password] 47 | else: 48 | user_password = getpass.getpass("Enter your password: ") 49 | 50 | match = re.match( 51 | r"^http[s]?:\/\/(?:[a-zA-Z0-9-]|[.])+(?::[0-9]+)?(?:[a-zA-Z0-9]+)(?:/.*)*$", 52 | connection_string, 53 | ) 54 | if match: 55 | 56 | return f"mode=dgapi;url={connection_string};verify_ssl={'false' if no_verify_tls else 'true'};username={user_name};password={user_password}" 57 | 58 | print("Invalid connection string") 59 | exit(1) 60 | 61 | 62 | def start_jupyter(connection_string, log: bool = False): 63 | sig_stopper = signal.signal(signal.SIGINT, signal.SIG_IGN) 64 | 65 | env = os.environ.copy() 66 | env["SMFPY_CONNECTION_STRING"] = connection_string 67 | 68 | run_list = ( 69 | [ 70 | "jupyter", 71 | "lab", 72 | "--notebook-dir=Notebooks", 73 | "--LabApp.use_redirect_file=False", 74 | ] 75 | if os.name == "nt" 76 | else [f"jupyter lab --notebook-dir=Notebooks --LabApp.use_redirect_file=False"] 77 | ) 78 | 79 | print("Starting Jupyter Lab...") 80 | 81 | subprocess.run( 82 | run_list, shell=True, env=env, stderr=None if log else subprocess.DEVNULL 83 | ) 84 | 85 | signal.signal(signal.SIGINT, sig_stopper) 86 | 87 | 88 | parser = argparse.ArgumentParser() 89 | 90 | parser.add_argument("--log-output", action="store_true") 91 | parser.add_argument("--no-verify-tls", default=False) 92 | parser.add_argument("---cacert", help='Path to the TLS CA certificate file for the specified host') 93 | 94 | def main(): 95 | try: 96 | ns = parser.parse_args() 97 | custom_ca_path = ns.cacert 98 | if custom_ca_path: 99 | os.environ['REQUESTS_CA_BUNDLE'] = custom_ca_path 100 | 101 | # Select Server to use 102 | connection_string = select_server(ns.no_verify_tls) 103 | 104 | start_jupyter(connection_string, log=ns.log_output) 105 | except KeyboardInterrupt: 106 | print("") 107 | pass 108 | 109 | 110 | if __name__ == "__main__": 111 | main() 112 | -------------------------------------------------------------------------------- /.util/common.ps1: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Copyright 2022 IBM Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Settings 19 | $INSTALL_FILES = @(".util/common.ps1", ".util/setup.sh", ".util/setup.ps1", ".util/requirements.txt", "setup") 20 | $ENVIRONMENT_PATH = ".setup_venv" 21 | $REQUIREMENTS_FILE = ".util/requirements.txt" 22 | 23 | # Arguments 24 | $ARG_RESET = $Reset 25 | 26 | $Env:ENVIRONMENT_PATH = $ENVIRONMENT_PATH 27 | $Env:REQUIREMENTS_FILE = $REQUIREMENTS_FILE 28 | 29 | # Load dotenv file 30 | if ((Test-Path -Path "./.env.ps1" -PathType Leaf)) { 31 | . ./.env.ps1 32 | } 33 | 34 | # Util Functions 35 | function TestCommand ($command) { 36 | try { 37 | Get-Command $command 38 | return $True 39 | } 40 | catch { 41 | return $False 42 | } 43 | } 44 | 45 | function PrompUserYesOrNo($Prompt, $Default) { 46 | $prompt_helper = "(y/n)" 47 | if ($Default -eq "Y") { $prompt_helper = "(Y/n)" } 48 | elseif ($Default -eq "N") { $prompt_helper = "(y/N)" } 49 | while ($True) { 50 | $answer = Read-Host "$Prompt ${prompt_helper}? " 51 | switch -regex ($answer) { 52 | "[Yy].*" { return $True } 53 | "[Nn].*" { return $False } 54 | Default { 55 | if ( $Default -eq "Y" ) { return $True } 56 | elseif ($Default -eq "N") { return $False } 57 | else { 58 | Write-Host "Please select yes or no." 59 | } 60 | } 61 | } 62 | } 63 | 64 | } 65 | 66 | $PY = $null 67 | # Find suitable python version 68 | function FindPythonVersion() { 69 | $python_candidates = @("python", "python3", "python", "py", "python3.9", "python3.8") 70 | $PYTHON_VERSION_CHECK_CMD = "import platform;major, minor, _ = platform.python_version_tuple();exit(1) if int(major) < 3 or int(minor) < 8 else ();exit(2) if '64' not in platform.architecture()[0] else ();" 71 | foreach ($candidate in $python_candidates) { 72 | if (TestCommand $candidate) { 73 | try { 74 | if (Invoke-Expression ($candidate + " -c `"${PYTHON_VERSION_CHECK_CMD}`" 2>&1 | out-null " + ';$?')) { 75 | $script:PY = $candidate 76 | break 77 | } 78 | } 79 | catch {} 80 | } 81 | } 82 | 83 | if ($null -eq $PY) { 84 | Write-Host "No appropriate Python found! Make sure that Python 3.8 64bit or newer is installed and your PATH is setup correctly" -ForegroundColor Red 85 | exit 1 86 | } 87 | } 88 | 89 | function WriteChecksums() { 90 | Get-FileHash -Path $INSTALL_FILES | Export-Clixml .util/.setup_sha.xml 91 | } 92 | 93 | function CheckChecksums() { 94 | $checksums = (Import-Clixml .\.util\.setup_sha.xml) 95 | 96 | foreach ($checksum in $checksums) { 97 | if ((Get-FileHash -Path $checksum.Path).Hash -ne $checksum.Hash) { 98 | return $False 99 | } 100 | } 101 | 102 | return $True 103 | } 104 | 105 | function ActivateEnvironment() { 106 | Write-Host "Activating environment..." -ForegroundColor Blue 107 | . $ENVIRONMENT_PATH\*\Activate.ps1 108 | } 109 | 110 | if (!([Environment]::GetEnvironmentVariable('SMF_EXPLORER_PATH'))) { 111 | $Env:SMF_EXPLORER_PATH = "../IBM-SMF-Explorer" 112 | } 113 | -------------------------------------------------------------------------------- /.util/common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Copyright 2022 IBM Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # Colors 20 | RED="$(tput setaf 1)" 21 | BLUE="$(tput setaf 4)" 22 | ORANGE="$(tput setaf 3)" 23 | BOLD="$(tput bold)" 24 | NC="$(tput sgr0)" 25 | 26 | # Check OS Type 27 | #system="$(uname -s)" 28 | case "$(uname -s)" in 29 | CYGWIN*) MACHINE=Windows;; 30 | MINGW*) MACHINE=Windows;; 31 | MSYS*) MACHINE=Windows;; 32 | Darwin*) MACHINE=Darwin;; 33 | *) MACHINE=Unix;; 34 | esac 35 | 36 | # Settings 37 | INSTALL_FILES=(".util/common.sh" ".util/setup.sh" ".util/requirements.txt" "setup") 38 | ENVIRONMENT_PATH=".setup_venv" 39 | REQUIREMENTS_FILE=".util/requirements.txt" 40 | 41 | # Arguments 42 | ARG_RESET=false 43 | 44 | # Load dotenv file 45 | if [[ -f "./.env" ]];then 46 | set -a 47 | source ./.env 48 | set +a 49 | fi 50 | 51 | parse_args() { 52 | while [[ $# -gt 0 ]]; do 53 | case $1 in 54 | -r|--reset) 55 | ARG_RESET=true 56 | shift 57 | ;; 58 | *) 59 | shift 60 | ;; 61 | esac 62 | done 63 | } 64 | 65 | # Util Functions 66 | 67 | command_exists() { 68 | type "$1" &> /dev/null ; 69 | } 70 | 71 | prompt_user_yes_or_no() { 72 | if [[ ${2:-} = "Y" ]]; then prompt_help="(${BOLD}Y${NC}/n)" 73 | elif [[ ${2:-} = "N" ]]; then prompt_help="(y/${BOLD}N${NC})" 74 | else prompt_help="(y/n)"; fi 75 | 76 | while true; do 77 | read -p "$1 $prompt_help?: " yn 78 | case $yn in 79 | [Yy]* ) return 0;; 80 | [Nn]* ) return 1;; 81 | * ) 82 | if [[ ${2:-} = "Y" ]];then return 0; 83 | elif [[ ${2:-} = "N" ]]; then return 1; 84 | else echo "Please select ${BOLD}yes${NC} or ${BOLD}no${NC}."; fi 85 | ;; 86 | esac 87 | done 88 | } 89 | 90 | # Find suitable python version 91 | find_python_version() { 92 | PY="" 93 | local CANDIDATES=("python" "python3" "python3.9" "python3.8") 94 | local PYTHON_VERSION_CHECK_CMD="import platform;major, minor, _ = platform.python_version_tuple();exit(1) if int(major) < 3 or int(minor) < 8 else ();exit(2) if '64' not in platform.architecture()[0] else ();" 95 | for candidate in "${CANDIDATES[@]}"; do 96 | if command_exists $candidate; then 97 | if $candidate -c "$PYTHON_VERSION_CHECK_CMD";then 98 | PY="$candidate" 99 | break 100 | fi 101 | fi 102 | done 103 | if [[ $PY = "" ]]; then 104 | echo -e "${RED}No appropriate Python found! Make sure that Python 3.8 64bit or newer is installed and your PATH is setup correctly${NC}" 105 | exit 1 106 | fi 107 | } 108 | 109 | # Write Checksums 110 | write_checksums() { 111 | if [[ "$MACHINE" = "Darwin" ]];then 112 | shasum ${INSTALL_FILES[*]} > .util/.setup.sha1 113 | else 114 | sha1sum ${INSTALL_FILES[*]} > .util/.setup.sha1 115 | fi 116 | } 117 | 118 | check_checksums() { 119 | if [[ "$MACHINE" = "Darwin" ]]; then 120 | shasum -c .util/.setup.sha1 &> /dev/null 121 | return $? 122 | else 123 | sha1sum -c .util/.setup.sha1 &> /dev/null 124 | return $? 125 | fi 126 | } 127 | 128 | activate_environment() { 129 | echo -e "${BLUE}Activating environment${NC}..." 130 | source $ENVIRONMENT_PATH/bin/activate 131 | } 132 | 133 | export ENVIRONMENT_PATH 134 | export REQUIREMENTS_FILE 135 | 136 | if [[ -z "${SMF_EXPLORER_PATH+x}" ]]; then 137 | SMF_EXPLORER_PATH="../IBM-SMF-Explorer" 138 | fi 139 | 140 | export SMF_EXPLORER_PATH 141 | -------------------------------------------------------------------------------- /Notebooks/Tutorial/Skeleton Notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 4, 4 | "metadata": { 5 | "kernelspec": { 6 | "display_name": "Python 3 (ipykernel)", 7 | "language": "python", 8 | "name": "python3" 9 | } 10 | }, 11 | "cells": [ 12 | { 13 | "metadata": {}, 14 | "source": [ 15 | "# Skeleton Notebook\n", 16 | "\n", 17 | "This Jupyter Notebook serves as a starting point for your future Notebooks. You can copy this Notebook to your directory, rename it, and extend it. " 18 | ], 19 | "cell_type": "markdown" 20 | }, 21 | { 22 | "metadata": {}, 23 | "source": [ 24 | "# imports\n", 25 | "import smfexplorer\n", 26 | "from smfexplorer.fields import SMF70S1, SMF72S3 # change or add fields of interest here\n", 27 | "from smfexplorer import names\n", 28 | "\n", 29 | "from plotly import express as px # for visualization\n", 30 | "\n", 31 | "# for data processing\n", 32 | "import pandas as pd\n", 33 | "import numpy as np" 34 | ], 35 | "cell_type": "code", 36 | "outputs": [], 37 | "execution_count": null 38 | }, 39 | { 40 | "metadata": {}, 41 | "source": [ 42 | "# define Context\n", 43 | "DATASET = \"YOUR.SMF.DATA\"\n", 44 | "ctx = smfexplorer.new_context(DATASET)" 45 | ], 46 | "cell_type": "code", 47 | "outputs": [], 48 | "execution_count": null 49 | }, 50 | { 51 | "metadata": {}, 52 | "source": [ 53 | "# ------------------------------------------------#\n", 54 | "# Fetch the data (select one of 3 methods)\n", 55 | "# ------------------------------------------------#\n", 56 | "\n", 57 | "# 1. using request() method\n", 58 | "df = ctx.request([RECORD.FIELD1, RECORD.FIELD2]).run() # add fields\n", 59 | "# example: df = ctx.request([SMF70S1.timestamp, SMF70S1.sid]).run()\n", 60 | "\n", 61 | "# 2. using samples()\n", 62 | "df = ctx.samples.SAMPLENAME().run() # add sample name\n", 63 | "# example: df = ctx.samples.lpar_information().run()\n", 64 | "\n", 65 | "# 3. using samples() with display\n", 66 | "df = ctx.samples.SAMPLENAME().run(\n", 67 | " display=[RECORD.FIELD1]\n", 68 | ") # add sample name and field name in display\n", 69 | "# example: df = ctx.samples.lpar_information().run(display=[SMF70S1.capactiy_group_member])\n", 70 | "\n", 71 | "\n", 72 | "# if you want to reduce the data before fetching, you can use where():\n", 73 | "df = ctx.samples.SAMPLENAME().where((CONDITION1) & (CONDITION2)).run()\n", 74 | "# example: ctx.samples.lpar_information().where((SMF70S1.lpar_name == SMF70S1.system_name) & (SMF70S1.lpar_cpu_count > 5)).run()" 75 | ], 76 | "cell_type": "code", 77 | "outputs": [], 78 | "execution_count": null 79 | }, 80 | { 81 | "metadata": {}, 82 | "source": [ 83 | "# display the data\n", 84 | "display(df)" 85 | ], 86 | "cell_type": "code", 87 | "outputs": [], 88 | "execution_count": null 89 | }, 90 | { 91 | "metadata": {}, 92 | "source": [ 93 | "# -----------#\n", 94 | "# Filter\n", 95 | "# -----------#\n", 96 | "\n", 97 | "# filter out date-time range\n", 98 | "df = df.loc[\n", 99 | " (df[\"timestamp\"] > \"YYYY-MM-DD HH:MM:SS\")\n", 100 | " & (df[\"timestamp\"] < \"YYYY-MM-DD HH:MM:SS\")\n", 101 | "]\n", 102 | "\n", 103 | "# Get all Systems used in the dump\n", 104 | "systems = df[\n", 105 | " names(RECORD.sid)\n", 106 | "].unique() # insted of sid you can use another field, to get at overview (e.g., lpar_name, cpu_type, etc.)\n", 107 | "print(systems)\n", 108 | "\n", 109 | "# Select one system\n", 110 | "df = df[df[names(RECORD.sid)] == \"SYSTEM_NAME\"]\n", 111 | "\n", 112 | "# Calculate sum or average\n", 113 | "# get the average of FIELD1\n", 114 | "avg = df[names(RECORD.FIELD1)].mean()\n", 115 | "print(\"Average FIELD1 is \", avg)\n", 116 | "\n", 117 | "# get the sum of FIELD1\n", 118 | "sum_f = df[names(RECORD.FIELD1)].sum()\n", 119 | "print(\"Sum FIELD1 is \", sum_f)" 120 | ], 121 | "cell_type": "code", 122 | "outputs": [], 123 | "execution_count": null 124 | }, 125 | { 126 | "metadata": {}, 127 | "source": [ 128 | "# -----------#\n", 129 | "# Plot\n", 130 | "# -----------#\n", 131 | "\n", 132 | "# plot your data\n", 133 | "# line plot\n", 134 | "plot = px.line(\n", 135 | " df,\n", 136 | " x=names(RECORD.timestamp), # usually x axis depics time\n", 137 | " y=names(RECORD.FIELD1),\n", 138 | " title=\"TITLE\",\n", 139 | " labels={\n", 140 | " \"initial_value\": \"Renamed value\",\n", 141 | " \"initial_value2\": \"Second renamed value\",\n", 142 | " names(RECORD.timestamp): \"Time\",\n", 143 | " },\n", 144 | ")\n", 145 | "display(plot)" 146 | ], 147 | "cell_type": "code", 148 | "outputs": [], 149 | "execution_count": null 150 | }, 151 | { 152 | "metadata": {}, 153 | "source": [ 154 | "# create bar-chart\n", 155 | "bar = px.bar(\n", 156 | " df,\n", 157 | " x=\"timestamp\",\n", 158 | " y=[\"FIELD1\", \"FIELD2\"],\n", 159 | " title=\"Ratio of FIELD1 and FIELD2 over time\",\n", 160 | " labels={\n", 161 | " \"initial_value\": \"Renamed value\",\n", 162 | " \"initial_value2\": \"Second renamed value\",\n", 163 | " names(RECORD.timestamp): \"Time\",\n", 164 | " },\n", 165 | ")\n", 166 | "display(bar)" 167 | ], 168 | "cell_type": "code", 169 | "outputs": [], 170 | "execution_count": null 171 | } 172 | ] 173 | } 174 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Notebooks/Tutorial/Tutorial 5 - Interactive Notebooks.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "2e1a244a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Tutorial 5 - Interactive Notebooks (Advanced)\n", 9 | "\n", 10 | "Welcome to the _Interactive Notebooks Tutorial_.\n", 11 | "You will learn how to create clean Notebooks that allow users to interact with the data with the help of various widgets. \n", 12 | "In this tutorial, we provide examples on how to initialize multiple datasets, how the user can select the system or date from SMF data and get the Notebook updated automatically without rerunning it.\n", 13 | "\n", 14 | "To start, you may refer to [ipywidgets documentation](https://ipywidgets.readthedocs.io/en/latest/).\n", 15 | "\n", 16 | "## Dataset(s) Initialization\n", 17 | "\n", 18 | "For a graphical interface, you need to import the ``smfexplorer.util.jupyter`` module and create an instance of the ``ConfigWidget`` class, providing a **Context** as an argument.\n", 19 | "After the ConfigWidget was created, you can display it by calling the IPython ``display()`` function.\n", 20 | "> **Note**: You can specify a dataset as an argument in the ``new_context()`` function. This will make the text field of the ConfigWidget contain the predefined dataset name. " 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 1, 26 | "id": "1874669e", 27 | "metadata": { 28 | "tags": [] 29 | }, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "application/vnd.jupyter.widget-view+json": { 34 | "model_id": "87470da3391d4e25b7698f4fbe9127f3", 35 | "version_major": 2, 36 | "version_minor": 0 37 | }, 38 | "text/plain": [ 39 | "ConfigWidget(children=(Text(value='', layout=Layout(width='500px'), placeholder='Dataset Name(s)'), HTML(value…" 40 | ] 41 | }, 42 | "metadata": {}, 43 | "output_type": "display_data" 44 | } 45 | ], 46 | "source": [ 47 | "import smfexplorer\n", 48 | "from smfexplorer.fields import SMF70S1\n", 49 | "from smfexplorer import names\n", 50 | "from smfexplorer.util import jupyter\n", 51 | "\n", 52 | "# text field with predefined dataset name\n", 53 | "# ctx = smfexplorer.new_context('YOUR.SMF.DATA')\n", 54 | "\n", 55 | "ctx = smfexplorer.new_context()\n", 56 | "config_widget = jupyter.ConfigWidget(ctx)\n", 57 | "display(config_widget)" 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "id": "d63445e9", 63 | "metadata": {}, 64 | "source": [ 65 | "> **Hint:** In some cases you may want to work with multiple datasets. For example, in one Notebook you want to analyse LPAR utilization from \"*YOUR.SMF.SMF70*\" and Cache-Hits from \"*YOUR.SMF.HIS*\". To do so, you can assign both datasets to one context, separating them with a comma ‘,’ (e.g., ``YOUR.SMF.SMF70,YOUR.SMF.HIS``). When you create a request, *IBM SMF Explorer* fetches the data for all specified datasets and concatenates the results. " 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "id": "d2516d17", 71 | "metadata": {}, 72 | "source": [ 73 | "To make the user-provided dataset available to the next cell, you need to do the following:\n", 74 | " * Reference widget instance with ``@`` and call ``register_output()`` function that automatically reruns the cell when the dataset instance was changed\n", 75 | " * Create a function that takes as an argument dsn (this is our dataset instance)\n", 76 | " \n", 77 | "------\n", 78 | "Let's give it a try!\n", 79 | "\n", 80 | "Enter a dataset name into the textfield above and press the **Init** button.\n", 81 | "The `ConfigWidget` will automatically check the specified dataset for availability and call the `give_me_dsn()` function when a user presses `Init`.\n", 82 | " \n", 83 | "As you see, the output of the cell below is changed when the dataset name is changed. " 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 2, 89 | "id": "fba2e32e", 90 | "metadata": { 91 | "tags": [] 92 | }, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "application/vnd.jupyter.widget-view+json": { 97 | "model_id": "21962753c1904f1e822f499478c24600", 98 | "version_major": 2, 99 | "version_minor": 0 100 | }, 101 | "text/plain": [ 102 | "Output()" 103 | ] 104 | }, 105 | "metadata": {}, 106 | "output_type": "display_data" 107 | } 108 | ], 109 | "source": [ 110 | "@config_widget.register_output()\n", 111 | "def give_me_dsn(dsn, **kwds):\n", 112 | " print(dsn)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "id": "99a99c98", 118 | "metadata": {}, 119 | "source": [ 120 | "You have seen how we can create a **Context** instance and let the user interactively define the datasets for that **Context**. \n", 121 | "However, we know that having dataset names is not enough.\n", 122 | "We want to create a request and fetch a DataFrame. \n", 123 | "Therefore, we define another function that executes a request and returns the fetched DataFrame. \n", 124 | "\n", 125 | "The `name` argument in ``@config_widget.register_output(name=\"df\")`` denotes that the function returns the DataFrame as the name **df**.\n", 126 | "We will see why this is useful in the next cell." 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 3, 132 | "id": "827113b3", 133 | "metadata": { 134 | "tags": [] 135 | }, 136 | "outputs": [ 137 | { 138 | "data": { 139 | "application/vnd.jupyter.widget-view+json": { 140 | "model_id": "b68aa408377a46208052fd7a0d14398a", 141 | "version_major": 2, 142 | "version_minor": 0 143 | }, 144 | "text/plain": [ 145 | "Output()" 146 | ] 147 | }, 148 | "metadata": {}, 149 | "output_type": "display_data" 150 | } 151 | ], 152 | "source": [ 153 | "@config_widget.register_output(name=\"df\")\n", 154 | "def fetch_df(dsn, **kwds):\n", 155 | " df = ctx.samples.lpar_information().run()\n", 156 | " return df" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "id": "1a293029", 162 | "metadata": {}, 163 | "source": [ 164 | "To make use of our **df** DataFrame, we can register the next function not with `config_widget` but with `fetch_df` (the function defined above).\n", 165 | "\n", 166 | "Now, we can see what the `name` argument from `fetch_df` does.\n", 167 | "By default, *IBM SMF Explorer* uses the function `name` as the name for the parameter that we want to pass down the chain.\n", 168 | "The `name` argument makes *IBM SMF Explorer* change `fetch_df` to `df` in `working_with_df`.\n", 169 | "\n", 170 | "This kind of chaining can be repeated with `working_with_df` and any subsequent registered function.\n", 171 | "This allows you to define a flow of operations that should be triggered when the user presses **Init**." 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 4, 177 | "id": "d856b1fc", 178 | "metadata": { 179 | "tags": [] 180 | }, 181 | "outputs": [ 182 | { 183 | "data": { 184 | "application/vnd.jupyter.widget-view+json": { 185 | "model_id": "d6a94c5f0253458e98b24595b504e66b", 186 | "version_major": 2, 187 | "version_minor": 0 188 | }, 189 | "text/plain": [ 190 | "Output()" 191 | ] 192 | }, 193 | "metadata": {}, 194 | "output_type": "display_data" 195 | } 196 | ], 197 | "source": [ 198 | "@fetch_df.register_output()\n", 199 | "def working_with_df(df, **kwds):\n", 200 | " display(df.head())" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "id": "86b4b8df", 206 | "metadata": {}, 207 | "source": [ 208 | "You can see that `working_with_df` was able to render the DataFrame by calling the `display()` function.\n", 209 | "This is the case, because `register_output` was used.\n", 210 | "If you have a function that you know will not display anything to the Notebook, you can use the `register` function instead.\n", 211 | "\n", 212 | "> **Note**: `register_output` creates an `Output` widget that can be dynamically updated.\n", 213 | "> Every time a function that is registered with `register_output` is called, the `Output` widget is cleared and repopulated with the input given to the `display()` function." 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "id": "4831fedc", 219 | "metadata": {}, 220 | "source": [ 221 | "## Creating Widgets\n", 222 | "\n", 223 | "Just having the initial dataframe passed down the chain is a good start.\n", 224 | "To be fully interactive, we sometimes need to react to specifics in the fetched data and ask the user for additional input to provide a useful output.\n", 225 | "The following example shows how to use IPyWidgets together with *IBM SMF Explorer*'s interactive features.\n", 226 | "\n", 227 | "When we are working with SMF data, we often notice that one dataset can contain information from multiple systems/dates/LPARS.\n", 228 | "To minimize data and ease processing, it makes sense to provide filters or selectors that allow the user to select a subset of the data. \n", 229 | "\n", 230 | "Consider the following widgets:\n", 231 | "\n", 232 | " * FloatSlider\n", 233 | " * IntProgress\n", 234 | " * FloatText\n", 235 | " * ToggleButton\n", 236 | " * Checkbox\n", 237 | " * Dropdown\n", 238 | "\n", 239 | "See [here](https://ipywidgets.readthedocs.io/en/stable/examples/Widget%20List.html) for more information\n", 240 | " \n", 241 | "\n", 242 | "We start by importing ``ipywidgets``.\n", 243 | "The next step is to create the widget instance (in our case _ToggleButtons_).\n", 244 | "\n", 245 | "We can now add our widget to `register_output`.\n", 246 | "This will result in any function that registers against `select_name` to be called with the value of the widget whenever the DataFrame changes or the user changes the selection of the widget." 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": 5, 252 | "id": "5c51dfff", 253 | "metadata": { 254 | "tags": [] 255 | }, 256 | "outputs": [ 257 | { 258 | "data": { 259 | "application/vnd.jupyter.widget-view+json": { 260 | "model_id": "ce9436597fd649b4bb128fa604ff0ca3", 261 | "version_major": 2, 262 | "version_minor": 0 263 | }, 264 | "text/plain": [ 265 | "Output()" 266 | ] 267 | }, 268 | "metadata": {}, 269 | "output_type": "display_data" 270 | } 271 | ], 272 | "source": [ 273 | "import ipywidgets as widgets\n", 274 | "\n", 275 | "selection_widget = widgets.ToggleButtons()\n", 276 | "\n", 277 | "\n", 278 | "@fetch_df.register_output(name=\"df\", name_selected=selection_widget)\n", 279 | "def select_name(df, **kwds):\n", 280 | "\n", 281 | " lpar_system_name = df[names(SMF70S1.lpar_system_name)].unique().dropna()\n", 282 | " selection_widget.options = lpar_system_name\n", 283 | "\n", 284 | " display(selection_widget)\n", 285 | "\n", 286 | " selection_widget.value = lpar_system_name[0]\n", 287 | "\n", 288 | " return df" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 6, 294 | "id": "f6da8992", 295 | "metadata": { 296 | "tags": [] 297 | }, 298 | "outputs": [ 299 | { 300 | "data": { 301 | "application/vnd.jupyter.widget-view+json": { 302 | "model_id": "2299442384094675a0cc6c32d157324d", 303 | "version_major": 2, 304 | "version_minor": 0 305 | }, 306 | "text/plain": [ 307 | "Output()" 308 | ] 309 | }, 310 | "metadata": {}, 311 | "output_type": "display_data" 312 | } 313 | ], 314 | "source": [ 315 | "@select_name.register_output()\n", 316 | "def filter_df(df, name_selected, **kwds):\n", 317 | "\n", 318 | " # The best practice is to guard against calls, where df is None or to check the input for validity in general.\n", 319 | " # Returning early from the function will clear the output.\n", 320 | " if df is None:\n", 321 | " return\n", 322 | " print(\"Selected LPAR-system name is: \" + name_selected)\n", 323 | " df = df[df[names(SMF70S1.lpar_system_name)] == name_selected].reset_index(drop=True)\n", 324 | " display(df.head())" 325 | ] 326 | }, 327 | { 328 | "cell_type": "markdown", 329 | "id": "beca7db5", 330 | "metadata": {}, 331 | "source": [ 332 | "If you select another LPAR-system name from the selection, you should see that the table above is redrawn.\n", 333 | "Or if you change the dataset name in the very beginning and press *Init* again the entire output will be redrawn.\n", 334 | "\n", 335 | "> **Note**: Getting interactive Notebooks right can be very challenging. We advise you to first implement a normal sequential Notebook without widgets that you can later convert into an interactive one. The logic for desired output in every situation can be very complex (e.g. Filtering for exception cases: what happens if the specified dataset does not contain the necessary data?)." 336 | ] 337 | } 338 | ], 339 | "metadata": { 340 | "kernelspec": { 341 | "display_name": "Python 3 (ipykernel)", 342 | "language": "python", 343 | "name": "python3" 344 | }, 345 | "language_info": { 346 | "codemirror_mode": { 347 | "name": "ipython", 348 | "version": 3 349 | }, 350 | "file_extension": ".py", 351 | "mimetype": "text/x-python", 352 | "name": "python", 353 | "nbconvert_exporter": "python", 354 | "pygments_lexer": "ipython3", 355 | "version": "3.9.10" 356 | } 357 | }, 358 | "nbformat": 4, 359 | "nbformat_minor": 5 360 | } 361 | -------------------------------------------------------------------------------- /Notebooks/Tutorial/Tutorial 4 - Visualizing.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tutorial 4 - Visualization\n", 8 | "\n", 9 | "\n", 10 | "Welcome to the *IBM SMF Explorer* Visualization Tutorial.\n", 11 | "This Tutorial provides examples and inspirations on how to create helpful visualizations using SMF data.\n", 12 | "\n", 13 | "> Examples are based on the engine utilization data taken from SMF72 Subtype 3\n", 14 | "\n" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "## Getting started\n", 22 | "\n", 23 | "Initialize a Context for the dataset that you want to work with. Note, *plotly-express* package is imported for plotting the data." 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "autorun": false 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "# imports\n", 35 | "import smfexplorer\n", 36 | "from smfexplorer.fields import SMF72S3\n", 37 | "from smfexplorer import names\n", 38 | "from plotly import express as px\n", 39 | "import pandas as pd\n", 40 | "\n", 41 | "DATASET = \"YOUR.SMF.DATA\"\n", 42 | "\n", 43 | "# data fetching\n", 44 | "ctx = smfexplorer.new_context(DATASET)\n", 45 | "df = ctx.samples.smf_72_03_sample().run()" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | " ## Prepare your data" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "Before plotting, make sure that the data is meaningful and clean. If your analysis does not require Report Class data, make sure to sort it out: " 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": { 66 | "autorun": false 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "df = df[~df[\"is_report_class\"]].drop(\"is_report_class\", axis=1)\n", 71 | "\n", 72 | "# create a df subset:\n", 73 | "df = df[\n", 74 | " [\n", 75 | " \"timestamp\",\n", 76 | " \"sid\",\n", 77 | " \"utilization_cp\",\n", 78 | " \"utilization_ziip\",\n", 79 | " \"utilization_zaap\",\n", 80 | " \"utilization_ziip_on_cp\",\n", 81 | " \"utilization_total\",\n", 82 | " ]\n", 83 | "]" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "Consider data aggregation for meaningful visualization. In the following example, we group values by the *timestamp* and *sid* fields:" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": { 97 | "autorun": false 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "df = df.groupby([\"timestamp\", \"sid\"], as_index=False).sum()" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "Rounding long decimals makes your data more readable and easy to visualize" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": { 115 | "autorun": false 116 | }, 117 | "outputs": [], 118 | "source": [ 119 | "df = df.round(1)" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "metadata": {}, 125 | "source": [ 126 | "Below we check whether workload was running on all processors. If some processor types were not engaged, they are excluded from the report." 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": { 133 | "autorun": false 134 | }, 135 | "outputs": [], 136 | "source": [ 137 | "utilization_fields = [\n", 138 | " SMF72S3.utilization_cp,\n", 139 | " SMF72S3.utilization_ziip,\n", 140 | " SMF72S3.utilization_zaap,\n", 141 | " SMF72S3.utilization_ziip_on_cp,\n", 142 | " SMF72S3.utilization_total,\n", 143 | "]\n", 144 | "display_fields = []\n", 145 | "for field in utilization_fields:\n", 146 | " if df[names(field)].sum() > 0:\n", 147 | " display_fields.append(field)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "## Plot\n", 155 | "\n", 156 | "> we recommend to start small and then extend your plots." 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "metadata": {}, 162 | "source": [ 163 | "For this analysis, we create a line plot that shows the system utilization percentage for each processor type over time. To create visualization, we use plotly-express package, the documentation can be found [here](https://plotly.com/python/plotly-express/).\n", 164 | " " 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": { 171 | "autorun": false 172 | }, 173 | "outputs": [], 174 | "source": [ 175 | "plot = px.line(\n", 176 | " df, x=names(SMF72S3.timestamp), y=names(display_fields), title=\"System Utilisation\"\n", 177 | ")\n", 178 | "\n", 179 | "display(plot)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "### Tooltip and Labels\n", 187 | "\n", 188 | "*Data Plot Tooltip* appears when you hover on a data plot. The information shown in a tooltip can be adjusted. Consider using ```hover_name``` and ```hover_data``` attributes. For more inspiration visit [hover-text-and-formatting documentatoin](https://plotly.com/python/hover-text-and-formatting/#hovermode-x-or-y). \n", 189 | "\n", 190 | "\n", 191 | "If you use Plotly Express, axes and legend are automatically labelled, however, as in example above, labelling does not always provide meaningful information. We recommend overriding it using the ```labels``` keyword argument. \n" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": { 198 | "autorun": false 199 | }, 200 | "outputs": [], 201 | "source": [ 202 | "# adjust your tooltip\n", 203 | "plot_hover = px.line(\n", 204 | " df,\n", 205 | " x=names(SMF72S3.timestamp),\n", 206 | " y=names(display_fields),\n", 207 | " title=\"System Utilisation\",\n", 208 | " hover_name=names(SMF72S3.sid),\n", 209 | " hover_data={\n", 210 | " names(SMF72S3.timestamp): False, # remove timestamp from hover data\n", 211 | " \"value\": \":.0f\", # format utilization value\n", 212 | " },\n", 213 | " labels={ # labels for axes\n", 214 | " \"value\": \"Utilization %\",\n", 215 | " \"variable\": \"Type\",\n", 216 | " names(SMF72S3.timestamp): \"Time\",\n", 217 | " },\n", 218 | ")\n", 219 | "\n", 220 | "display(plot_hover)" 221 | ] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": {}, 226 | "source": [ 227 | "### Color schemes and Legend \n", 228 | "\n", 229 | "Working with big amounts of data you may consider using diverse colouring schemes. Plotly provides you with a verity of default [color sequences](https://plotly.com/python/discrete-color/). You can choose built-in qualitative color sequences from ```px.colors.qualitative``` module or define your own." 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "metadata": { 236 | "autorun": false 237 | }, 238 | "outputs": [], 239 | "source": [ 240 | "plot = px.line(\n", 241 | " df,\n", 242 | " x=names(SMF72S3.timestamp),\n", 243 | " y=names(display_fields),\n", 244 | " color_discrete_sequence=px.colors.qualitative.Prism, # 10 colors\n", 245 | " title=\"System Utilisation\",\n", 246 | " labels={\n", 247 | " \"value\": \"Utilization %\",\n", 248 | " \"variable\": \"Type\",\n", 249 | " names(SMF72S3.timestamp): \"Time\",\n", 250 | " },\n", 251 | ")\n", 252 | "\n", 253 | "display(plot)" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "metadata": {}, 259 | "source": [ 260 | "You can define your colors as well, see example below:" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": { 267 | "autorun": false 268 | }, 269 | "outputs": [], 270 | "source": [ 271 | "MY_COLORS = [\n", 272 | " \"#520408\",\n", 273 | " \"#878d96\",\n", 274 | " \"#31135e\",\n", 275 | " \"#fa4d56\",\n", 276 | " \"#ee5396\",\n", 277 | " \"#a56eff\",\n", 278 | " \"#0f62fe\",\n", 279 | " \"#0072c3\",\n", 280 | " \"#007d79\",\n", 281 | " \"#044317\",\n", 282 | "]\n", 283 | "plot = px.line(\n", 284 | " df,\n", 285 | " x=names(SMF72S3.timestamp),\n", 286 | " y=names(display_fields),\n", 287 | " color_discrete_sequence=MY_COLORS,\n", 288 | " title=\"System Utilisation\",\n", 289 | " labels={\n", 290 | " \"value\": \"Utilization %\",\n", 291 | " \"variable\": \"Type\",\n", 292 | " names(SMF72S3.timestamp): \"Time\",\n", 293 | " },\n", 294 | ")\n", 295 | "\n", 296 | "display(plot)" 297 | ] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": {}, 302 | "source": [ 303 | "You can hide and show legend. Sometimes, when two plots share the same legend, you may want to hide one using ```layout.showlegend``` attribute. Moreover, you can position legend within or outside plot area. " 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": null, 309 | "metadata": { 310 | "autorun": false 311 | }, 312 | "outputs": [], 313 | "source": [ 314 | "plot = px.line(\n", 315 | " df,\n", 316 | " x=names(SMF72S3.timestamp),\n", 317 | " y=names(SMF72S3.utilization_cp),\n", 318 | " title=\"CP Utilisation\",\n", 319 | " labels={\n", 320 | " \"utilization_cp\": \"Utilization %\",\n", 321 | " \"variable\": \"Type\",\n", 322 | " names(SMF72S3.timestamp): \"Time\",\n", 323 | " },\n", 324 | ")\n", 325 | "plot.update_layout(showlegend=False) # hide legend\n", 326 | "\n", 327 | "display(plot)\n", 328 | "\n", 329 | "plot_2 = px.line(\n", 330 | " df,\n", 331 | " x=names(SMF72S3.timestamp),\n", 332 | " y=names(display_fields),\n", 333 | " title=\"System Utilisation\",\n", 334 | " labels={\n", 335 | " \"value\": \"Utilization %\",\n", 336 | " \"variable\": \"Type\",\n", 337 | " names(SMF72S3.timestamp): \"Time\",\n", 338 | " },\n", 339 | ")\n", 340 | "plot_2.update_layout(\n", 341 | " legend=dict(\n", 342 | " orientation=\"h\", # horizontal positioning\n", 343 | " yanchor=\"bottom\",\n", 344 | " y=-0.8, # add some spaces\n", 345 | " x=0.3,\n", 346 | " )\n", 347 | ")\n", 348 | "display(plot_2)" 349 | ] 350 | }, 351 | { 352 | "cell_type": "markdown", 353 | "metadata": {}, 354 | "source": [ 355 | "#### Interacting with Legend and Axes\n", 356 | "\n", 357 | "- Click on a legend item to hide or show its trace\n", 358 | "- Double-click on legend to reset the selection\n", 359 | "- Drag the mouse diagonally to zoom to the resulting box\n", 360 | "- Drag the mouse vertically to zoom to this part of the y axis\n", 361 | "- Drag the mouse horizontally to zoom to this part of the x axis\n", 362 | "- Double-click within chart to reset the zoom\n", 363 | "\n", 364 | "\n", 365 | "See [documentation](https://plotly.com/python/legend/) for more examples." 366 | ] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "metadata": {}, 371 | "source": [ 372 | "### Create Bar Chart\n", 373 | "\n", 374 | "to get an overview of your data or to analyze the ratio, you may consider using bar- or histogram- plots. " 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": null, 380 | "metadata": { 381 | "autorun": false 382 | }, 383 | "outputs": [], 384 | "source": [ 385 | "hist = px.histogram(\n", 386 | " df,\n", 387 | " x=\"timestamp\",\n", 388 | " y=[\"utilization_cp\", \"utilization_ziip\"],\n", 389 | " title=\"Ratio of CP to zIIP Utilization over time\",\n", 390 | " barmode=\"group\",\n", 391 | " labels={\n", 392 | " \"value\": \"Utilization %\",\n", 393 | " \"variable\": \"Type\",\n", 394 | " names(SMF72S3.timestamp): \"Time\",\n", 395 | " },\n", 396 | ")\n", 397 | "display(hist)\n", 398 | "\n", 399 | "# to get an overview, aggregate your data\n", 400 | "\n", 401 | "df_aggr = df.agg(\"mean\", numeric_only=True) # aggregation\n", 402 | "df_aggr = pd.DataFrame(df_aggr).T.round(1) # transpose your df\n", 403 | "\n", 404 | "hist_aggr = px.bar(\n", 405 | " df_aggr,\n", 406 | " x=\"utilization_total\",\n", 407 | " y=[\"utilization_cp\", \"utilization_ziip\"],\n", 408 | " title=\"Ratio of CP to zIIP Utilization\",\n", 409 | " labels={\n", 410 | " \"value\": \"Utilization %\",\n", 411 | " \"variable\": \"Type\",\n", 412 | " },\n", 413 | ")\n", 414 | "hist_aggr.update_layout(xaxis={\"visible\": False}) # hide x-axis\n", 415 | "display(hist_aggr)" 416 | ] 417 | } 418 | ], 419 | "metadata": { 420 | "kernelspec": { 421 | "display_name": "Python 3 (ipykernel)", 422 | "language": "python", 423 | "name": "python3" 424 | }, 425 | "language_info": { 426 | "codemirror_mode": { 427 | "name": "ipython", 428 | "version": 3 429 | }, 430 | "file_extension": ".py", 431 | "mimetype": "text/x-python", 432 | "name": "python", 433 | "nbconvert_exporter": "python", 434 | "pygments_lexer": "ipython3", 435 | "version": "3.9.10" 436 | } 437 | }, 438 | "nbformat": 4, 439 | "nbformat_minor": 4 440 | } 441 | -------------------------------------------------------------------------------- /Notebooks/Reports/LPAR Topology Report.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# LPAR Topology Report\n", 8 | "\n", 9 | " Select a **_Dataset_** to request data from \n", 10 | "**_(If the text field to select a Dataframe does not show up after a few seconds or you want to reset the notebook, restart the Kernel using the `>>`(Restart and run all) button above)_**" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 22, 16 | "metadata": { 17 | "autorun": true 18 | }, 19 | "outputs": [ 20 | { 21 | "data": { 22 | "application/vnd.jupyter.widget-view+json": { 23 | "model_id": "c298c66123074755941ec851566d12e2", 24 | "version_major": 2, 25 | "version_minor": 0 26 | }, 27 | "text/plain": [ 28 | "ConfigWidget(children=(Text(value='', layout=Layout(width='500px'), placeholder='Dataset Name(s)'), HTML(value…" 29 | ] 30 | }, 31 | "metadata": {}, 32 | "output_type": "display_data" 33 | }, 34 | { 35 | "data": { 36 | "application/vnd.jupyter.widget-view+json": { 37 | "model_id": "e4001d50c73d47359c8cb7b50f9409ff", 38 | "version_major": 2, 39 | "version_minor": 0 40 | }, 41 | "text/plain": [ 42 | "Output()" 43 | ] 44 | }, 45 | "metadata": {}, 46 | "output_type": "display_data" 47 | }, 48 | { 49 | "data": { 50 | "application/vnd.jupyter.widget-view+json": { 51 | "model_id": "35fd62c36f1e4e19b8e67142b116e728", 52 | "version_major": 2, 53 | "version_minor": 0 54 | }, 55 | "text/plain": [ 56 | "Output()" 57 | ] 58 | }, 59 | "metadata": {}, 60 | "output_type": "display_data" 61 | }, 62 | { 63 | "data": { 64 | "application/vnd.jupyter.widget-view+json": { 65 | "model_id": "13dd68b4621a48bc89e6cdad9ddd6c01", 66 | "version_major": 2, 67 | "version_minor": 0 68 | }, 69 | "text/plain": [ 70 | "Output()" 71 | ] 72 | }, 73 | "metadata": {}, 74 | "output_type": "display_data" 75 | }, 76 | { 77 | "data": { 78 | "application/vnd.jupyter.widget-view+json": { 79 | "model_id": "cff53fb587e34063877c92676d1d82e0", 80 | "version_major": 2, 81 | "version_minor": 0 82 | }, 83 | "text/plain": [ 84 | "Output()" 85 | ] 86 | }, 87 | "metadata": {}, 88 | "output_type": "display_data" 89 | }, 90 | { 91 | "data": { 92 | "application/vnd.jupyter.widget-view+json": { 93 | "model_id": "6cd9693b79ba45bc97219ad5e91c8116", 94 | "version_major": 2, 95 | "version_minor": 0 96 | }, 97 | "text/plain": [ 98 | "Output()" 99 | ] 100 | }, 101 | "metadata": {}, 102 | "output_type": "display_data" 103 | } 104 | ], 105 | "source": [ 106 | "import smfexplorer\n", 107 | "import pandas as pd\n", 108 | "import ipywidgets as widgets\n", 109 | "import plotly as pl\n", 110 | "import numpy as np\n", 111 | "\n", 112 | "from smfexplorer.fields import SMF99S12 as F12\n", 113 | "from smfexplorer.fields import SMF99S14\n", 114 | "from smfexplorer.core.expressions import ASC\n", 115 | "from smfexplorer.util import jupyter\n", 116 | "import plotly.graph_objs as go\n", 117 | "from plotly.subplots import make_subplots\n", 118 | "from IPython.display import Markdown, display, Javascript, HTML\n", 119 | "from ipywidgets import VBox, interact, Button, Output\n", 120 | "import plotly.express as px\n", 121 | "from smfexplorer.error import EmptyDataSetError\n", 122 | "\n", 123 | "import warnings\n", 124 | "\n", 125 | "warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n", 126 | "\n", 127 | "ctx = smfexplorer.new_context()\n", 128 | "config_widget = jupyter.ConfigWidget(ctx)\n", 129 | "display(config_widget)\n", 130 | "\n", 131 | "selection_widget_topo = widgets.ToggleButtons()\n", 132 | "selection_widget_date = widgets.ToggleButtons()\n", 133 | "selection_widget_color = widgets.ToggleButtons()\n", 134 | "selection_widget_time = widgets.ToggleButtons()\n", 135 | "\n", 136 | "@config_widget.register_output(name=\"df_topo\", system_topo=selection_widget_topo)\n", 137 | "def fetch_topo_df(dsn, **kwds):\n", 138 | " if dsn is None:\n", 139 | " return\n", 140 | " try:\n", 141 | " display(Markdown(f\"# Topology \"))\n", 142 | " fields = list(ctx.samples.topology().fields)\n", 143 | " fields.append(SMF99S14.cp_polar)\n", 144 | " df_topo = ctx.samples.topology().run()\n", 145 | " \n", 146 | " df_topo = df_topo.rename(\n", 147 | " columns={\n", 148 | " \"timestamp\": \"Timestamp\",\n", 149 | " \"sid\": \"System\",\n", 150 | " \"hd_topochg_cpu_index\": \"Topology Index\",\n", 151 | " \"processor\": \"Processor Type\",\n", 152 | " \"cp_cputype\": \"CPU Type\",\n", 153 | " \"vcm_lparphysprocshr\": \"LPAR Share\",\n", 154 | " \"speed_change\": \"speed\",\n", 155 | " \"honor_priority_change\": \"honor priority\",\n", 156 | " \"topology_change\": \"topology\",\n", 157 | " \"affinity_nodes_rebuild\": \"affinity nodes\",\n", 158 | " \"mpwq_affinity_node\": \"Affinity Node\",\n", 159 | " }\n", 160 | " )\n", 161 | " \n", 162 | " unique_system = np.sort(df_topo[\"System\"].unique())\n", 163 | "\n", 164 | " selection_widget_topo.options = unique_system\n", 165 | "\n", 166 | " if unique_system.size > 1:\n", 167 | " display(Markdown(\"# Select system for analysis\"))\n", 168 | " display(selection_widget_topo)\n", 169 | "\n", 170 | " else:\n", 171 | " display(Markdown(f\"## Using data of system {unique_system[0]}\"))\n", 172 | "\n", 173 | " selection_widget_topo.value = unique_system[0]\n", 174 | " \n", 175 | " if len(unique_system) > 1:\n", 176 | " display(selection_widget_topo)\n", 177 | "\n", 178 | " return df_topo\n", 179 | "\n", 180 | " except EmptyDataSetError:\n", 181 | " display(Markdown(\"### Data set does not contain SMF99 Subtype 14 data\"))\n", 182 | "\n", 183 | "\n", 184 | "@fetch_topo_df.register_output(name=\"topo\", date=selection_widget_date)\n", 185 | "def dataframe_data(df_topo, system_topo, **kwds):\n", 186 | " global topo\n", 187 | "\n", 188 | " if df_topo is None:\n", 189 | " return\n", 190 | " topo = df_topo[df_topo[\"System\"] == system_topo]\n", 191 | " topo.reset_index(inplace=True, drop=True)\n", 192 | "\n", 193 | " timestamps_aval = topo[\"Timestamp\"].unique()\n", 194 | " rows = []\n", 195 | " for time_point in timestamps_aval:\n", 196 | " iterator = 0\n", 197 | " df_tmp = topo[topo[\"Timestamp\"] == time_point]\n", 198 | " buckets = len(df_tmp[\"Topology Index\"].unique())\n", 199 | " if df_tmp.shape[0] % buckets == 0:\n", 200 | " p = int(df_tmp.shape[0] / buckets)\n", 201 | " dpoints = df_tmp.values.reshape(p, -1, df_tmp.shape[1])\n", 202 | " for point in dpoints:\n", 203 | " if iterator < buckets:\n", 204 | " rows.append(point[iterator])\n", 205 | " iterator = iterator + 1\n", 206 | " tmp = pd.DataFrame(rows, columns=topo.columns)\n", 207 | " topo = tmp\n", 208 | " date_aval = [\n", 209 | " date.strftime(\"%m-%d-%y \") for date in topo[\"Timestamp\"].dt.date.unique()\n", 210 | " ]\n", 211 | " selection_widget_date.options = date_aval\n", 212 | " if np.size(date_aval) == 1:\n", 213 | " display(Markdown(f\"**Available date:** {date_aval[0]}\"))\n", 214 | "\n", 215 | " else:\n", 216 | " display(Markdown(\"# Select date for analysis\"))\n", 217 | " display(selection_widget_date)\n", 218 | "\n", 219 | " selection_widget_date.value = date_aval[0]\n", 220 | "\n", 221 | " return topo\n", 222 | "\n", 223 | "\n", 224 | "@dataframe_data.register_output(name=\"topo\", time=selection_widget_time)\n", 225 | "def dataframe_time(topo, date, **kwds):\n", 226 | " if topo is None:\n", 227 | " return\n", 228 | "\n", 229 | " time_placeholder = pd.to_datetime(topo[\"Timestamp\"][0]).strftime(\"%H:%M:%S\")\n", 230 | " selection_widget_time.placeholder = time_placeholder\n", 231 | " topo[\"date\"] = topo[\"Timestamp\"].dt.strftime(\"%m-%d-%y \")\n", 232 | " topo[\"time\"] = topo[\"Timestamp\"].dt.strftime(\"%H:%M:%S\")\n", 233 | " topo = topo[topo[\"date\"] == date]\n", 234 | " topo.reset_index(inplace=True, drop=True)\n", 235 | " if (len(topo.index) != 0) and len(topo.index) % len(topo[\"Topology Index\"].unique()) == 0:\n", 236 | " entry_time = topo[\"Timestamp\"][0]\n", 237 | " topo_comp = topo.drop(columns=[\"Timestamp\", \"date\", \"time\"])\n", 238 | " topo_comp = topo_comp.drop_duplicates()\n", 239 | " if len(topo_comp.index) == len(topo[\"Topology Index\"].unique()):\n", 240 | " topo = topo_comp\n", 241 | "\n", 242 | " else:\n", 243 | " display(Markdown(\"# Select time for analysis\"))\n", 244 | " display(selection_widget_time)\n", 245 | " else:\n", 246 | " display(Markdown(\"# Select time for analysis\"))\n", 247 | " display(selection_widget_time)\n", 248 | "\n", 249 | " selection_widget_time.options = topo[\"time\"].unique()\n", 250 | " selection_widget_time.value = time_placeholder\n", 251 | "\n", 252 | " return topo\n", 253 | "\n", 254 | "\n", 255 | "@dataframe_time.register_output(\n", 256 | " name=\"df_filtered\", time=selection_widget_time, coloring=selection_widget_color\n", 257 | ")\n", 258 | "def dataframe_topo(topo, time, **kwds):\n", 259 | " global df_filtered\n", 260 | "\n", 261 | " if topo is None:\n", 262 | " return\n", 263 | "\n", 264 | " time_selection = topo[topo[\"time\"] >= time].reset_index()\n", 265 | "\n", 266 | " if time_selection.empty:\n", 267 | " time_selection = topo[topo[\"time\"] <= time].reset_index()[\"Timestamp\"].iloc[-1]\n", 268 | " else:\n", 269 | " time_selection = time_selection[\"Timestamp\"][0]\n", 270 | " display(\n", 271 | " Markdown(\n", 272 | " f\"Selected timestamp for analysis is: {time_selection.strftime('%m-%d-%y %H:%M:%S')}\"\n", 273 | " )\n", 274 | " )\n", 275 | " df_filtered = topo[topo[\"Timestamp\"] == time_selection].copy(deep=True)\n", 276 | " colorings = [\"CPU Type\", \"Affinity Node\", \"Processor Type\"]\n", 277 | " selection_widget_color.options = colorings\n", 278 | " display(Markdown(\"## Select color diferentiation\"))\n", 279 | " display(selection_widget_color)\n", 280 | " selection_widget_color.value = colorings[0]\n", 281 | "\n", 282 | " return df_filtered\n", 283 | "\n", 284 | "\n", 285 | "@dataframe_topo.register_output()\n", 286 | "def topology_viz(df_filtered, coloring, **kwds):\n", 287 | " if df_filtered is None:\n", 288 | " return\n", 289 | " colors = [\n", 290 | " \"#dee2ff\",\n", 291 | " \"#edf2fb\",\n", 292 | " \"#efd3d7\",\n", 293 | " \"#F8ECEE\",\n", 294 | " \"#EAE6ED\",\n", 295 | " \"#E4F2C2\",\n", 296 | " \"#F0F7E0\",\n", 297 | " \"#FFF5CC\",\n", 298 | " \"#FFFAE6\",\n", 299 | " ]\n", 300 | " changes = [\"speed\", \"honor priority\", \"topology\", \"affinity nodes\"]\n", 301 | " STISI = topo[\"cp_ci_nlinuse\"].unique()[0]\n", 302 | "\n", 303 | " def label(row):\n", 304 | " set_label = (\n", 305 | " \"System: \"\n", 306 | " + row[\"System\"]\n", 307 | " + \"
Affinity Node: \"\n", 308 | " + str(row[\"Affinity Node\"])\n", 309 | " + \"
CPU: \"\n", 310 | " + row[\"CPU Type\"]\n", 311 | " + \"
Processor: \"\n", 312 | " + row[\"Processor Type\"]\n", 313 | " + \"
Index: \"\n", 314 | " + str(row[\"Topology Index\"])\n", 315 | " )\n", 316 | " return set_label\n", 317 | "\n", 318 | " def change(change_type):\n", 319 | " if not topo[change_type].any():\n", 320 | " display(Markdown(f\"No changes in {change_type}\"))\n", 321 | " else:\n", 322 | " display(Markdown(f\"**Changes in {change_type}:**\"))\n", 323 | " topo_output = topo.loc[topo[change_type] == True]\n", 324 | " topo_output = topo_output[\"Timestamp\"].unique()\n", 325 | " timestamps = []\n", 326 | "\n", 327 | " for time in topo_output:\n", 328 | " timestamps.append(pd.to_datetime(time).strftime(\"%m-%d-%y %H:%M:%S\"))\n", 329 | " if len(timestamps) < 10:\n", 330 | " display(Markdown(f\"{', '.join(timestamps)}\"))\n", 331 | " else:\n", 332 | " display(Markdown(f\"{len(timestamps)} changes observed\"))\n", 333 | "\n", 334 | " def init_layout(data_fig):\n", 335 | " global df_processed\n", 336 | " data_fig[\"Affinity Node\"] = data_fig[\"Affinity Node\"].astype(str)\n", 337 | " if STISI == \"STISI_15_1_3\":\n", 338 | " data_fig = data_fig.rename(\n", 339 | " columns={\"cp_ci_nl1\": \"Chip ID\", \"cp_ci_nl2\": \"Book ID\"}\n", 340 | " ).drop(columns=[\"cp_ci_nl3\"])\n", 341 | " data_fig[\"Chip ID\"] = \"Chip: \" + data_fig[\"Chip ID\"].astype(str)\n", 342 | " data_fig[\"Book ID\"] = \"Book: \" + data_fig[\"Book ID\"].astype(str)\n", 343 | " fig = px.treemap(\n", 344 | " data_fig,\n", 345 | " path=[\"Book ID\", \"Chip ID\", \"Label\"],\n", 346 | " color=coloring,\n", 347 | " values=None,\n", 348 | " color_discrete_sequence=colors,\n", 349 | " )\n", 350 | " elif STISI == \"STISI_15_1_4\":\n", 351 | " data_fig = data_fig.rename(\n", 352 | " columns={\n", 353 | " \"cp_ci_nl1\": \"Chip ID\",\n", 354 | " \"cp_ci_nl2\": \"Node ID\",\n", 355 | " \"cp_ci_nl3\": \"Drawer\",\n", 356 | " }\n", 357 | " )\n", 358 | " data_fig[\"Chip ID\"] = \"Chip: \" + data_fig[\"Chip ID\"].astype(str)\n", 359 | " data_fig[\"Node ID\"] = \"Node: \" + data_fig[\"Node ID\"].astype(str)\n", 360 | " data_fig[\"Drawer\"] = \"Drawer: \" + data_fig[\"Drawer\"].astype(str)\n", 361 | " fig = px.treemap(\n", 362 | " data_fig,\n", 363 | " path=[\"Drawer\", \"Node ID\", \"Chip ID\", \"Label\".replace(\"_\", \"\\n\")],\n", 364 | " values=None,\n", 365 | " color=coloring,\n", 366 | " color_discrete_sequence=colors,\n", 367 | " )\n", 368 | " elif STISI == \"STISI_15_1_2\" or STISI == \"STISI_15_1_20\":\n", 369 | " data_fig = data_fig.rename(\n", 370 | " columns={\"cp_ci_nl2\": \"Chip ID\", \"cp_ci_nl1\": \"Book ID\"}\n", 371 | " ).drop(columns=[\"cp_ci_nl3\"])\n", 372 | " data_fig[\"Chip ID\"] = \"Chip: \" + data_fig[\"Chip ID\"].astype(str)\n", 373 | " data_fig[\"Book ID\"] = \"Book: \" + data_fig[\"Book ID\"].astype(str)\n", 374 | " fig = px.treemap(\n", 375 | " data_fig,\n", 376 | " path=[\"Book ID\", \"Chip ID\", \"Label\"],\n", 377 | " color=coloring,\n", 378 | " color_discrete_sequence=colors,\n", 379 | " )\n", 380 | " else:\n", 381 | " data_fig = data_fig.rename(\n", 382 | " columns={\n", 383 | " \"cp_ci_nl1\": \"Chip ID\",\n", 384 | " \"cp_ci_nl2\": \"Cluster ID\",\n", 385 | " \"cp_ci_nl3\": \"Drawer\",\n", 386 | " }\n", 387 | " )\n", 388 | " data_fig[\"Chip ID\"] = \"Chip: \" + data_fig[\"Chip ID\"].astype(str)\n", 389 | " data_fig[\"Cluster ID\"] = \"Cluster: \" + data_fig[\"Cluster ID\"].astype(str)\n", 390 | " data_fig[\"Drawer\"] = \"Drawer: \" + data_fig[\"Drawer\"].astype(str)\n", 391 | " fig = px.treemap(\n", 392 | " data_fig,\n", 393 | " path=[\"Drawer\", \"Cluster ID\", \"Chip ID\", \"Label\"],\n", 394 | " color=coloring,\n", 395 | " color_discrete_sequence=colors,\n", 396 | " )\n", 397 | " df_processed = data_fig\n", 398 | " df_processed = df_processed.drop(\n", 399 | " columns=[\n", 400 | " \"cp_ci_nlinuse\",\n", 401 | " \"cp_ci_nl4\",\n", 402 | " \"cp_ci_nl5\",\n", 403 | " \"speed\",\n", 404 | " \"wuq_error\",\n", 405 | " \"honor priority\",\n", 406 | " \"affinity nodes\",\n", 407 | " \"topology\",\n", 408 | " ]\n", 409 | " )\n", 410 | " fig.data[0].hovertemplate = \"%{parent}\"\n", 411 | " fig.show()\n", 412 | "\n", 413 | " if (\n", 414 | " (not topo[\"speed\"].any())\n", 415 | " and (not topo[\"topology\"].any())\n", 416 | " and (not topo[\"honor priority\"].any())\n", 417 | " and (not topo[\"affinity nodes\"].any())\n", 418 | " ):\n", 419 | " display(\n", 420 | " Markdown(\n", 421 | " f\"No changes in speed and honor priority. Affinity nodes were not rebuilt\"\n", 422 | " )\n", 423 | " )\n", 424 | " else:\n", 425 | " for change_type in changes:\n", 426 | " change(change_type)\n", 427 | "\n", 428 | " df_filtered[\"Label\"] = df_filtered.apply(lambda row: label(row), axis=1)\n", 429 | "\n", 430 | " init_layout(df_filtered)" 431 | ] 432 | } 433 | ], 434 | "metadata": { 435 | "kernelspec": { 436 | "display_name": "Python 3 (ipykernel)", 437 | "language": "python", 438 | "name": "python3" 439 | }, 440 | "language_info": { 441 | "codemirror_mode": { 442 | "name": "ipython", 443 | "version": 3 444 | }, 445 | "file_extension": ".py", 446 | "mimetype": "text/x-python", 447 | "name": "python", 448 | "nbconvert_exporter": "python", 449 | "pygments_lexer": "ipython3", 450 | "version": "3.11.9" 451 | } 452 | }, 453 | "nbformat": 4, 454 | "nbformat_minor": 4 455 | } 456 | -------------------------------------------------------------------------------- /Notebooks/Tutorial/Tutorial 1 - Basics.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tutorial 1 - Basics\n", 8 | "\n", 9 | "Welcome to the *IBM SMF Explorer* Basics Tutorial.\n", 10 | "You will learn how to get started with the *IBM SMF Explorer* framework and access SMF data.\n", 11 | "\n", 12 | "The tutorial has cells containing code. When you come across such a code cell you should execute it by selecting it and pressing ``Ctrl``+``Enter``.\n", 13 | "Feel free to change the code, but keep in mind that other parts of the tutorial might be affected. \n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## Getting started\n", 21 | "\n", 22 | "To start working with the *IBM SMF Explorer*, you need to import the ``smfexplorer`` package:" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "metadata": { 29 | "tags": [] 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "import smfexplorer" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "This tutorial was created for *SMF Explorer* version **1.0.2**.\n", 41 | "To check the version, execute the following command: ``smfexplorer.__version__``." 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 2, 47 | "metadata": { 48 | "tags": [] 49 | }, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/plain": [ 54 | "'1.0.4'" 55 | ] 56 | }, 57 | "execution_count": 2, 58 | "metadata": {}, 59 | "output_type": "execute_result" 60 | } 61 | ], 62 | "source": [ 63 | "smfexplorer.__version__" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "All requests in the *SMF Explorer* are managed by an object called a **Context**.\n", 71 | "A Context represents a connection to one or multiple SMF dumps/datasets and manages the state of all requests dispatched against it.\n", 72 | "Creating separate contexts allows you to run the same requests against different SMF dumps in an easy manner.\n", 73 | "\n", 74 | "The Context can be created by calling the ``new_context()`` function of the ``smfexplorer`` module.\n", 75 | "If multiple datasets are defined (i.e., provided as arguments to the ``new_context()``), the *SMF Explorer* will execute requests against all of them and concatenate the data in the order the names were specified.\n", 76 | "\n", 77 | "To separately fetch the data from different datasets, you can create as many Context objects as you like. \n", 78 | "\n", 79 | "> **Note**: if you are working with two or more datasets that were created as a result of the dump-dataset switch (i.e., one dataset is a continuation of another), we recommend assigning these datasets to one Context. \n", 80 | "If you want to work with datasets that contain different SMF records, we advise creation of multiple Contexts for better performance. " 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 3, 86 | "metadata": { 87 | "tags": [] 88 | }, 89 | "outputs": [], 90 | "source": [ 91 | "DATASET = \"YOUR.SMF.DATA\"\n", 92 | "ctx = smfexplorer.new_context(DATASET)\n", 93 | "\n", 94 | "# Multiple datasets assigned to one Context instance:\n", 95 | "# ctx2 = smfexplorer.new_context('YOUR.SMF.DATA1','YOUR.SMF.DATA2')" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "*IBM SMF Explorer* provides utility functions that can help you to understand your input dataset. \n", 103 | "\n", 104 | "You can check if a dataset is available:" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 4, 110 | "metadata": { 111 | "tags": [] 112 | }, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "True\n", 119 | "False\n" 120 | ] 121 | } 122 | ], 123 | "source": [ 124 | "print(smfexplorer.check_dataset(DATASET))\n", 125 | "print(smfexplorer.check_dataset(\"WRONG.SMF.DATA\"))" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "## Get meta data of the dataset" 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "### Get available records" 140 | ] 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "You can fetch a list of SMF types/subtypes available in your SMF dataset using ```get_available_records()```function. \n", 147 | "\n", 148 | "To get the list of types you can run ```get_available_types()``` function and for subtypes - ```get_available_subtypes()```." 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 4, 154 | "metadata": { 155 | "tags": [] 156 | }, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/html": [ 161 | "
\n", 162 | "\n", 175 | "\n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | "
typecount
021
131
21450480
3157753
46230265
57036
68075008
78212065
81132664
\n", 231 | "
" 232 | ], 233 | "text/plain": [ 234 | " type count\n", 235 | "0 2 1\n", 236 | "1 3 1\n", 237 | "2 14 50480\n", 238 | "3 15 7753\n", 239 | "4 62 30265\n", 240 | "5 70 36\n", 241 | "6 80 75008\n", 242 | "7 82 12065\n", 243 | "8 113 2664" 244 | ] 245 | }, 246 | "execution_count": 4, 247 | "metadata": {}, 248 | "output_type": "execute_result" 249 | } 250 | ], 251 | "source": [ 252 | "# get available types\n", 253 | "ctx.get_available_types()" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 6, 259 | "metadata": { 260 | "tags": [] 261 | }, 262 | "outputs": [ 263 | { 264 | "data": { 265 | "text/html": [ 266 | "
\n", 267 | "\n", 280 | "\n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | "
subtypecount
0127
129
\n", 301 | "
" 302 | ], 303 | "text/plain": [ 304 | " subtype count\n", 305 | "0 1 27\n", 306 | "1 2 9" 307 | ] 308 | }, 309 | "execution_count": 6, 310 | "metadata": {}, 311 | "output_type": "execute_result" 312 | } 313 | ], 314 | "source": [ 315 | "# get available subtypes for the SMF type 70\n", 316 | "ctx.get_available_subtypes(70)" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": 7, 322 | "metadata": { 323 | "tags": [] 324 | }, 325 | "outputs": [ 326 | { 327 | "data": { 328 | "text/html": [ 329 | "
\n", 330 | "\n", 343 | "\n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | "
typesubtypecount
0201
1301
214050480
31507753
462030265
570127
67029
780075008
88211
982182
10822011430
1182216
128224284
1382289
148231333
1511311296
1611321368
\n", 457 | "
" 458 | ], 459 | "text/plain": [ 460 | " type subtype count\n", 461 | "0 2 0 1\n", 462 | "1 3 0 1\n", 463 | "2 14 0 50480\n", 464 | "3 15 0 7753\n", 465 | "4 62 0 30265\n", 466 | "5 70 1 27\n", 467 | "6 70 2 9\n", 468 | "7 80 0 75008\n", 469 | "8 82 1 1\n", 470 | "9 82 18 2\n", 471 | "10 82 20 11430\n", 472 | "11 82 21 6\n", 473 | "12 82 24 284\n", 474 | "13 82 28 9\n", 475 | "14 82 31 333\n", 476 | "15 113 1 1296\n", 477 | "16 113 2 1368" 478 | ] 479 | }, 480 | "execution_count": 7, 481 | "metadata": {}, 482 | "output_type": "execute_result" 483 | } 484 | ], 485 | "source": [ 486 | "# get a complete SMF record overview\n", 487 | "ctx.get_available_records()" 488 | ] 489 | }, 490 | { 491 | "cell_type": "markdown", 492 | "metadata": {}, 493 | "source": [ 494 | "### Get description of the dataset\n", 495 | "\n", 496 | "You can request the meta information of your SMF dataset using ```get_dataset_description()```function. " 497 | ] 498 | }, 499 | { 500 | "cell_type": "code", 501 | "execution_count": 5, 502 | "metadata": { 503 | "tags": [] 504 | }, 505 | "outputs": [], 506 | "source": [ 507 | "desc = ctx.get_dataset_description()" 508 | ] 509 | }, 510 | { 511 | "cell_type": "markdown", 512 | "metadata": {}, 513 | "source": [ 514 | "The result of the method is a dictionary ```dict```. The keys of the dictionary are the dataset names used to create the context, and the values are the descripton of the corresponding dataset. You can use the name of the dataset to extract the description of a specific dataset from the retrieved description as follows:" 515 | ] 516 | }, 517 | { 518 | "cell_type": "code", 519 | "execution_count": 7, 520 | "metadata": { 521 | "tags": [] 522 | }, 523 | "outputs": [], 524 | "source": [ 525 | "dataset_desc = desc[DATASET]" 526 | ] 527 | }, 528 | { 529 | "cell_type": "markdown", 530 | "metadata": {}, 531 | "source": [ 532 | "The description of each dataset is saved in a ```dict``` object. Use function ```keys()``` of a ```dict```to see what kinds of information are available:" 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": 9, 538 | "metadata": { 539 | "tags": [] 540 | }, 541 | "outputs": [ 542 | { 543 | "data": { 544 | "text/plain": [ 545 | "dict_keys(['creation_date', 'count', 'system_ids', 'estimated_size_in_bytes', 'first_time_in_buffer', 'last_time_in_buffer', 'earliest_start', 'lastest_start', 'earliest_end', 'lastest_end', 'type_info'])" 546 | ] 547 | }, 548 | "execution_count": 9, 549 | "metadata": {}, 550 | "output_type": "execute_result" 551 | } 552 | ], 553 | "source": [ 554 | "dataset_desc.keys()" 555 | ] 556 | }, 557 | { 558 | "cell_type": "markdown", 559 | "metadata": {}, 560 | "source": [ 561 | "You can use the key to retrieve information from the description. For example, you can get all the system IDs inside the dataset by calling:" 562 | ] 563 | }, 564 | { 565 | "cell_type": "code", 566 | "execution_count": 10, 567 | "metadata": { 568 | "tags": [] 569 | }, 570 | "outputs": [ 571 | { 572 | "data": { 573 | "text/plain": [ 574 | "['J80 ']" 575 | ] 576 | }, 577 | "execution_count": 10, 578 | "metadata": {}, 579 | "output_type": "execute_result" 580 | } 581 | ], 582 | "source": [ 583 | "dataset_desc['system_ids']" 584 | ] 585 | }, 586 | { 587 | "cell_type": "markdown", 588 | "metadata": {}, 589 | "source": [ 590 | "Or you can get detailed information of records of each subtype by calling:" 591 | ] 592 | }, 593 | { 594 | "cell_type": "code", 595 | "execution_count": 13, 596 | "metadata": { 597 | "tags": [] 598 | }, 599 | "outputs": [ 600 | { 601 | "data": { 602 | "text/html": [ 603 | "
\n", 604 | "\n", 617 | "\n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | "
typesubtypecountsystem_idsestimated_size_in_bytesfirst_time_in_bufferlast_time_in_bufferearliest_startlastest_startearliest_endlastest_end
011311296[J80 ]19733762019-07-25 00:00:20.5502019-07-25 05:46:04.660NaTNaTNaTNaT
111321368[J80 ]20082242019-07-25 00:00:20.5502019-07-25 05:46:04.660NaTNaTNaTNaT
214050480[J80 ]228110032019-07-25 00:00:00.0002019-07-25 05:59:57.750NaTNaTNaTNaT
31507753[J80 ]37312702019-07-25 00:00:01.9902019-07-25 05:59:55.880NaTNaTNaTNaT
4201[J80 ]182019-07-25 08:15:10.4302019-07-25 08:15:10.430NaTNaTNaTNaT
5301[J80 ]182019-07-25 08:15:16.5102019-07-25 08:15:16.510NaTNaTNaTNaT
662030265[J80 ]77353302019-07-25 00:00:00.5102019-07-25 05:59:39.070NaTNaTNaTNaT
770127[J80 ]6942242019-07-25 00:29:35.0702019-07-25 05:59:35.0802019-07-24 23:59:352019-07-25 05:29:352019-07-25 00:29:34.9192019-07-25 05:59:34.926
87029[J80 ]371522019-07-25 00:29:35.1402019-07-25 05:59:35.2002019-07-24 23:59:352019-07-25 05:29:352019-07-25 00:29:34.9192019-07-25 05:59:34.926
980075008[J80 ]224387072019-07-25 00:00:00.0002019-07-25 05:59:59.220NaTNaTNaTNaT
108211[J80 ]2682019-07-25 02:59:50.0402019-07-25 02:59:50.040NaTNaTNaTNaT
1182182[J80 ]922019-07-25 02:59:48.8202019-07-25 02:59:48.830NaTNaTNaTNaT
12822011430[J80 ]13716002019-07-25 00:00:00.2902019-07-25 05:59:58.280NaTNaTNaTNaT
1382216[J80 ]6722019-07-25 02:34:42.9402019-07-25 02:59:48.640NaTNaTNaTNaT
148224284[J80 ]1259522019-07-25 02:59:48.2702019-07-25 02:59:48.610NaTNaTNaTNaT
1582289[J80 ]24482019-07-25 02:59:52.3402019-07-25 03:04:36.920NaTNaTNaTNaT
168231333[J80 ]720722019-07-25 00:00:05.0302019-07-25 05:30:05.000NaTNaTNaTNaT
\n", 875 | "
" 876 | ], 877 | "text/plain": [ 878 | " type subtype count system_ids estimated_size_in_bytes \\\n", 879 | "0 113 1 1296 [J80 ] 1973376 \n", 880 | "1 113 2 1368 [J80 ] 2008224 \n", 881 | "2 14 0 50480 [J80 ] 22811003 \n", 882 | "3 15 0 7753 [J80 ] 3731270 \n", 883 | "4 2 0 1 [J80 ] 18 \n", 884 | "5 3 0 1 [J80 ] 18 \n", 885 | "6 62 0 30265 [J80 ] 7735330 \n", 886 | "7 70 1 27 [J80 ] 694224 \n", 887 | "8 70 2 9 [J80 ] 37152 \n", 888 | "9 80 0 75008 [J80 ] 22438707 \n", 889 | "10 82 1 1 [J80 ] 268 \n", 890 | "11 82 18 2 [J80 ] 92 \n", 891 | "12 82 20 11430 [J80 ] 1371600 \n", 892 | "13 82 21 6 [J80 ] 672 \n", 893 | "14 82 24 284 [J80 ] 125952 \n", 894 | "15 82 28 9 [J80 ] 2448 \n", 895 | "16 82 31 333 [J80 ] 72072 \n", 896 | "\n", 897 | " first_time_in_buffer last_time_in_buffer earliest_start \\\n", 898 | "0 2019-07-25 00:00:20.550 2019-07-25 05:46:04.660 NaT \n", 899 | "1 2019-07-25 00:00:20.550 2019-07-25 05:46:04.660 NaT \n", 900 | "2 2019-07-25 00:00:00.000 2019-07-25 05:59:57.750 NaT \n", 901 | "3 2019-07-25 00:00:01.990 2019-07-25 05:59:55.880 NaT \n", 902 | "4 2019-07-25 08:15:10.430 2019-07-25 08:15:10.430 NaT \n", 903 | "5 2019-07-25 08:15:16.510 2019-07-25 08:15:16.510 NaT \n", 904 | "6 2019-07-25 00:00:00.510 2019-07-25 05:59:39.070 NaT \n", 905 | "7 2019-07-25 00:29:35.070 2019-07-25 05:59:35.080 2019-07-24 23:59:35 \n", 906 | "8 2019-07-25 00:29:35.140 2019-07-25 05:59:35.200 2019-07-24 23:59:35 \n", 907 | "9 2019-07-25 00:00:00.000 2019-07-25 05:59:59.220 NaT \n", 908 | "10 2019-07-25 02:59:50.040 2019-07-25 02:59:50.040 NaT \n", 909 | "11 2019-07-25 02:59:48.820 2019-07-25 02:59:48.830 NaT \n", 910 | "12 2019-07-25 00:00:00.290 2019-07-25 05:59:58.280 NaT \n", 911 | "13 2019-07-25 02:34:42.940 2019-07-25 02:59:48.640 NaT \n", 912 | "14 2019-07-25 02:59:48.270 2019-07-25 02:59:48.610 NaT \n", 913 | "15 2019-07-25 02:59:52.340 2019-07-25 03:04:36.920 NaT \n", 914 | "16 2019-07-25 00:00:05.030 2019-07-25 05:30:05.000 NaT \n", 915 | "\n", 916 | " lastest_start earliest_end lastest_end \n", 917 | "0 NaT NaT NaT \n", 918 | "1 NaT NaT NaT \n", 919 | "2 NaT NaT NaT \n", 920 | "3 NaT NaT NaT \n", 921 | "4 NaT NaT NaT \n", 922 | "5 NaT NaT NaT \n", 923 | "6 NaT NaT NaT \n", 924 | "7 2019-07-25 05:29:35 2019-07-25 00:29:34.919 2019-07-25 05:59:34.926 \n", 925 | "8 2019-07-25 05:29:35 2019-07-25 00:29:34.919 2019-07-25 05:59:34.926 \n", 926 | "9 NaT NaT NaT \n", 927 | "10 NaT NaT NaT \n", 928 | "11 NaT NaT NaT \n", 929 | "12 NaT NaT NaT \n", 930 | "13 NaT NaT NaT \n", 931 | "14 NaT NaT NaT \n", 932 | "15 NaT NaT NaT \n", 933 | "16 NaT NaT NaT " 934 | ] 935 | }, 936 | "execution_count": 13, 937 | "metadata": {}, 938 | "output_type": "execute_result" 939 | } 940 | ], 941 | "source": [ 942 | "dataset_desc['type_info']" 943 | ] 944 | }, 945 | { 946 | "cell_type": "markdown", 947 | "metadata": {}, 948 | "source": [ 949 | "## Fetching Data\n", 950 | "\n", 951 | "The standard way to fetch the data using the *IBM SMF Explorer* is by defining the list of SMF fields you want to request.\n", 952 | "Therefore, the Framework provides you with predefined definitions for many SMF fields.\n", 953 | "To access those definitions you need to import them from the ``smfexplorer.fields`` module.\n", 954 | "Fields are defined in classes that correspond to the SMF type and subtype.\n", 955 | "\n", 956 | "\n", 957 | "In the following example, we import all fields for the SMF type 70 subtype 1 record.\n", 958 | "\n", 959 | "> **Note**: the following naming scheme: SMF**XX**S**Y**, where *XX* represents record type and *Y* its subtype " 960 | ] 961 | }, 962 | { 963 | "cell_type": "code", 964 | "execution_count": 8, 965 | "metadata": { 966 | "tags": [] 967 | }, 968 | "outputs": [], 969 | "source": [ 970 | "from smfexplorer.fields import SMF70S1" 971 | ] 972 | }, 973 | { 974 | "cell_type": "markdown", 975 | "metadata": {}, 976 | "source": [ 977 | "After importing from the fields module, you can access SMF field documentation by pressing ``shift``+``tab``.\n", 978 | "To test this out for the field ``lpar_name``, just place the cursor behind the ``SMF70S1.lpar_name`` line below and press the key combination.\n", 979 | "Alternatively, you can use the ipython **?** syntax to get help." 980 | ] 981 | }, 982 | { 983 | "cell_type": "code", 984 | "execution_count": 9, 985 | "metadata": { 986 | "tags": [] 987 | }, 988 | "outputs": [ 989 | { 990 | "data": { 991 | "text/plain": [ 992 | ">>" 993 | ] 994 | }, 995 | "execution_count": 9, 996 | "metadata": {}, 997 | "output_type": "execute_result" 998 | } 999 | ], 1000 | "source": [ 1001 | "# select this cell and press shift+tab to see the documentation\n", 1002 | "SMF70S1.lpar_busy" 1003 | ] 1004 | }, 1005 | { 1006 | "cell_type": "code", 1007 | "execution_count": null, 1008 | "metadata": {}, 1009 | "outputs": [], 1010 | "source": [ 1011 | "?SMF70S1.lpar_name" 1012 | ] 1013 | }, 1014 | { 1015 | "cell_type": "markdown", 1016 | "metadata": {}, 1017 | "source": [ 1018 | "Such information is provided on each field.\n", 1019 | "Feel free to have a look into the properties available.\n", 1020 | "\n", 1021 | "To make working with SMF data easier, additional **virtual fields** are provided.\n", 1022 | "Virtual fields are derived from the SMF fields by the *IBM SMF Explorer*.\n", 1023 | "For example, the ``ziip_boost`` field shows whether zIIP Boost was active." 1024 | ] 1025 | }, 1026 | { 1027 | "cell_type": "markdown", 1028 | "metadata": {}, 1029 | "source": [ 1030 | "The virtual field ``ziip_boost`` uses ``fla``(SMF70FLA) field to extract the information." 1031 | ] 1032 | }, 1033 | { 1034 | "cell_type": "code", 1035 | "execution_count": null, 1036 | "metadata": {}, 1037 | "outputs": [], 1038 | "source": [ 1039 | "?SMF70S1.ziip_boost" 1040 | ] 1041 | }, 1042 | { 1043 | "cell_type": "markdown", 1044 | "metadata": {}, 1045 | "source": [ 1046 | "### Request() Method" 1047 | ] 1048 | }, 1049 | { 1050 | "cell_type": "markdown", 1051 | "metadata": {}, 1052 | "source": [ 1053 | "To use the field definitions for data fetching, you can use the ``request()`` method of a Context.\n", 1054 | "You need to provide an array of fields you would like to request.\n", 1055 | "To trigger the request call ``.run()``.\n", 1056 | "When the request succeeds, it returns a pandas DataFrame." 1057 | ] 1058 | }, 1059 | { 1060 | "cell_type": "code", 1061 | "execution_count": 10, 1062 | "metadata": { 1063 | "tags": [] 1064 | }, 1065 | "outputs": [ 1066 | { 1067 | "data": { 1068 | "text/html": [ 1069 | "
\n", 1070 | "\n", 1083 | "\n", 1084 | " \n", 1085 | " \n", 1086 | " \n", 1087 | " \n", 1088 | " \n", 1089 | " \n", 1090 | " \n", 1091 | " \n", 1092 | " \n", 1093 | " \n", 1094 | " \n", 1095 | " \n", 1096 | " \n", 1097 | " \n", 1098 | " \n", 1099 | " \n", 1100 | " \n", 1101 | " \n", 1102 | " \n", 1103 | " \n", 1104 | " \n", 1105 | " \n", 1106 | " \n", 1107 | " \n", 1108 | " \n", 1109 | " \n", 1110 | " \n", 1111 | " \n", 1112 | " \n", 1113 | " \n", 1114 | " \n", 1115 | " \n", 1116 | " \n", 1117 | " \n", 1118 | " \n", 1119 | " \n", 1120 | " \n", 1121 | " \n", 1122 | " \n", 1123 | " \n", 1124 | " \n", 1125 | " \n", 1126 | " \n", 1127 | " \n", 1128 | " \n", 1129 | " \n", 1130 | " \n", 1131 | " \n", 1132 | " \n", 1133 | " \n", 1134 | " \n", 1135 | " \n", 1136 | " \n", 1137 | " \n", 1138 | " \n", 1139 | " \n", 1140 | " \n", 1141 | " \n", 1142 | " \n", 1143 | " \n", 1144 | " \n", 1145 | " \n", 1146 | " \n", 1147 | " \n", 1148 | " \n", 1149 | " \n", 1150 | " \n", 1151 | " \n", 1152 | " \n", 1153 | " \n", 1154 | " \n", 1155 | " \n", 1156 | " \n", 1157 | " \n", 1158 | " \n", 1159 | " \n", 1160 | " \n", 1161 | " \n", 1162 | " \n", 1163 | " \n", 1164 | " \n", 1165 | " \n", 1166 | " \n", 1167 | " \n", 1168 | " \n", 1169 | " \n", 1170 | " \n", 1171 | " \n", 1172 | " \n", 1173 | " \n", 1174 | " \n", 1175 | " \n", 1176 | " \n", 1177 | " \n", 1178 | " \n", 1179 | " \n", 1180 | " \n", 1181 | " \n", 1182 | " \n", 1183 | " \n", 1184 | " \n", 1185 | " \n", 1186 | " \n", 1187 | " \n", 1188 | " \n", 1189 | " \n", 1190 | " \n", 1191 | " \n", 1192 | " \n", 1193 | " \n", 1194 | " \n", 1195 | " \n", 1196 | " \n", 1197 | " \n", 1198 | " \n", 1199 | " \n", 1200 | " \n", 1201 | " \n", 1202 | " \n", 1203 | " \n", 1204 | " \n", 1205 | " \n", 1206 | " \n", 1207 | " \n", 1208 | " \n", 1209 | " \n", 1210 | " \n", 1211 | " \n", 1212 | " \n", 1213 | " \n", 1214 | " \n", 1215 | " \n", 1216 | " \n", 1217 | " \n", 1218 | " \n", 1219 | " \n", 1220 | "
timestampsidlpar_namesystem_namesysplex_namelpar_system_namelpar_numberlpar_cpu_count
02019-07-25 00:29:35.070J80J80J80UTCPLXJ8J80-J80788
12019-07-25 00:29:35.070J80CF22<NA>CF22-11
22019-07-25 00:29:35.070J80CF3<NA>CF3-26
32019-07-25 00:29:35.070J80CT2CT2CT2PLEXCT2-CT2315
42019-07-25 00:29:35.070J80JA0JA0UTCPLXJ8JA0-JA0484
...........................
2472019-07-25 05:59:35.080J80JJ0<NA><NA><NA>240
2482019-07-25 05:59:35.080J80Z2Z2ZPETPLX2Z2-Z22535
2492019-07-25 05:59:35.080J80CT1<NA><NA><NA>260
2502019-07-25 05:59:35.080J80ISKLMLX1<NA>ISKLMLX1-271
2512019-07-25 05:59:35.080J80PHYSICAL<NA><NA><NA>0167
\n", 1221 | "

252 rows × 8 columns

\n", 1222 | "
" 1223 | ], 1224 | "text/plain": [ 1225 | " timestamp sid lpar_name system_name sysplex_name \\\n", 1226 | "0 2019-07-25 00:29:35.070 J80 J80 J80 UTCPLXJ8 \n", 1227 | "1 2019-07-25 00:29:35.070 J80 CF22 \n", 1228 | "2 2019-07-25 00:29:35.070 J80 CF3 \n", 1229 | "3 2019-07-25 00:29:35.070 J80 CT2 CT2 CT2PLEX \n", 1230 | "4 2019-07-25 00:29:35.070 J80 JA0 JA0 UTCPLXJ8 \n", 1231 | ".. ... ... ... ... ... \n", 1232 | "247 2019-07-25 05:59:35.080 J80 JJ0 \n", 1233 | "248 2019-07-25 05:59:35.080 J80 Z2 Z2 ZPETPLX2 \n", 1234 | "249 2019-07-25 05:59:35.080 J80 CT1 \n", 1235 | "250 2019-07-25 05:59:35.080 J80 ISKLMLX1 \n", 1236 | "251 2019-07-25 05:59:35.080 J80 PHYSICAL \n", 1237 | "\n", 1238 | " lpar_system_name lpar_number lpar_cpu_count \n", 1239 | "0 J80-J80 7 88 \n", 1240 | "1 CF22- 1 1 \n", 1241 | "2 CF3- 2 6 \n", 1242 | "3 CT2-CT2 3 15 \n", 1243 | "4 JA0-JA0 4 84 \n", 1244 | ".. ... ... ... \n", 1245 | "247 24 0 \n", 1246 | "248 Z2-Z2 25 35 \n", 1247 | "249 26 0 \n", 1248 | "250 ISKLMLX1- 27 1 \n", 1249 | "251 0 167 \n", 1250 | "\n", 1251 | "[252 rows x 8 columns]" 1252 | ] 1253 | }, 1254 | "execution_count": 10, 1255 | "metadata": {}, 1256 | "output_type": "execute_result" 1257 | } 1258 | ], 1259 | "source": [ 1260 | "ctx.request(\n", 1261 | " [\n", 1262 | " SMF70S1.timestamp,\n", 1263 | " SMF70S1.sid,\n", 1264 | " SMF70S1.lpar_name,\n", 1265 | " SMF70S1.system_name,\n", 1266 | " SMF70S1.sysplex_name,\n", 1267 | " SMF70S1.lpar_system_name,\n", 1268 | " SMF70S1.lpar_number,\n", 1269 | " SMF70S1.lpar_cpu_count,\n", 1270 | " ]\n", 1271 | ").run()" 1272 | ] 1273 | }, 1274 | { 1275 | "cell_type": "markdown", 1276 | "metadata": {}, 1277 | "source": [ 1278 | "The *IBM SMF Explorer* is a Fluent-API.\n", 1279 | "That means you chain methods together to configure a request.\n", 1280 | "You have seen the most basic form of such a chain: ``request().run()``. The ``request()`` method can start a chain and the ``run()`` method ends it.\n", 1281 | "You will learn about other methods that can be used in the chain later and in other tutorials.\n", 1282 | "\n", 1283 | "Note, that you cannot combine fields arbitrarily in a request.\n", 1284 | "Not all fields are compatible, because they may originate from different structures and cannot be displayed in a single table in a logical/useful way.\n", 1285 | "\n", 1286 | "The following request, for example, tries to combine SMF 70 Subtype 1 and SMF 72 Subtype 3 data into one table and **throws an exception**.\n", 1287 | "In general, you cannot combine fields of different record types.\n", 1288 | "\n", 1289 | "**WARNING**: the following code causes an error" 1290 | ] 1291 | }, 1292 | { 1293 | "cell_type": "code", 1294 | "execution_count": null, 1295 | "metadata": { 1296 | "tags": [] 1297 | }, 1298 | "outputs": [], 1299 | "source": [ 1300 | "from smfexplorer.fields import SMF72S3\n", 1301 | "\n", 1302 | "ctx.request([SMF70S1.timestamp, SMF70S1.sid, SMF72S3.utilization_total]).run()" 1303 | ] 1304 | }, 1305 | { 1306 | "cell_type": "markdown", 1307 | "metadata": {}, 1308 | "source": [ 1309 | "#### Raw fields\n", 1310 | "\n", 1311 | "The *IBM SMF Explorer* applies different transformations to the raw SMF data before returning it to the user.\n", 1312 | "Sometimes you might want to disable the post-processing of some fields to get the original values.\n", 1313 | "\n", 1314 | "You can use the `raw` property of each field to see raw SMF values.\n", 1315 | "What `raw` returns depends on the type of the field.\n", 1316 | "For normal fields, post-processing is disabled.\n", 1317 | "For virtual fields, the raw value of the source field will be returned.\n", 1318 | "\n", 1319 | "In the example below, you can see how `raw` fetches the `cpu_type` value without the *IBM SMF Explorer* post-processing." 1320 | ] 1321 | }, 1322 | { 1323 | "cell_type": "code", 1324 | "execution_count": 13, 1325 | "metadata": { 1326 | "tags": [] 1327 | }, 1328 | "outputs": [ 1329 | { 1330 | "data": { 1331 | "text/html": [ 1332 | "
\n", 1333 | "\n", 1346 | "\n", 1347 | " \n", 1348 | " \n", 1349 | " \n", 1350 | " \n", 1351 | " \n", 1352 | " \n", 1353 | " \n", 1354 | " \n", 1355 | " \n", 1356 | " \n", 1357 | " \n", 1358 | " \n", 1359 | " \n", 1360 | " \n", 1361 | " \n", 1362 | " \n", 1363 | " \n", 1364 | " \n", 1365 | " \n", 1366 | " \n", 1367 | " \n", 1368 | " \n", 1369 | " \n", 1370 | " \n", 1371 | " \n", 1372 | " \n", 1373 | " \n", 1374 | " \n", 1375 | " \n", 1376 | " \n", 1377 | " \n", 1378 | " \n", 1379 | " \n", 1380 | " \n", 1381 | " \n", 1382 | " \n", 1383 | " \n", 1384 | " \n", 1385 | " \n", 1386 | " \n", 1387 | " \n", 1388 | " \n", 1389 | " \n", 1390 | " \n", 1391 | " \n", 1392 | " \n", 1393 | " \n", 1394 | " \n", 1395 | " \n", 1396 | " \n", 1397 | " \n", 1398 | " \n", 1399 | " \n", 1400 | " \n", 1401 | " \n", 1402 | " \n", 1403 | " \n", 1404 | " \n", 1405 | " \n", 1406 | " \n", 1407 | " \n", 1408 | " \n", 1409 | " \n", 1410 | " \n", 1411 | "
cpu_typecpu_type_raw
0CP0
1CP0
2CP0
3CP0
4CP0
.........
643zIIP2
644zIIP2
645zIIP2
646zIIP2
647zIIP2
\n", 1412 | "

648 rows × 2 columns

\n", 1413 | "
" 1414 | ], 1415 | "text/plain": [ 1416 | " cpu_type cpu_type_raw\n", 1417 | "0 CP 0\n", 1418 | "1 CP 0\n", 1419 | "2 CP 0\n", 1420 | "3 CP 0\n", 1421 | "4 CP 0\n", 1422 | ".. ... ...\n", 1423 | "643 zIIP 2\n", 1424 | "644 zIIP 2\n", 1425 | "645 zIIP 2\n", 1426 | "646 zIIP 2\n", 1427 | "647 zIIP 2\n", 1428 | "\n", 1429 | "[648 rows x 2 columns]" 1430 | ] 1431 | }, 1432 | "execution_count": 13, 1433 | "metadata": {}, 1434 | "output_type": "execute_result" 1435 | } 1436 | ], 1437 | "source": [ 1438 | "ctx.request(\n", 1439 | " [\n", 1440 | " SMF70S1.cpu_type,\n", 1441 | " SMF70S1.cpu_type.raw,\n", 1442 | " ]\n", 1443 | ").run()" 1444 | ] 1445 | }, 1446 | { 1447 | "cell_type": "markdown", 1448 | "metadata": {}, 1449 | "source": [ 1450 | "### Using Sample Requests\n", 1451 | "To make common tasks easier, *IBM SMF Explorer* comes with a collection of predefined requests.\n", 1452 | "The data can be fetched from related fields without listing them individually.\n", 1453 | "These requests can be found in the ``samples`` property of any context.\n", 1454 | "In other tutorials, you will be shown how to create and register your own samples.\n", 1455 | "\n", 1456 | "To fetch the same information as above (i.e., in the first successful ``ctx.request()`` call), we can use the ``lpar_information()`` sample request." 1457 | ] 1458 | }, 1459 | { 1460 | "cell_type": "code", 1461 | "execution_count": 12, 1462 | "metadata": { 1463 | "tags": [] 1464 | }, 1465 | "outputs": [ 1466 | { 1467 | "data": { 1468 | "text/html": [ 1469 | "
\n", 1470 | "\n", 1483 | "\n", 1484 | " \n", 1485 | " \n", 1486 | " \n", 1487 | " \n", 1488 | " \n", 1489 | " \n", 1490 | " \n", 1491 | " \n", 1492 | " \n", 1493 | " \n", 1494 | " \n", 1495 | " \n", 1496 | " \n", 1497 | " \n", 1498 | " \n", 1499 | " \n", 1500 | " \n", 1501 | " \n", 1502 | " \n", 1503 | " \n", 1504 | " \n", 1505 | " \n", 1506 | " \n", 1507 | " \n", 1508 | " \n", 1509 | " \n", 1510 | " \n", 1511 | " \n", 1512 | " \n", 1513 | " \n", 1514 | " \n", 1515 | " \n", 1516 | " \n", 1517 | " \n", 1518 | " \n", 1519 | " \n", 1520 | " \n", 1521 | " \n", 1522 | " \n", 1523 | " \n", 1524 | " \n", 1525 | " \n", 1526 | " \n", 1527 | " \n", 1528 | " \n", 1529 | " \n", 1530 | " \n", 1531 | " \n", 1532 | " \n", 1533 | " \n", 1534 | " \n", 1535 | " \n", 1536 | " \n", 1537 | " \n", 1538 | " \n", 1539 | " \n", 1540 | " \n", 1541 | " \n", 1542 | " \n", 1543 | " \n", 1544 | " \n", 1545 | " \n", 1546 | " \n", 1547 | " \n", 1548 | " \n", 1549 | " \n", 1550 | " \n", 1551 | " \n", 1552 | " \n", 1553 | " \n", 1554 | " \n", 1555 | " \n", 1556 | " \n", 1557 | " \n", 1558 | " \n", 1559 | " \n", 1560 | " \n", 1561 | " \n", 1562 | " \n", 1563 | " \n", 1564 | " \n", 1565 | " \n", 1566 | " \n", 1567 | " \n", 1568 | " \n", 1569 | " \n", 1570 | " \n", 1571 | " \n", 1572 | " \n", 1573 | " \n", 1574 | " \n", 1575 | " \n", 1576 | " \n", 1577 | " \n", 1578 | " \n", 1579 | " \n", 1580 | " \n", 1581 | " \n", 1582 | " \n", 1583 | " \n", 1584 | " \n", 1585 | " \n", 1586 | " \n", 1587 | " \n", 1588 | " \n", 1589 | " \n", 1590 | " \n", 1591 | " \n", 1592 | " \n", 1593 | " \n", 1594 | " \n", 1595 | " \n", 1596 | " \n", 1597 | " \n", 1598 | " \n", 1599 | " \n", 1600 | " \n", 1601 | " \n", 1602 | " \n", 1603 | " \n", 1604 | " \n", 1605 | " \n", 1606 | " \n", 1607 | " \n", 1608 | " \n", 1609 | " \n", 1610 | " \n", 1611 | " \n", 1612 | " \n", 1613 | " \n", 1614 | " \n", 1615 | " \n", 1616 | " \n", 1617 | " \n", 1618 | " \n", 1619 | " \n", 1620 | "
timestampsidlpar_namesystem_namesysplex_namelpar_system_namelpar_numberlpar_cpu_count
02019-07-25 00:29:35.070J80J80J80UTCPLXJ8J80-J80788
12019-07-25 00:29:35.070J80CF22<NA>CF22-11
22019-07-25 00:29:35.070J80CF3<NA>CF3-26
32019-07-25 00:29:35.070J80CT2CT2CT2PLEXCT2-CT2315
42019-07-25 00:29:35.070J80JA0JA0UTCPLXJ8JA0-JA0484
...........................
2472019-07-25 05:59:35.080J80JJ0<NA><NA><NA>240
2482019-07-25 05:59:35.080J80Z2Z2ZPETPLX2Z2-Z22535
2492019-07-25 05:59:35.080J80CT1<NA><NA><NA>260
2502019-07-25 05:59:35.080J80ISKLMLX1<NA>ISKLMLX1-271
2512019-07-25 05:59:35.080J80PHYSICAL<NA><NA><NA>0167
\n", 1621 | "

252 rows × 8 columns

\n", 1622 | "
" 1623 | ], 1624 | "text/plain": [ 1625 | " timestamp sid lpar_name system_name sysplex_name \\\n", 1626 | "0 2019-07-25 00:29:35.070 J80 J80 J80 UTCPLXJ8 \n", 1627 | "1 2019-07-25 00:29:35.070 J80 CF22 \n", 1628 | "2 2019-07-25 00:29:35.070 J80 CF3 \n", 1629 | "3 2019-07-25 00:29:35.070 J80 CT2 CT2 CT2PLEX \n", 1630 | "4 2019-07-25 00:29:35.070 J80 JA0 JA0 UTCPLXJ8 \n", 1631 | ".. ... ... ... ... ... \n", 1632 | "247 2019-07-25 05:59:35.080 J80 JJ0 \n", 1633 | "248 2019-07-25 05:59:35.080 J80 Z2 Z2 ZPETPLX2 \n", 1634 | "249 2019-07-25 05:59:35.080 J80 CT1 \n", 1635 | "250 2019-07-25 05:59:35.080 J80 ISKLMLX1 \n", 1636 | "251 2019-07-25 05:59:35.080 J80 PHYSICAL \n", 1637 | "\n", 1638 | " lpar_system_name lpar_number lpar_cpu_count \n", 1639 | "0 J80-J80 7 88 \n", 1640 | "1 CF22- 1 1 \n", 1641 | "2 CF3- 2 6 \n", 1642 | "3 CT2-CT2 3 15 \n", 1643 | "4 JA0-JA0 4 84 \n", 1644 | ".. ... ... ... \n", 1645 | "247 24 0 \n", 1646 | "248 Z2-Z2 25 35 \n", 1647 | "249 26 0 \n", 1648 | "250 ISKLMLX1- 27 1 \n", 1649 | "251 0 167 \n", 1650 | "\n", 1651 | "[252 rows x 8 columns]" 1652 | ] 1653 | }, 1654 | "execution_count": 12, 1655 | "metadata": {}, 1656 | "output_type": "execute_result" 1657 | } 1658 | ], 1659 | "source": [ 1660 | "ctx.samples.lpar_information().run()" 1661 | ] 1662 | }, 1663 | { 1664 | "cell_type": "markdown", 1665 | "metadata": {}, 1666 | "source": [ 1667 | "To fetch additional fields together with a sample, the ``run()`` method has a named parameter called ``display``.\n", 1668 | "You can provide an array of fields to this parameter just like in the ``request()`` method." 1669 | ] 1670 | }, 1671 | { 1672 | "cell_type": "code", 1673 | "execution_count": 11, 1674 | "metadata": { 1675 | "tags": [] 1676 | }, 1677 | "outputs": [ 1678 | { 1679 | "data": { 1680 | "text/html": [ 1681 | "
\n", 1682 | "\n", 1695 | "\n", 1696 | " \n", 1697 | " \n", 1698 | " \n", 1699 | " \n", 1700 | " \n", 1701 | " \n", 1702 | " \n", 1703 | " \n", 1704 | " \n", 1705 | " \n", 1706 | " \n", 1707 | " \n", 1708 | " \n", 1709 | " \n", 1710 | " \n", 1711 | " \n", 1712 | " \n", 1713 | " \n", 1714 | " \n", 1715 | " \n", 1716 | " \n", 1717 | " \n", 1718 | " \n", 1719 | " \n", 1720 | " \n", 1721 | " \n", 1722 | " \n", 1723 | " \n", 1724 | " \n", 1725 | " \n", 1726 | " \n", 1727 | " \n", 1728 | " \n", 1729 | " \n", 1730 | " \n", 1731 | " \n", 1732 | " \n", 1733 | " \n", 1734 | " \n", 1735 | " \n", 1736 | " \n", 1737 | " \n", 1738 | " \n", 1739 | " \n", 1740 | " \n", 1741 | " \n", 1742 | " \n", 1743 | " \n", 1744 | " \n", 1745 | " \n", 1746 | " \n", 1747 | " \n", 1748 | " \n", 1749 | " \n", 1750 | " \n", 1751 | " \n", 1752 | " \n", 1753 | " \n", 1754 | " \n", 1755 | " \n", 1756 | " \n", 1757 | " \n", 1758 | " \n", 1759 | " \n", 1760 | " \n", 1761 | " \n", 1762 | " \n", 1763 | " \n", 1764 | " \n", 1765 | " \n", 1766 | " \n", 1767 | " \n", 1768 | " \n", 1769 | " \n", 1770 | " \n", 1771 | " \n", 1772 | " \n", 1773 | " \n", 1774 | " \n", 1775 | " \n", 1776 | " \n", 1777 | " \n", 1778 | " \n", 1779 | " \n", 1780 | " \n", 1781 | " \n", 1782 | " \n", 1783 | " \n", 1784 | " \n", 1785 | " \n", 1786 | " \n", 1787 | " \n", 1788 | " \n", 1789 | " \n", 1790 | " \n", 1791 | " \n", 1792 | " \n", 1793 | " \n", 1794 | " \n", 1795 | " \n", 1796 | " \n", 1797 | " \n", 1798 | " \n", 1799 | " \n", 1800 | " \n", 1801 | " \n", 1802 | " \n", 1803 | " \n", 1804 | " \n", 1805 | " \n", 1806 | " \n", 1807 | " \n", 1808 | " \n", 1809 | " \n", 1810 | " \n", 1811 | " \n", 1812 | " \n", 1813 | " \n", 1814 | " \n", 1815 | " \n", 1816 | " \n", 1817 | " \n", 1818 | " \n", 1819 | " \n", 1820 | " \n", 1821 | " \n", 1822 | " \n", 1823 | " \n", 1824 | " \n", 1825 | " \n", 1826 | " \n", 1827 | " \n", 1828 | " \n", 1829 | " \n", 1830 | " \n", 1831 | " \n", 1832 | " \n", 1833 | " \n", 1834 | " \n", 1835 | " \n", 1836 | " \n", 1837 | " \n", 1838 | " \n", 1839 | " \n", 1840 | " \n", 1841 | " \n", 1842 | " \n", 1843 | " \n", 1844 | "
timestampsidlpar_namesystem_namesysplex_namelpar_system_namelpar_numberlpar_cpu_countcapactiy_group_name
02019-07-25 00:29:35.070J80J80J80UTCPLXJ8J80-J80788<NA>
12019-07-25 00:29:35.070J80CF22<NA>CF22-11<NA>
22019-07-25 00:29:35.070J80CF3<NA>CF3-26<NA>
32019-07-25 00:29:35.070J80CT2CT2CT2PLEXCT2-CT2315<NA>
42019-07-25 00:29:35.070J80JA0JA0UTCPLXJ8JA0-JA0484<NA>
..............................
2472019-07-25 05:59:35.080J80JJ0<NA><NA><NA>240<NA>
2482019-07-25 05:59:35.080J80Z2Z2ZPETPLX2Z2-Z22535<NA>
2492019-07-25 05:59:35.080J80CT1<NA><NA><NA>260<NA>
2502019-07-25 05:59:35.080J80ISKLMLX1<NA>ISKLMLX1-271<NA>
2512019-07-25 05:59:35.080J80PHYSICAL<NA><NA><NA>0167<NA>
\n", 1845 | "

252 rows × 9 columns

\n", 1846 | "
" 1847 | ], 1848 | "text/plain": [ 1849 | " timestamp sid lpar_name system_name sysplex_name \\\n", 1850 | "0 2019-07-25 00:29:35.070 J80 J80 J80 UTCPLXJ8 \n", 1851 | "1 2019-07-25 00:29:35.070 J80 CF22 \n", 1852 | "2 2019-07-25 00:29:35.070 J80 CF3 \n", 1853 | "3 2019-07-25 00:29:35.070 J80 CT2 CT2 CT2PLEX \n", 1854 | "4 2019-07-25 00:29:35.070 J80 JA0 JA0 UTCPLXJ8 \n", 1855 | ".. ... ... ... ... ... \n", 1856 | "247 2019-07-25 05:59:35.080 J80 JJ0 \n", 1857 | "248 2019-07-25 05:59:35.080 J80 Z2 Z2 ZPETPLX2 \n", 1858 | "249 2019-07-25 05:59:35.080 J80 CT1 \n", 1859 | "250 2019-07-25 05:59:35.080 J80 ISKLMLX1 \n", 1860 | "251 2019-07-25 05:59:35.080 J80 PHYSICAL \n", 1861 | "\n", 1862 | " lpar_system_name lpar_number lpar_cpu_count capactiy_group_name \n", 1863 | "0 J80-J80 7 88 \n", 1864 | "1 CF22- 1 1 \n", 1865 | "2 CF3- 2 6 \n", 1866 | "3 CT2-CT2 3 15 \n", 1867 | "4 JA0-JA0 4 84 \n", 1868 | ".. ... ... ... ... \n", 1869 | "247 24 0 \n", 1870 | "248 Z2-Z2 25 35 \n", 1871 | "249 26 0 \n", 1872 | "250 ISKLMLX1- 27 1 \n", 1873 | "251 0 167 \n", 1874 | "\n", 1875 | "[252 rows x 9 columns]" 1876 | ] 1877 | }, 1878 | "execution_count": 11, 1879 | "metadata": {}, 1880 | "output_type": "execute_result" 1881 | } 1882 | ], 1883 | "source": [ 1884 | "ctx.samples.lpar_information().run(display=[SMF70S1.capactiy_group_name])" 1885 | ] 1886 | }, 1887 | { 1888 | "cell_type": "markdown", 1889 | "metadata": {}, 1890 | "source": [ 1891 | "# Some available Samples\n", 1892 | "\n", 1893 | "## Samples for SMF70S1:\n", 1894 | "\n", 1895 | "``lpar_information()`` -- fields from SMF70S1 on LPARs\n", 1896 | "\n", 1897 | "``processor_information()`` -- fields from SMF70S1 on processors\n", 1898 | "\n", 1899 | "## Samples for SMF72S3:\n", 1900 | "\n", 1901 | "``smf_72_03_sample()`` -- fields for SMF72 subtype 3 analysis\n", 1902 | "\n", 1903 | "\n", 1904 | "## Samples for SMF99S1:\n", 1905 | " \n", 1906 | "``p_utilization()`` -- CP, zIIP, zAAP and total utilization\n", 1907 | " \n", 1908 | "``rg_capping()`` -- Resource Group and Tenant Resource Group capping information\n", 1909 | " \n", 1910 | "``smf_99_01_sample()`` -- commonly used SMF 99 subtype 1 data\n", 1911 | "\n", 1912 | "\n", 1913 | "## Samples for SMF99S2:\n", 1914 | " \n", 1915 | "\n", 1916 | "``srv_service()`` -- service Class service consumption for CP, zIIP and zAAP\n", 1917 | "\n", 1918 | "``smf_99_02_sample()`` -- commonly used SMF 99 Subtype 2 data\n", 1919 | "\n", 1920 | "## Samples for SMF99S12:\n", 1921 | "\n", 1922 | "``hiper_dispatch()`` -- hiper dispatch information per processor type\n", 1923 | "\n", 1924 | "## Samples for SMF99S14:\n", 1925 | "\n", 1926 | "``topology()`` -- topology information per processor\n" 1927 | ] 1928 | } 1929 | ], 1930 | "metadata": { 1931 | "kernelspec": { 1932 | "display_name": "Python 3 (ipykernel)", 1933 | "language": "python", 1934 | "name": "python3" 1935 | }, 1936 | "language_info": { 1937 | "codemirror_mode": { 1938 | "name": "ipython", 1939 | "version": 3 1940 | }, 1941 | "file_extension": ".py", 1942 | "mimetype": "text/x-python", 1943 | "name": "python", 1944 | "nbconvert_exporter": "python", 1945 | "pygments_lexer": "ipython3", 1946 | "version": "3.9.10" 1947 | } 1948 | }, 1949 | "nbformat": 4, 1950 | "nbformat_minor": 4 1951 | } 1952 | --------------------------------------------------------------------------------