├── .gitattributes ├── .gitignore ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── beamer ├── beamercolorthemepop.sty ├── beamerfontthemepop.sty ├── beamerinnerthemepop.sty ├── beamerouterthemepop.sty ├── beamerthemepop.sty ├── eu_logo.pdf ├── flag_yellow_eps.ps ├── pop_logo.eps └── test.tex ├── bin ├── chop_and_analyze.sh ├── chop_roi.sh ├── extraempirun.sh ├── ipynb_to_report └── modelfactors2precomputed.py ├── doc ├── Makefile └── source │ ├── _static │ └── css │ │ └── kbd.css │ ├── api_hl.rst │ ├── api_ll.rst │ ├── conf.py │ ├── doc.rst │ ├── install.rst │ ├── layout.rst │ └── quickstart.rst ├── ipython_templates ├── graphics │ ├── EU-logo.jpg │ └── pop-logo.png ├── html_nocode.tpl ├── pop_report.cls └── pop_report.tplx ├── presentations └── launch_blog │ └── launch_blog.md ├── pypop ├── __init__.py ├── _plotsettings.py ├── backends │ ├── __init__.py │ └── extrae │ │ ├── __init__.py │ │ └── pcf.py ├── cfgs │ ├── clock_frequency.cfg │ ├── dimemas_ideal.cfg │ ├── dimemas_ideal.tmpconfig.cfg │ ├── ideal.collectives │ ├── omp_total_runtime.cfg │ ├── omp_total_runtime_loop.cfg │ ├── omp_total_runtime_task.cfg │ ├── omp_useful_computation.cfg │ ├── omp_useful_computation_loop.cfg │ ├── omp_useful_computation_task.cfg │ ├── serial_useful_computation.cfg │ ├── serial_useful_computation_no_omp.cfg │ ├── serial_useful_computation_omp_loop.cfg │ ├── serial_useful_computation_omp_task.cfg │ ├── total_res_stl_cycles.cfg │ ├── total_runtime.cfg │ ├── total_runtime_excl_disabled.cfg │ ├── total_useful_computation.cfg │ ├── useful_cycles.cfg │ └── useful_instructions.cfg ├── cli │ ├── __init__.py │ └── cli.py ├── config.py ├── cutters │ └── single_region.skel ├── dimemas.py ├── examples.py ├── examples │ ├── AutoMetrics.ipynb │ ├── html_examples │ │ ├── README.md │ │ ├── insight_blog.html │ │ └── insight_blog.ipynb │ ├── mpi │ │ ├── MPI Metrics Example.ipynb │ │ ├── epoch_example_traces │ │ │ ├── epoch_16proc.pcf │ │ │ ├── epoch_16proc.prv.gz │ │ │ ├── epoch_16proc.row │ │ │ ├── epoch_1proc.pcf │ │ │ ├── epoch_1proc.prv.gz │ │ │ ├── epoch_1proc.row │ │ │ ├── epoch_2proc.pcf │ │ │ ├── epoch_2proc.prv.gz │ │ │ ├── epoch_2proc.row │ │ │ ├── epoch_4proc.pcf │ │ │ ├── epoch_4proc.prv.gz │ │ │ ├── epoch_4proc.row │ │ │ ├── epoch_8proc.pcf │ │ │ ├── epoch_8proc.prv.gz │ │ │ ├── epoch_8proc.row │ │ │ ├── input.deck │ │ │ ├── modelfactors.csv │ │ │ └── precomputed.csv │ │ └── pypop_gui.ipynb │ └── openmp │ │ ├── Analyze OpenMP Regions.ipynb │ │ ├── OMP Metrics Example.ipynb │ │ ├── extrae-omp.xml │ │ ├── imagemagick_example_traces │ │ ├── omp1.pcf │ │ ├── omp1.prv.gz │ │ ├── omp1.row │ │ ├── omp2.pcf │ │ ├── omp2.prv.gz │ │ ├── omp2.row │ │ ├── omp4.pcf │ │ ├── omp4.prv.gz │ │ ├── omp4.row │ │ ├── omp6.pcf │ │ ├── omp6.prv.gz │ │ ├── omp6.row │ │ ├── omp8.pcf │ │ ├── omp8.prv.gz │ │ └── omp8.row │ │ ├── omp_analyze.ipynb │ │ ├── omp_detail.pcf │ │ ├── omp_detail.prv.gz │ │ ├── omp_detail.row │ │ └── profile_imagemagick.sh ├── extrae.py ├── filters │ └── tracing_state.xml ├── metrics │ ├── __init__.py │ ├── hybrid.py │ ├── judit.py │ ├── metricset.py │ ├── mpi.py │ ├── mpi_openmp_ineff.py │ ├── multiplot.py │ ├── openmp.py │ └── threads.py ├── mplplotting │ ├── __init__.py │ ├── builtin.py │ └── mplplotbase.py ├── notebook_interface │ ├── __init__.py │ ├── bokeh_widget.py │ ├── fileselector.py │ ├── omp_explorer.py │ ├── palettes.py │ ├── plotting.py │ ├── reporting.py │ ├── tqdm_widget.py │ └── wizard.py ├── prv.py ├── server.py ├── trace │ ├── __init__.py │ ├── prvtrace.py │ ├── trace.py │ └── tracemetadata.py ├── traceset.py ├── utils │ ├── __init__.py │ ├── asserters.py │ ├── exceptions.py │ ├── io.py │ ├── pandas.py │ ├── plot.py │ ├── pop_logo.npy │ └── tex.py └── version.py ├── pytest.ini ├── setup.py ├── tests ├── integration │ └── test_mpi_epoch.py └── unit │ └── test_traceset.py ├── util └── publish_doc.sh └── versionate.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb diff=jupyternotebook 2 | *.ipynb merge=jupyternotebook 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pypop/version 2 | __pycache__ 3 | *.egg-info 4 | *.pkl 5 | *.png 6 | *.tiff 7 | gh-pages 8 | *.pypopsummary 9 | 10 | ## Core latex/pdflatex auxiliary files: 11 | *.aux 12 | *.lof 13 | *.log 14 | *.lot 15 | *.fls 16 | *.out 17 | *.toc 18 | *.fmt 19 | *.fot 20 | *.cb 21 | *.cb2 22 | .*.lb 23 | 24 | ## Intermediate documents: 25 | *.dvi 26 | *.xdv 27 | *-converted-to.* 28 | # these rules might exclude image files for figures etc. 29 | # *.ps 30 | # *.eps 31 | # *.pdf 32 | 33 | ## Generated if empty string is given at "Please type another file name for output:" 34 | .pdf 35 | 36 | ## Bibliography auxiliary files (bibtex/biblatex/biber): 37 | *.bbl 38 | *.bcf 39 | *.blg 40 | *-blx.aux 41 | *-blx.bib 42 | *.run.xml 43 | 44 | ## Build tool auxiliary files: 45 | *.fdb_latexmk 46 | *.synctex 47 | *.synctex(busy) 48 | *.synctex.gz 49 | *.synctex.gz(busy) 50 | *.pdfsync 51 | 52 | ## Build tool directories for auxiliary files 53 | # latexrun 54 | latex.out/ 55 | 56 | ## Auxiliary and intermediate files from other packages: 57 | # algorithms 58 | *.alg 59 | *.loa 60 | 61 | # achemso 62 | acs-*.bib 63 | 64 | # amsthm 65 | *.thm 66 | 67 | # beamer 68 | *.nav 69 | *.pre 70 | *.snm 71 | *.vrb 72 | 73 | # changes 74 | *.soc 75 | 76 | # comment 77 | *.cut 78 | 79 | # cprotect 80 | *.cpt 81 | 82 | # elsarticle (documentclass of Elsevier journals) 83 | *.spl 84 | 85 | # endnotes 86 | *.ent 87 | 88 | # fixme 89 | *.lox 90 | 91 | # feynmf/feynmp 92 | *.mf 93 | *.mp 94 | *.t[1-9] 95 | *.t[1-9][0-9] 96 | *.tfm 97 | 98 | #(r)(e)ledmac/(r)(e)ledpar 99 | *.end 100 | *.?end 101 | *.[1-9] 102 | *.[1-9][0-9] 103 | *.[1-9][0-9][0-9] 104 | *.[1-9]R 105 | *.[1-9][0-9]R 106 | *.[1-9][0-9][0-9]R 107 | *.eledsec[1-9] 108 | *.eledsec[1-9]R 109 | *.eledsec[1-9][0-9] 110 | *.eledsec[1-9][0-9]R 111 | *.eledsec[1-9][0-9][0-9] 112 | *.eledsec[1-9][0-9][0-9]R 113 | 114 | # glossaries 115 | *.acn 116 | *.acr 117 | *.glg 118 | *.glo 119 | *.gls 120 | *.glsdefs 121 | *.lzo 122 | *.lzs 123 | 124 | # uncomment this for glossaries-extra (will ignore makeindex's style files!) 125 | # *.ist 126 | 127 | # gnuplottex 128 | *-gnuplottex-* 129 | 130 | # gregoriotex 131 | *.gaux 132 | *.gtex 133 | 134 | # htlatex 135 | *.4ct 136 | *.4tc 137 | *.idv 138 | *.lg 139 | *.trc 140 | *.xref 141 | 142 | # hyperref 143 | *.brf 144 | 145 | # knitr 146 | *-concordance.tex 147 | # TODO Comment the next line if you want to keep your tikz graphics files 148 | *.tikz 149 | *-tikzDictionary 150 | 151 | # listings 152 | *.lol 153 | 154 | # luatexja-ruby 155 | *.ltjruby 156 | 157 | # makeidx 158 | *.idx 159 | *.ilg 160 | *.ind 161 | 162 | # minitoc 163 | *.maf 164 | *.mlf 165 | *.mlt 166 | *.mtc[0-9]* 167 | *.slf[0-9]* 168 | *.slt[0-9]* 169 | *.stc[0-9]* 170 | 171 | # minted 172 | _minted* 173 | *.pyg 174 | 175 | # morewrites 176 | *.mw 177 | 178 | # nomencl 179 | *.nlg 180 | *.nlo 181 | *.nls 182 | 183 | # pax 184 | *.pax 185 | 186 | # pdfpcnotes 187 | *.pdfpc 188 | 189 | # sagetex 190 | *.sagetex.sage 191 | *.sagetex.py 192 | *.sagetex.scmd 193 | 194 | # scrwfile 195 | *.wrt 196 | 197 | # sympy 198 | *.sout 199 | *.sympy 200 | sympy-plots-for-*.tex/ 201 | 202 | # pdfcomment 203 | *.upa 204 | *.upb 205 | 206 | # pythontex 207 | *.pytxcode 208 | pythontex-files-*/ 209 | 210 | # tcolorbox 211 | *.listing 212 | 213 | # thmtools 214 | *.loe 215 | 216 | # TikZ & PGF 217 | *.dpth 218 | *.md5 219 | *.auxlock 220 | 221 | # todonotes 222 | *.tdo 223 | 224 | # vhistory 225 | *.hst 226 | *.ver 227 | 228 | # easy-todo 229 | *.lod 230 | 231 | # xcolor 232 | *.xcp 233 | 234 | # xmpincl 235 | *.xmpi 236 | 237 | # xindy 238 | *.xdy 239 | 240 | # xypic precompiled matrices and outlines 241 | *.xyc 242 | *.xyd 243 | 244 | # endfloat 245 | *.ttt 246 | *.fff 247 | 248 | # Latexian 249 | TSWLatexianTemp* 250 | 251 | ## Editors: 252 | # WinEdt 253 | *.bak 254 | *.sav 255 | 256 | # Texpad 257 | .texpadtmp 258 | 259 | # LyX 260 | *.lyx~ 261 | 262 | # Kile 263 | *.backup 264 | 265 | # gummi 266 | .*.swp 267 | 268 | # KBibTeX 269 | *~[0-9]* 270 | 271 | # auto folder when using emacs and auctex 272 | ./auto/* 273 | *.el 274 | 275 | # expex forward references with \gathertags 276 | *-tags.tex 277 | 278 | # standalone packages 279 | *.sta 280 | 281 | # Makeindex log files 282 | *.lpz 283 | 284 | # Byte-compiled / optimized / DLL files 285 | __pycache__/ 286 | *.py[cod] 287 | *$py.class 288 | 289 | # C extensions 290 | *.so 291 | 292 | # Distribution / packaging 293 | .Python 294 | build/ 295 | develop-eggs/ 296 | dist/ 297 | downloads/ 298 | eggs/ 299 | .eggs/ 300 | lib/ 301 | lib64/ 302 | parts/ 303 | sdist/ 304 | var/ 305 | wheels/ 306 | pip-wheel-metadata/ 307 | share/python-wheels/ 308 | *.egg-info/ 309 | .installed.cfg 310 | *.egg 311 | MANIFEST 312 | 313 | # PyInstaller 314 | # Usually these files are written by a python script from a template 315 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 316 | *.manifest 317 | *.spec 318 | 319 | # Installer logs 320 | pip-log.txt 321 | pip-delete-this-directory.txt 322 | 323 | # Unit test / coverage reports 324 | htmlcov/ 325 | .tox/ 326 | .nox/ 327 | .coverage 328 | .coverage.* 329 | .cache 330 | nosetests.xml 331 | coverage.xml 332 | *.cover 333 | *.py,cover 334 | .hypothesis/ 335 | .pytest_cache/ 336 | 337 | # Translations 338 | *.mo 339 | *.pot 340 | 341 | # Django stuff: 342 | *.log 343 | local_settings.py 344 | db.sqlite3 345 | db.sqlite3-journal 346 | 347 | # Flask stuff: 348 | instance/ 349 | .webassets-cache 350 | 351 | # Scrapy stuff: 352 | .scrapy 353 | 354 | # Sphinx documentation 355 | docs/_build/ 356 | 357 | # PyBuilder 358 | target/ 359 | 360 | # Jupyter Notebook 361 | .ipynb_checkpoints 362 | 363 | # IPython 364 | profile_default/ 365 | ipython_config.py 366 | 367 | # pyenv 368 | .python-version 369 | 370 | # pipenv 371 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 372 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 373 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 374 | # install all needed dependencies. 375 | #Pipfile.lock 376 | 377 | # celery beat schedule file 378 | celerybeat-schedule 379 | 380 | # SageMath parsed files 381 | *.sage.py 382 | 383 | # Environments 384 | .env 385 | .venv 386 | env/ 387 | venv/ 388 | ENV/ 389 | env.bak/ 390 | venv.bak/ 391 | 392 | # Spyder project settings 393 | .spyderproject 394 | .spyproject 395 | 396 | # Rope project settings 397 | .ropeproject 398 | 399 | # mkdocs documentation 400 | /site 401 | 402 | # mypy 403 | .mypy_cache/ 404 | .dmypy.json 405 | dmypy.json 406 | 407 | # Pyre type checker 408 | .pyre/ 409 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # BSD 3-Clause Clear License 2 | 3 | SPDX-License-Identifier: BSD-3-Clause-Clear 4 | 5 | Copyright (c) 2019, The Numerical Algorithms Group, Ltd. 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without modification, are permitted 9 | (subject to the limitations in the disclaimer below) provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | 15 | * Redistributions in binary form must reproduce the above copyright 16 | notice, this list of conditions and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | 19 | * Neither the name of the Numerical Algorithms Group nor the 20 | names of its contributors may be used to endorse or promote products 21 | derived from this software without specific prior written permission. 22 | 23 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS 24 | SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 25 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 26 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 28 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 29 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 31 | THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include versionate.py 2 | include pypop/version 3 | include pypop/utils/pop_logo.npy 4 | recursive-include pypop/examples * 5 | recursive-include pypop/cfgs * 6 | recursive-include pypop/cutters * 7 | recursive-include pypop/filters * 8 | global-exclude */.ipynb_checkpoints/* 9 | global-exclude */*.bincache 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyPOP 2 | 3 | Documentation: https://numericalalgorithmsgroup.github.io/pypop/doc.html 4 | 5 | A python package for calculating POP metrics from application profiles, primarily designed for 6 | literate programming using Jupyter notebooks. 7 | 8 | PyPOP currently consumes Extrae `*.prv` traces, but is designed with a view to adding support for 9 | additional formats. 10 | 11 | ## Requirements 12 | 13 | * [Extrae] (for trace capture) 14 | * [Paraver/Paramedir] (for trace analysis) 15 | * [Dimemas] *optional* (for ideal network analysis) 16 | * [Numpy] 17 | * [Pandas] 18 | 19 | [Extrae]: https://tools.bsc.es/extrae 20 | [Paraver/Paramedir]: https://tools.bsc.es/paraver#batchprocessing 21 | [Dimemas]: https://tools.bsc.es/dimemas 22 | [Numpy]: https://numpy.org/ 23 | [Pandas]: https://pandas.pydata.org/ 24 | 25 | 26 | ``Paramedir`` and ``Dimemas`` must be available on the system PATH (Linux ``$PATH`` or Windows 27 | ``%PATH%`` variables) so that they can be found by PyPOP. 28 | 29 | ## Installation 30 | 31 | PyPOP itself can be installed using the `pip` package manager: 32 | 33 | ```bash 34 | $ pip install [--user] NAG-PyPOP 35 | ``` 36 | 37 | The optional `--user` directive instructs pip to install to the users home directory instead of the 38 | system site package directory. 39 | 40 | **N.B: Some linux distributions are still shipping Python2 alongside Python3. PyPOP requires 41 | Python3 and so you may need to install using the `pip3` command rather than the `pip` command.** 42 | 43 | Note that the [Extrae] tool is needed to collect application traces and the [Paraver/Paramedir] and 44 | [Dimemas] tools must be installed for trace Analysis. 45 | 46 | ## Usage 47 | 48 | Jupyter notebooks are intended to be the primary interface to PyPOP. This guide uses several 49 | example notebooks to demonstrate the core functionality of PyPOP for calculation of the POP Metrics 50 | as well as advanced analysis of trace files. 51 | 52 | PyPOP comes with example notebooks. These are located in the examples directory, which can be 53 | found using the `pypop.examples` module: 54 | 55 | ```bash 56 | $ python -m pypop.examples 57 | /home/phil/repos/pypop/pypop/examples 58 | ``` 59 | 60 | Copy these to directory where you have read permissions, e.g. 61 | 62 | ```bash 63 | $ cp -vr $(python -m pypop.examples) $HOME/pypop_examples 64 | ``` 65 | 66 | These notebooks demonstrate usage of the main elements of the package. 67 | 68 | See the [quickstart] guide and [API documentation] for more detail on usage. 69 | 70 | [quickstart]: https://numericalalgorithmsgroup.github.io/pypop/quickstart.html 71 | [API documentation]: https://numericalalgorithmsgroup.github.io/pypop/api/pypop.html 72 | 73 | Copyright (c) 2019, Numerical Algorithms Group Ltd. 74 | -------------------------------------------------------------------------------- /beamer/beamercolorthemepop.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Till Tantau 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | % 8 | % See the file doc/licenses/LICENSE for more details. 9 | 10 | 11 | %logo 12 | \def\poplogo{pop_logo} 13 | \def\eulogo{eu_logo} 14 | 15 | %colors 16 | \definecolor{popblue}{RGB}{6,6,132} 17 | \definecolor{popgreen}{RGB}{85,138,17} 18 | \definecolor{popnavy}{RGB}{37,58,96} 19 | 20 | \definecolor{popgrey}{HTML}{A7A9AC} 21 | \definecolor{popgreylight}{HTML}{DBDCDC} 22 | \definecolor{popgreydark}{RGB}{61, 63, 66} 23 | 24 | \mode 25 | 26 | \setbeamercolor*{title}{fg=white,bg=popblue} 27 | 28 | \setbeamercolor*{normal text}{fg=black,bg=white} 29 | \setbeamercolor*{alerted text}{fg=popgreen} 30 | \setbeamercolor*{example text}{fg=black} 31 | \setbeamercolor*{structure}{fg=popnavy} 32 | 33 | \providecommand*{\beamer@bftext@only}{% 34 | \relax 35 | \ifmmode 36 | \expandafter\beamer@bftext@warning 37 | \else 38 | \expandafter\bfseries 39 | \fi 40 | } 41 | \providecommand*{\beamer@bftext@warning}{% 42 | \ClassWarning{beamer} 43 | {Cannot use bold for alerted text in math mode}% 44 | } 45 | 46 | \setbeamerfont{alerted text}{series=\beamer@bftext@only} 47 | 48 | \setbeamercolor{palette primary}{fg=black,bg=white} 49 | \setbeamercolor{palette secondary}{fg=popgreydark,bg=white} 50 | \setbeamercolor{palette tertiary}{fg=black,bg=white} 51 | \setbeamercolor{palette quaternary}{fg=black,bg=white} 52 | 53 | \setbeamercolor{sidebar}{bg=black!20} 54 | 55 | \setbeamercolor{palette sidebar primary}{fg=black} 56 | \setbeamercolor{palette sidebar secondary}{fg=black} 57 | \setbeamercolor{palette sidebar tertiary}{fg=black} 58 | \setbeamercolor{palette sidebar quaternary}{fg=black} 59 | 60 | \setbeamercolor{item projected}{fg=black,bg=black!20} 61 | 62 | \setbeamercolor*{block body}{} 63 | \setbeamercolor*{block body alerted}{} 64 | \setbeamercolor*{block body example}{} 65 | \setbeamercolor*{block title}{parent=structure} 66 | \setbeamercolor*{block title alerted}{parent=alerted text} 67 | \setbeamercolor*{block title example}{parent=example text} 68 | 69 | \setbeamercolor*{titlelike}{parent=structure} 70 | 71 | \setbeamercolor{title note}{fg=popgreydark} 72 | 73 | \mode 74 | 75 | -------------------------------------------------------------------------------- /beamer/beamerfontthemepop.sty: -------------------------------------------------------------------------------- 1 | \mode 2 | 3 | \setbeamerfont{title}{size=\LARGE} 4 | \setbeamerfont{author}{size=\large} 5 | \setbeamerfont{title note}{size=\scriptsize} 6 | 7 | \mode 8 | 9 | -------------------------------------------------------------------------------- /beamer/beamerinnerthemepop.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Till Tantau 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | % 8 | % See the file doc/licenses/LICENSE for more details. 9 | 10 | 11 | \mode 12 | 13 | %\setbeamertemplate{background}{% 14 | % \ifnum\thepage>1\relax% No teeny headerline on title page 15 | % \begin{tikzpicture} 16 | % \useasboundingbox (0,0) rectangle(\the\paperwidth,\the\paperheight); 17 | % \coordinate[anchor=north east] (logopos) at ($(current page.north east) - (0.2, 0.10)$); 18 | % \node[anchor=north east] (poplogo) at (logopos) {\includegraphics[height=0.8cm]{\poplogo}}; 19 | % \coordinate (barleft) at ($(current page.north west) - (-0.3, 1.25)$); 20 | % \coordinate (barright) at ($(current page.north east) - (0.3, 1.35)$); 21 | % \shade[left color=popblue, right color=popgreen] (barleft) rectangle (barright); 22 | % \end{tikzpicture} 23 | % \fi 24 | %} 25 | 26 | \defbeamertemplate*{title page}{pop} 27 | {% 28 | \begingroup 29 | \usebeamerfont{title note} 30 | \usebeamercolor[fg]{title note} 31 | \begin{tikzpicture}[remember picture, overlay] 32 | \coordinate (reportnumpos) at (current page.north west); 33 | \node[anchor=north west] (reportnum) at (reportnumpos) {\thereportnumber}; 34 | \end{tikzpicture} 35 | \endgroup 36 | \vbox{} 37 | \vfill 38 | \centering 39 | \includegraphics[width=0.6\textwidth]{pop_logo} 40 | \vskip1em% 41 | \begingroup 42 | \begin{beamercolorbox}[wd=\paperwidth, ht=4.0em, dp=0.5em, center]{title} 43 | \vfill% 44 | \usebeamerfont{title}\inserttitle 45 | \vskip0.2em% 46 | \usebeamerfont{author}\insertauthor~---~\insertinstitute% 47 | \vfill% 48 | \end{beamercolorbox} 49 | \endgroup% 50 | \vskip2cm 51 | \centering 52 | \usebeamerfont{title note} 53 | \usebeamercolor[fg]{title note} 54 | \begin{tikzpicture}[remember picture, overlay] 55 | \coordinate (logopos) at ($(current page.south) + (0.0, 0.0)$); 56 | \coordinate (lstrappos) at ($(current page.south west) + (0.1, 0.1)$); 57 | \coordinate (rstrappos) at ($(current page.south east) + (-0.1, 0.1)$); 58 | \node[anchor=south] (eulogo) at (logopos) {\includegraphics[height=1.8cm]{\eulogo}}; 59 | \node[anchor=south west] (coe) at (lstrappos) {EU H2020 Centre of Excellence}; 60 | \node[anchor=south east] (grantagreement) at (rstrappos) {Grant Agreement No 824080}; 61 | \end{tikzpicture} 62 | } 63 | 64 | \defbeamertemplate*{itemize/enumerate body begin}{pop} 65 | { 66 | \let\this@beamer@cramped\beamer@cramped 67 | \renewcommand{\beamer@cramped}{ 68 | \this@beamer@cramped 69 | \addtolength{\itemsep}{.5\baselineskip} 70 | } 71 | } 72 | 73 | \defbeamertemplate*{itemize/enumerate body end}{pop} 74 | { 75 | \let\beamer@cramped\this@beamer@cramped 76 | } 77 | 78 | \mode 79 | 80 | -------------------------------------------------------------------------------- /beamer/beamerouterthemepop.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2007 by Till Tantau 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | % 8 | % See the file doc/licenses/LICENSE for more details. 9 | 10 | \mode 11 | 12 | \setbeamercolor*{author in head/foot}{parent=palette secondary} 13 | \setbeamercolor*{title in head/foot}{parent=palette secondary} 14 | \setbeamercolor*{date in head/foot}{parent=palette secondary} 15 | 16 | \setbeamercolor*{section in head/foot}{parent=palette tertiary} 17 | \setbeamercolor*{subsection in head/foot}{parent=palette primary} 18 | 19 | \defbeamertemplate*{frametitle}{pop}[1][left] 20 | { 21 | \vskip0.15cm% 22 | \usebeamerfont{frametitle}% 23 | \strut\insertframetitle\strut\par% 24 | \vskip0.4cm% 25 | } 26 | 27 | \defbeamertemplate*{footline}{pop} 28 | {% 29 | \ifnum\thepage>1\relax% 30 | \begin{tikzpicture}[remember picture, overlay] 31 | \useasboundingbox (0,0) rectangle(\the\paperwidth,\the\paperheight); 32 | \coordinate[anchor=north east] (logopos) at ($(current page.north east) - (0.2, 0.10)$); 33 | \node[anchor=north east] (poplogo) at (logopos) {\includegraphics[height=0.8cm]{\poplogo}}; 34 | \coordinate (barleft) at ($(current page.north west) - (-0.0, 1.25)$); 35 | \coordinate (barright) at ($(current page.north east) - (0.0, 1.35)$); 36 | \shade[left color=popblue, right color=popgreen] (barleft) rectangle (barright); \coordinate (leftpos) at ($(current page.south west) + (0.05, 0.02)$); 37 | \coordinate (rightpos) at ($(current page.south east) + (-0.05, 0.02)$); 38 | \coordinate (centerpos) at ($(current page.south) + (0.0, 0.02)$); 39 | \node[anchor=south west, text=popgreydark] (author) at (leftpos) {\insertshortauthor~\insertshortinstitute}; 40 | \node[anchor=south, text=popgreydark] (title) at (centerpos) {\insertshorttitle}; 41 | \node[anchor=south east, text=popgreydark] (pagenum) at (rightpos) {\insertdate\ \ \ \insertframenumber/\inserttotalframenumber}; 42 | \end{tikzpicture}% 43 | \fi% 44 | } 45 | 46 | \setbeamersize{text margin left=1em,text margin right=1em} 47 | 48 | \mode 49 | 50 | -------------------------------------------------------------------------------- /beamer/beamerthemepop.sty: -------------------------------------------------------------------------------- 1 | % Copyright 2004 by Madhusudan Singh 2 | % 3 | % This file may be distributed and/or modified 4 | % 5 | % 1. under the LaTeX Project Public License and/or 6 | % 2. under the GNU Public License. 7 | % 8 | % See the file doc/licenses/LICENSE for more details. 9 | 10 | \RequirePackage{helvet} 11 | \RequirePackage{tikz} 12 | \usetikzlibrary{calc, positioning} 13 | 14 | \mode 15 | 16 | \def\thereportnumber{ } 17 | \newcommand{\reportnumber}[1]{\def\thereportnumber{#1}} 18 | \def\thecodename{} 19 | \newcommand{\codename}[1]{\def\thecodename{#1}} 20 | 21 | 22 | \useinnertheme{pop} 23 | \useoutertheme{pop} 24 | \usecolortheme{pop} 25 | \usefonttheme{pop} 26 | 27 | %disable nav symbols 28 | \setbeamertemplate{navigation symbols}{} 29 | 30 | \mode 31 | 32 | -------------------------------------------------------------------------------- /beamer/eu_logo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/beamer/eu_logo.pdf -------------------------------------------------------------------------------- /beamer/test.tex: -------------------------------------------------------------------------------- 1 | \documentclass[aspectratio=169]{beamer} 2 | \usepackage[utf8]{inputenc} 3 | \usepackage[T1]{fontenc} 4 | 5 | \usetheme{pop} 6 | 7 | \reportnumber{POP-XXX} 8 | \codename{} 9 | \title[\thereportnumber~---~\thecodename]{\thecodename Performance Analysis} 10 | \date[1970/01/01]{1 January 1970} 11 | \author[Watney (NASA)]{Mark Watney --- NASA} 12 | 13 | 14 | \begin{document} 15 | 16 | \begin{frame} 17 | \titlepage 18 | \end{frame} 19 | 20 | 21 | \begin{frame} 22 | \frametitle{There Is No Largest Prime Number} 23 | \framesubtitle{The proof uses \textit{reductio ad absurdum}.} 24 | \begin{theorem} 25 | There is no largest prime number. \end{theorem} 26 | \begin{enumerate} 27 | \item<1-| alert@1> Suppose $p$ were the largest prime number. 28 | \item<2-> Let $q$ be the product of the first $p$ numbers. 29 | \item<3-> Then $q+1$ is not divisible by any of them. 30 | \item<1-> But $q + 1$ is greater than $1$, thus divisible by some prime 31 | number not in the first $p$ numbers. 32 | \end{enumerate} 33 | \end{frame} 34 | 35 | \begin{frame}{A longer title} 36 | \begin{itemize} 37 | \item one 38 | \item two 39 | \end{itemize} 40 | \end{frame} 41 | 42 | \end{document} 43 | -------------------------------------------------------------------------------- /bin/chop_and_analyze.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 4 | analysis_dir=$(readlink -f ${script_dir}/../basicanalysis) 5 | 6 | for prvfile in "$@"; do 7 | ${script_dir}/chop_roi.sh ${prvfile} 8 | done 9 | 10 | cp -r ${analysis_dir}/modelfactors.py ${analysis_dir}/cfgs . 11 | 12 | ./modelfactors.py */roi*.prv 13 | 14 | 15 | -------------------------------------------------------------------------------- /bin/chop_roi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script assumes the paramedir executable (found in the wxparaver/bin directory) 4 | # is in your PATH environment variable. 5 | 6 | # Pass in all prv files as arguments. 7 | # If no files are provided, exit 8 | if [ $# -eq 0 ] 9 | then 10 | echo "No arguments supplied" 11 | exit 1 12 | fi 13 | 14 | # Names for XML files the scrip generates. These don't need to be changed as the files 15 | # are created by this script (i.e. they are not prerequisite separate files). 16 | filter_file=filter.xml 17 | cutter_file=cutter.xml 18 | 19 | # Temporary file to hold just the events from a prv file. 20 | tmp_events=events.txt 21 | 22 | # Directory that will be created to hold the chopped traces 23 | ROI_dir=ROI_chops 24 | 25 | # For every input prv file... 26 | for f in "$@" 27 | do 28 | ( cd $(dirname $f) 29 | f=$(basename $f) 30 | # Create the XML filter file that paramedir will use to extract just the 31 | # 40000012 events from the trace files. These are the ones that occur 32 | # when tracing is enabled/disabled via the API. The filter file is the same 33 | # for all input prv files. 34 | printf "\n" > $filter_file 35 | printf "\n \n 1\n" >> $filter_file 36 | printf " 0\n" >> $filter_file 37 | printf " 1\n" >> $filter_file 38 | printf " Running\n" >> $filter_file 39 | printf " 0\n" >> $filter_file 40 | printf " \n" >> $filter_file 41 | printf " \n 40000012\n" >> $filter_file 42 | printf " \n" >> $filter_file 43 | printf " 0\n" >> $filter_file 44 | printf " \n\n" >> $filter_file 45 | 46 | # Check if the prv file is gzipped, if it is unzip so we can access the raw numbers and 47 | # change the filevariable to use the new name by removing .gz 48 | if [[ $f == *".gz" ]]; then 49 | echo "found gzip file" 50 | gunzip $f 51 | tmp=${f%.gz} 52 | f=$tmp 53 | echo "unzipped filename is: " $f 54 | fi 55 | 56 | # Extract the number of ranks/threads in the prv file. This is found in the 4th column 57 | # of the 2nd line (the columns are separated by ":") 58 | ncores=`head -2 $f | tail -1 | awk -F: '{print $4}'` 59 | echo $f $ncores 60 | 61 | # Call paramedir with the filter file to generate a new trace prv that contains only the 62 | # events that correspond to enabling/disabling tracing via the API, and then strip the header 63 | # line from this prv and stash the actual events in a temp file. The new trace prv is then 64 | # deleted as we don't need it anymore. 65 | paramedir --filter $f $filter_file 66 | grep -v 'c' *.filter1.prv | grep -v '#' > $tmp_events 67 | rm *.filter1.* 68 | 69 | # The bit of the trace we're interested in starts after tracing is first disabled. Because 70 | # tmp_events only contains enable/disable tracing events, the first ncores event will be 71 | # each rank/thread emitting a "tracing disabled" event. Therefore the time we're going to 72 | # start the cut at is immediately after the last one of these events (the "+1" in the print 73 | # statement adds 1ns to the last time). 74 | # 75 | # The logic for extracting this assumes each event is on a separate line and that the lines 76 | # are ordered by increasing time, and that enable/disable tracing events are not 77 | # intermingled (i.e. the trace has the disable events, all the enable events and finally 78 | # all the disable events). 79 | start_time=`head -${ncores} $tmp_events | tail -1 | awk -F: '{print $6+1}'` 80 | 81 | 82 | # The bit of the trace we're interested in ends after tracing is disabled again. As with the 83 | # start time we can find this by counting lines in the trace file. There will be ncores lines 84 | # of initial "tracing disabled" events, a further ncores lines of "tracing enabled" events 85 | # (when the RoI starts) and then a further ncores lines of "tracing disabled" events (at the 86 | # end of the RoI). We therefore take the end time for cutting as just after the last "tracing 87 | # disabled" event in the second group of such events. 88 | # 89 | # Again the logic for extracting this assumes each event is on a separate line, that the 90 | # lines are ordered by increasing time, and that enable/disable tracing events are not 91 | # intermingled (i.e. the trace has the disable events, all the enable events and finally 92 | # all the disable events). 93 | endline=$(( $ncores *3 )) 94 | end_time=`head -${endline} $tmp_events | tail -1 | awk -F: '{print $6+1}'` 95 | 96 | echo -e "ncores\tstart\tstop\tendline" 97 | echo -e "$ncores\t$start_time\t$end_time\t$endline" 98 | 99 | # Create the XML cutter file to be used by paramedir to actually extract the RoI from 100 | # the full trace. This uses the start and end times identified from the filtered file. 101 | # We need to remove the first state and the last state because of the times we've chosen: 102 | # both times are in the "tracing disabled" states that surrounds the RoI. 103 | printf "\n" > $cutter_file 104 | printf "\n \n" >> $cutter_file 105 | printf " 0\n" >> $cutter_file 106 | printf " 1\n " >> $cutter_file 107 | printf " %d\n" $start_time >> $cutter_file 108 | printf " %d\n" $end_time >> $cutter_file 109 | printf " 0\n" >> $cutter_file 110 | printf " 100\n" >> $cutter_file 111 | printf " 0\n" >> $cutter_file 112 | printf " 0\n" >> $cutter_file 113 | printf " 1\n" >> $cutter_file 114 | printf " 1\n" >> $cutter_file 115 | printf " 1\n" >> $cutter_file 116 | printf " \n\n" >> $cutter_file 117 | 118 | # Call paramedir with the cutter file to produce the RoI trace for this number of cores. 119 | outfile=roi_$f 120 | paramedir -c $f $cutter_file -o $outfile 121 | 122 | # Delete the temporary file containing just the tracing enabled/disabled events. 123 | rm $tmp_events 124 | 125 | # Finally delete the filter file and cutter file. 126 | rm $cutter_file $filter_file 127 | ) 128 | done 129 | -------------------------------------------------------------------------------- /bin/extraempirun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Default values for certain parameters, can be overridden by environment 4 | 5 | : ${EMP_EXTRAE_DIR:=AUTO} # Extrae location (if AUTO, mpi2prv must be on $PATH) 6 | : ${EMP_MERGE_MAX_MEM:=512} # maximum memory (per process) for merge 7 | : ${EMP_MPI_LAUNCHER:=mpirun} # mpi launcher executable 8 | : ${EMP_MPI2PRV_OPTS:=} # Additional merge options to be passed to mpi2prv 9 | 10 | function usage { 11 | cat </dev/null) 2>/dev/null) 2>/dev/null) 41 | fi 42 | # Either way should now be able to find Extrae files 43 | if [ ! -e "$extrae_dir/etc/extrae.sh" ]; then 44 | echo -e "\nERROR: Cannot locate Extrae installation. Ensure mpi2prv etc. are in your PATH"\ 45 | "or set EMP_EXTRAE_DIR\n" 1>&2 46 | exit -1 47 | fi 48 | extrae_version=$(${extrae_dir}/bin/extrae-cmd --version 2>/dev/null | grep -oE "[0-9.]*") 49 | echo "Found Extrae version ${extrae_version} in ${extrae_dir}" 50 | 51 | # Check that the xml skeleton exists 52 | if [ ! -f "$exml_skel" ]; then 53 | echo -e "\nERROR: Configuration file ${exml_skel} not found.\n" 1>&2 54 | exit -1 55 | fi 56 | 57 | # Set the config path 58 | export EXTRAE_CONFIG_FILE=${exml_skel} 59 | 60 | # Finally create the tracing wrapper on the fly 61 | cat > extraewrapper.sh </dev/null 2>&1 && pwd )" 4 | template_dir=$(readlink -f ${script_dir}/../ipython_templates) 5 | 6 | report_nb=$1 7 | output_filename="$(echo ${report_nb} | sed 's/ /_/g' | sed 's/.ipynb$//').tex" 8 | 9 | export TEXINPUTS="${template_dir}//:" 10 | 11 | export PYPOP_HEADLESS="TRUE" 12 | 13 | jupyter nbconvert "${report_nb}" \ 14 | --execute \ 15 | --to latex \ 16 | --template latex \ 17 | --template-file ${template_dir}/pop_report.tplx \ 18 | --output=${output_filename} 19 | 20 | if [ $? -ne 0 ]; then 21 | echo Error: nbconvert failed - see documentation or PyPOP github for assistance. 22 | exit -1 23 | fi 24 | 25 | texi2pdf ${output_filename} &>/dev/null 26 | 27 | if [ $? -ne 0 ]; then 28 | echo Error: latex compilation failed. See ${output_filename/.tex/.log} for details 29 | exit -1 30 | fi 31 | 32 | echo ${output_filename/.tex/.pdf} written successfully. 33 | -------------------------------------------------------------------------------- /bin/modelfactors2precomputed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pandas 4 | 5 | import sys 6 | 7 | 8 | def usage(): 9 | print("Usage: {} ".format(sys.argv[0])) 10 | 11 | 12 | def main(): 13 | if len(sys.argv) > 2: 14 | usage() 15 | return -1 16 | 17 | inputfile = "modelfactors.csv" 18 | if len(sys.argv) == 2: 19 | inputfile = sys.argv[1] 20 | 21 | try: 22 | inputdata = pandas.read_csv( 23 | inputfile, sep=";", index_col=0, comment="#" 24 | ).transpose() 25 | except FileNotFoundError: 26 | print("FATAL: {} not found".format(inputfile)) 27 | return -1 28 | 29 | inputdata.columns.name = "Metrics" 30 | 31 | mapper = {m: m.title() for m in inputdata.columns} 32 | replacements = { 33 | "Ipc": "IPC", 34 | "Scalability": "Scaling", 35 | "Load Balance": "MPI Load Balance", 36 | "Communication Efficiency": "MPI Communication Efficiency", 37 | "Transfer Efficiency": "MPI Transfer Efficiency", 38 | "Serialization Efficiency": "MPI Serialisation Efficiency", 39 | } 40 | for rin, rout in replacements.items(): 41 | mapper = {k: v.replace(rin, rout) for k, v in mapper.items()} 42 | 43 | inputdata.rename(columns=mapper, inplace=True) 44 | 45 | for column in inputdata.columns: 46 | if column in ["Speedup", "Average IPC", "Average Frequency (Ghz)"]: 47 | continue 48 | inputdata[column] = inputdata[column] / 100 49 | 50 | inputdata["Number of Processes"] = inputdata.index 51 | inputdata["Threads per Process"] = 1 52 | inputdata["Total Threads"] = inputdata.index 53 | inputdata["Hybrid Layout"] = ["{}x1".format(i) for i in inputdata.index] 54 | 55 | inputdata.to_csv("precomputed.csv") 56 | 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = PyPOP 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /doc/source/_static/css/kbd.css: -------------------------------------------------------------------------------- 1 | kbd 2 | { 3 | -moz-border-radius:3px; 4 | -moz-box-shadow:0 1px 0 rgba(0,0,0,0.2),0 0 0 2px #fff inset; 5 | -webkit-border-radius:3px; 6 | -webkit-box-shadow:0 1px 0 rgba(0,0,0,0.2),0 0 0 2px #fff inset; 7 | background-color:#f7f7f7; 8 | border:1px solid #ccc; 9 | border-radius:3px; 10 | box-shadow:0 1px 0 rgba(0,0,0,0.2),0 0 0 2px #fff inset; 11 | color:#333; 12 | display:inline-block; 13 | font-family:Arial,Helvetica,sans-serif; 14 | font-size:11px; 15 | line-height:1.4; 16 | margin:0 .1em; 17 | padding:.1em .6em; 18 | text-shadow:0 1px 0 #fff; 19 | } 20 | -------------------------------------------------------------------------------- /doc/source/api_hl.rst: -------------------------------------------------------------------------------- 1 | High Level API 2 | ============== 3 | 4 | The high level API contains all the routines necessary for calculating POP metrics via the Jupyter 5 | notebook interface. 6 | 7 | .. automodule:: pypop.traceset 8 | :members: 9 | :undoc-members: 10 | :inherited-members: 11 | 12 | 13 | .. automodule:: pypop.metrics 14 | :members: 15 | :undoc-members: 16 | :inherited-members: 17 | 18 | -------------------------------------------------------------------------------- /doc/source/api_ll.rst: -------------------------------------------------------------------------------- 1 | Low Level API 2 | ============= 3 | 4 | *Beware, this package is under heavy development and the low level API is subject to change 5 | without warning!* 6 | 7 | .. automodule:: pypop.dimemas 8 | :members: 9 | :undoc-members: 10 | :inherited-members: 11 | 12 | .. automodule:: pypop.extrae 13 | :members: 14 | :undoc-members: 15 | :inherited-members: 16 | 17 | .. automodule:: pypop.prv 18 | :members: 19 | :undoc-members: 20 | :inherited-members: 21 | 22 | .. automodule:: pypop.utils 23 | :members: 24 | :undoc-members: 25 | :inherited-members: 26 | 27 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | from matplotlib import use 16 | use('Agg') 17 | import os 18 | import sys 19 | sys.path.insert(0, os.path.abspath('../../pypop/')) 20 | 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = 'PyPOP' 25 | copyright = '2019, The Numerical Algorithms Group, Ltd.' 26 | author = 'Phil Tooley (NAG)' 27 | 28 | # The short X.Y version 29 | version = '' 30 | # The full version, including alpha/beta/rc tags 31 | release = '0.1' 32 | 33 | # -- General configuration --------------------------------------------------- 34 | 35 | # If your documentation needs a minimal Sphinx version, state it here. 36 | # 37 | # needs_sphinx = '1.0' 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be 40 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 41 | # ones. 42 | extensions = [ 43 | 'sphinx.ext.autodoc', 44 | 'sphinx.ext.napoleon', 45 | 'sphinx.ext.doctest', 46 | 'sphinx.ext.intersphinx', 47 | 'sphinx.ext.autosummary', 48 | 'sphinx.ext.todo', 49 | 'sphinx.ext.coverage', 50 | 'sphinx.ext.mathjax', 51 | 'sphinx.ext.ifconfig', 52 | 'sphinx.ext.viewcode', 53 | 'sphinx.ext.githubpages', 54 | 'sphinxcontrib.programoutput', 55 | 'sphinx.ext.auosectionlabel', 56 | ] 57 | 58 | # Add any paths that contain templates here, relative to this directory. 59 | templates_path = ['_templates'] 60 | 61 | # The suffix(es) of source filenames. 62 | # You can specify multiple suffix as a list of string: 63 | # 64 | # source_suffix = ['.rst', '.md'] 65 | source_suffix = '.rst' 66 | 67 | # The master toctree document. 68 | master_doc = 'doc' 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | # 73 | # This is also used if you do content translation via gettext catalogs. 74 | # Usually you set "language" from the command line for these cases. 75 | language = None 76 | 77 | # List of patterns, relative to source directory, that match files and 78 | # directories to ignore when looking for source files. 79 | # This pattern also affects html_static_path and html_extra_path . 80 | exclude_patterns = [] 81 | 82 | # The name of the Pygments (syntax highlighting) style to use. 83 | pygments_style = 'sphinx' 84 | 85 | 86 | # -- Options for HTML output ------------------------------------------------- 87 | 88 | # The theme to use for HTML and HTML Help pages. See the documentation for 89 | # a list of builtin themes. 90 | # 91 | html_theme = 'sphinx_rtd_theme' 92 | 93 | # Theme options are theme-specific and customize the look and feel of a theme 94 | # further. For a list of options available for each theme, see the 95 | # documentation. 96 | # 97 | # html_theme_options = {} 98 | 99 | # Add any paths that contain custom static files (such as style sheets) here, 100 | # relative to this directory. They are copied after the builtin static files, 101 | # so a file named "default.css" will overwrite the builtin "default.css". 102 | html_static_path = ['_static'] 103 | 104 | html_css_files = ['css/kbd.css'] 105 | 106 | # Custom sidebar templates, must be a dictionary that maps document names 107 | # to template names. 108 | # 109 | # The default sidebars (for documents that don't match any pattern) are 110 | # defined by theme itself. Builtin themes are using these templates by 111 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 112 | # 'searchbox.html']``. 113 | # 114 | # html_sidebars = {} 115 | 116 | 117 | # -- Options for HTMLHelp output --------------------------------------------- 118 | 119 | # Output file base name for HTML help builder. 120 | htmlhelp_basename = 'PyPOPdoc' 121 | 122 | 123 | # -- Options for LaTeX output ------------------------------------------------ 124 | 125 | latex_elements = { 126 | # The paper size ('letterpaper' or 'a4paper'). 127 | # 128 | # 'papersize': 'letterpaper', 129 | 130 | # The font size ('10pt', '11pt' or '12pt'). 131 | # 132 | # 'pointsize': '10pt', 133 | 134 | # Additional stuff for the LaTeX preamble. 135 | # 136 | # 'preamble': '', 137 | 138 | # Latex figure (float) alignment 139 | # 140 | # 'figure_align': 'htbp', 141 | } 142 | 143 | # Grouping the document tree into LaTeX files. List of tuples 144 | # (source start file, target name, title, 145 | # author, documentclass [howto, manual, or own class]). 146 | latex_documents = [ 147 | (master_doc, 'PyPOP.tex', 'PyPOP Documentation', 148 | 'Phil Tooley (NAG)', 'manual'), 149 | ] 150 | 151 | 152 | # -- Options for manual page output ------------------------------------------ 153 | 154 | # One entry per manual page. List of tuples 155 | # (source start file, name, description, authors, manual section). 156 | man_pages = [ 157 | (master_doc, 'pypop', 'PyPOP Documentation', 158 | [author], 1) 159 | ] 160 | 161 | 162 | # -- Options for Texinfo output ---------------------------------------------- 163 | 164 | # Grouping the document tree into Texinfo files. List of tuples 165 | # (source start file, target name, title, author, 166 | # dir menu entry, description, category) 167 | texinfo_documents = [ 168 | (master_doc, 'PyPOP', 'PyPOP Documentation', 169 | author, 'PyPOP', 'One line description of project.', 170 | 'Miscellaneous'), 171 | ] 172 | 173 | 174 | # -- Extension configuration ------------------------------------------------- 175 | 176 | autoclass_content = 'both' 177 | 178 | autosectionlabel_prefix_document = True 179 | 180 | # -- Options for intersphinx extension --------------------------------------- 181 | 182 | # Example configuration for intersphinx: refer to the Python standard library. 183 | intersphinx_mapping = {'https://docs.python.org/': None} 184 | 185 | # -- Options for todo extension ---------------------------------------------- 186 | 187 | # If true, `todo` and `todoList` produce output, else they produce nothing. 188 | todo_include_todos = True 189 | -------------------------------------------------------------------------------- /doc/source/doc.rst: -------------------------------------------------------------------------------- 1 | Welcome to PyPOP's documentation! 2 | ================================= 3 | 4 | The PyPOP package is designed to make it easy to perform application performance analyses based on 5 | the POP methodology. The primary goals of PyPOP are: 6 | 7 | * Easy calculation of POP metrics 8 | * High quality figure generation 9 | * Easy access to underlying data and statistics (using Pandas) 10 | * Flexible and extensible design 11 | 12 | .. toctree:: 13 | :hidden: 14 | :maxdepth: 2 15 | :caption: Contents: 16 | 17 | install.rst 18 | quickstart.rst 19 | layout.rst 20 | api_hl.rst 21 | api_ll.rst 22 | -------------------------------------------------------------------------------- /doc/source/install.rst: -------------------------------------------------------------------------------- 1 | Installing PyPOP 2 | ================ 3 | 4 | Prerequisites 5 | ------------- 6 | 7 | The major prerequisite for PyPOP is that the Paramedir_ and Dimemas_ tools are installed and 8 | available on the path. 9 | 10 | These can be obtained from the `BSC tools project`_ along with instructions for their installation. 11 | 12 | .. _Paramedir: https://tools.bsc.es/paraver#batchprocessing 13 | .. _Dimemas: https://tools.bsc.es/dimemas 14 | .. _BSC tools project: https://tools.bsc.es 15 | 16 | These must be available on the system PATH (Linux ``$PATH`` or Windows ``%PATH%`` variables) so 17 | that they can be found by PyPOP. 18 | 19 | Installation 20 | ------------ 21 | 22 | PyPOP is designed to be installed using pip_ the Python package manager. 23 | 24 | .. _pip: https://pypi.org/project/pip/ 25 | 26 | This will install both pypop and any required dependencies which are not currently installed. 27 | 28 | To install from the github repository do one of the following: 29 | 30 | Directly install using pip 31 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | 33 | Instruct pip to install by downloading directly: 34 | 35 | .. code-block:: bash 36 | 37 | $ pip install [--user] git+https://github.com/numericalalgorithmsgroup/pypop 38 | 39 | The optional ``--user`` directive instructs pip to install to the users home directory instead of 40 | the system site package directory. 41 | 42 | 43 | Clone then install 44 | ^^^^^^^^^^^^^^^^^^ 45 | 46 | Alternatively, the repository can first be cloned, then the package installed: 47 | 48 | .. code-block:: bash 49 | 50 | $ git clone https://github.com/numericalalgorithmsgroup/pypop 51 | $ cd pypop 52 | $ pip install [--user] . 53 | -------------------------------------------------------------------------------- /doc/source/layout.rst: -------------------------------------------------------------------------------- 1 | PyPOP Package Layout 2 | ==================== 3 | 4 | The PyPOP package is laid out with two API levels. The first provides a high-level 5 | inteface to load summary statistics from traces and calculate various POP metrics from 6 | them. This is backed by a low level interface which provides more direct access to the 7 | capabilites of the various tools, allowing scripting of custom analyses. 8 | 9 | High Level Interface 10 | -------------------- 11 | 12 | The key components of the PyPOP package are the :py:mod:`~pypop.metrics` and 13 | :py:mod:`~pypop.traceset` modules. These contain the classes handling the loading and analysis of 14 | traces, and calculation of metrics. 15 | 16 | .. automodule:: pypop.traceset 17 | 18 | .. automodule:: pypop.metrics 19 | 20 | 21 | Low Level Interface 22 | ------------------- 23 | 24 | .. automodule:: pypop.extrae 25 | 26 | .. automodule:: pypop.dimemas 27 | 28 | .. automodule:: pypop.prv 29 | 30 | .. automodule:: pypop.utils 31 | 32 | 33 | -------------------------------------------------------------------------------- /doc/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart Guide 2 | ================ 3 | 4 | Jupyter notebooks are intended to be the primary interface to PyPOP. This guide uses several 5 | example notebooks to demonstrate the core functionality of PyPOP for calculation of the POP Metrics 6 | as well as advanced analysis of trace files. 7 | 8 | Requirements 9 | ------------ 10 | 11 | This guide assumes that you have successfully :doc:`installed PyPOP` and are able 12 | to open `Jupyter (IPython) notebooks`_. 13 | 14 | .. _Jupyter (IPython) notebooks: https://jupyter.org/install 15 | 16 | In addition, you will need a copy of the examples. These are located in the examples directory, 17 | which can be found using the `pypop.examples` module: 18 | 19 | .. command-output:: python -m pypop.examples 20 | 21 | Copy these to directory where you have read permissions, e.g. 22 | 23 | .. code-block:: bash 24 | 25 | $ cp -vr $(python -m pypop.examples) $HOME/pypop_examples 26 | 27 | 28 | Analysis using IPython Notebooks 29 | -------------------------------- 30 | 31 | Example notebooks are provided for the different analysis types 32 | 33 | Running the MPI Metrics Example Notebook 34 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 35 | 36 | To open the notebook you can either start a `Jupyter session`_, and navigate to the 37 | examples directory you created, or open the notebook directly from the console: 38 | 39 | .. _Jupyter session: https://jupyter.readthedocs.io/en/latest/running.html 40 | 41 | .. code-block:: bash 42 | 43 | $ cd $HOME/pypop_examples 44 | $ jupyter notebook MPI\ Metrics\ Example.ipynb 45 | 46 | This demonstrates basic, in-notebook use of the PyPOP tool for generating the POP MPI metrics, and 47 | shows how the underlying data can be accessed as Pandas Dataframes. The example notebook is 48 | self-documenting, and code cells can be run directly in the browser using :kbd:`shift` + 49 | :kbd:`enter`. 50 | 51 | Running the Hybrid Metrics Example Notebook 52 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 53 | 54 | There is also a hybrid metrics example notebook, which 55 | 56 | .. code-block:: bash 57 | 58 | $ cd $HOME/pypop_examples 59 | $ jupyter notebook Hybrid\ Metrics\ Example.ipynb 60 | 61 | This is a slightly more detailed example, showing how the code can be used to perform a more in 62 | depth analysis that is supported by the command line version of the tool. 63 | 64 | Command Line Scripts 65 | -------------------- 66 | 67 | To quickly generate strong scaling metrics for MPI and MPI/OpenMP hybrid codes on the command line, 68 | some convenience scripts are provided. 69 | 70 | MPI Strong Scaling Metrics 71 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 72 | 73 | The `pypop-mpi-metrics` script takes a list of trace files and calculates the POP MPI strong 74 | scalings metrics. By default output is provided in the form of a csv file containing the metrics, 75 | along with images with the metric table and a speedup scaling plot. 76 | 77 | .. command-output:: pypop-mpi-metrics -h 78 | 79 | 80 | MPI/OpenMP Hybrid Strong Scaling Metrics 81 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 82 | 83 | The `pypop-hybrid-metrics` script takes a list of trace files and calculates prototype hybrid 84 | MPI+OpenMP strong scalings metrics. By default output is provided in the form of a csv file 85 | containing the metrics, along with images with the metric table and a speedup scaling plot. 86 | 87 | .. command-output:: pypop-hybrid-metrics -h 88 | 89 | 90 | -------------------------------------------------------------------------------- /ipython_templates/graphics/EU-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/ipython_templates/graphics/EU-logo.jpg -------------------------------------------------------------------------------- /ipython_templates/graphics/pop-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/ipython_templates/graphics/pop-logo.png -------------------------------------------------------------------------------- /ipython_templates/html_nocode.tpl: -------------------------------------------------------------------------------- 1 | {%- extends 'full.tpl' -%} 2 | 3 | {% block input_group %} 4 | {% endblock input_group %} 5 | -------------------------------------------------------------------------------- /ipython_templates/pop_report.cls: -------------------------------------------------------------------------------- 1 | \LoadClass[11pt, a4paper, twoside]{article} 2 | % , parskip=half* 3 | \NeedsTeXFormat{LaTeX2e} 4 | \ProvidesClass{pop_report}[2015/11/11 POP Project class] 5 | 6 | \usepackage{hyperref} 7 | \usepackage{xcolor} 8 | \hypersetup{ 9 | colorlinks, 10 | linkcolor={black!50!black}, 11 | citecolor={blue!50!black}, 12 | urlcolor={blue!80!black} 13 | } 14 | 15 | \RequirePackage{color} 16 | \RequirePackage{colortbl} 17 | \RequirePackage{fancyhdr} 18 | \RequirePackage{fullpage} 19 | \RequirePackage{graphicx} 20 | \RequirePackage{tabularx} 21 | \RequirePackage[small]{titlesec} 22 | \RequirePackage{xspace} 23 | \RequirePackage{hyphenat} 24 | \RequirePackage{lmodern} 25 | \RequirePackage[english]{babel} 26 | 27 | \graphicspath{{./graphics}} 28 | 29 | % Title commands 30 | \newcommand{\reprefnum}[1]{\def\@reprefnum{#1}} 31 | \newcommand{\application}[1]{\def\@application{#1}} 32 | \newcommand{\servicelevel}[1]{\def\@servicelevel{#1}} 33 | \newcommand{\contributors}[1]{\def\@contributors{#1}} 34 | \newcommand{\keywords}[1]{\def\@keywords{#1}} 35 | \title{POP Ref.No.~\@reprefnum} 36 | 37 | \renewcommand\maketitle{ 38 | \thispagestyle{empty} 39 | \begin{center} 40 | \begin{figure}[h] 41 | \centering 42 | \hspace{1cm}\Oldincludegraphics[width=12cm]{pop-logo} 43 | \end{figure} 44 | \end{center} 45 | 46 | \vspace{1cm} 47 | 48 | \begin{center} 49 | \Large\bf\color[rgb]{0.44, 0.71, 0.04} 50 | \textsf{\@title} 51 | \end{center} 52 | 53 | \vspace{1.5cm} 54 | 55 | \begin{flushleft} 56 | \Large\bf\color[rgb]{0.44, 0.71, 0.04} 57 | \textsf{Document Information} 58 | \end{flushleft} 59 | \vspace{-2em} 60 | \begin{center} 61 | \begin{tabular}{|l|p{0.7\textwidth}|} 62 | \hline 63 | \color[rgb]{0.14, 0.25, 0.38}Reference Number & \@reprefnum \\ \hline 64 | \color[rgb]{0.14, 0.25, 0.38}Author & \@author \\ \hline 65 | \color[rgb]{0.14, 0.25, 0.38}Contributor(s) & \@contributors \\ \hline 66 | \color[rgb]{0.14, 0.25, 0.38}Date & \@date \\ \hline 67 | \end{tabular} 68 | \end{center} 69 | 70 | \vfill 71 | 72 | {\footnotesize 73 | \em 74 | \textbf{Notices:} 75 | The research leading to these results has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No. 676553. 76 | } 77 | \\ 78 | \\ 79 | {\begin{tabular}{p{2cm} p{12cm}} 80 | \vspace{0pt} 81 | \Oldincludegraphics[width=2cm]{EU-logo} 82 | & \footnotesize 83 | \vspace{0pt} 84 | \copyright 2015 POP Consortium Partners. All rights reserved.\\ 85 | \end{tabular} 86 | } 87 | \clearpage 88 | } 89 | 90 | \newcommand{\frontmatter}{ 91 | \vspace*{-0.9cm} 92 | \tableofcontents 93 | \clearpage 94 | } 95 | 96 | 97 | \newenvironment{changelog}{ 98 | \begin{flushleft} 99 | \LARGE\bf\color[rgb]{0.14, 0.25, 0.38} 100 | \vspace*{0.5cm} 101 | \textsf{Change Log} 102 | \end{flushleft} 103 | 104 | \begin{center} 105 | \begin{tabular}{|l|l|l|} 106 | \hline 107 | \rule[-0.3cm]{0mm}{0.8cm} \centering \bf \textsf{Version} & \centering \bf \textsf{Author} & \centering \bf \textsf{Description of Change} \tabularnewline 108 | } { 109 | \hline 110 | \end{tabular} 111 | \end{center} 112 | } 113 | 114 | \newcommand{\change}[3]{ 115 | \hline 116 | \rule[-0.3cm]{0mm}{0.8cm} \centering v#1 & #2 & #3 \\ 117 | } 118 | 119 | \titleformat{\section}[hang]{\huge\bfseries\sffamily\color[rgb]{0.14, 0.25, 0.38}}{\thesection}{1em}{} 120 | \titleformat{\subsection}[hang]{\Large\bfseries}{\thesubsection}{1em}{} 121 | 122 | \newcommand{\fixme}[1]{\textcolor{red}{\textit{FIXME $->$ #1 $<-$}}} 123 | \newcommand{\todo}[1]{\textcolor{red}{\textbf{TODO: #1}}} 124 | \newcommand{\eg}{e.g.,\xspace} 125 | \newcommand{\ie}{i.e.,\xspace} 126 | \newcommand{\etc}{etc.\@\xspace} 127 | \newcommand{\etal}{et.\ al.\xspace} 128 | \newcommand{\vs}{vs.\xspace} 129 | \newcommand{\IO}{I\slash O\xspace} 130 | 131 | \newcommand{\slogo}{\Oldincludegraphics[width=2cm]{pop-logo}} 132 | 133 | \AtBeginDocument{ 134 | \headheight=30pt 135 | \pagestyle{fancy} 136 | \fancyhf{} 137 | \headsep 2ex 138 | \footskip 4ex 139 | \rhead[\slogo]{\slogo} 140 | \chead[\rightmark]{\rightmark} 141 | \lhead[ {\scriptsize \@title}]{{\scriptsize \@title}} 142 | \cfoot{\thepage} 143 | \renewcommand{\sectionmark}[1]{\markboth{#1}{}} 144 | \renewcommand{\subsectionmark}[1]{\markright{#1}{}} 145 | } 146 | -------------------------------------------------------------------------------- /ipython_templates/pop_report.tplx: -------------------------------------------------------------------------------- 1 | ((*- extends 'report.tex.j2' -*)) 2 | 3 | ## Set pop_report class 4 | ((* block docclass *)) 5 | \documentclass{pop_report} 6 | ((* endblock docclass *)) 7 | 8 | ## Don't override cls setting of title 9 | ((* block title *)) 10 | ((* endblock title *)) 11 | 12 | ((* block commands *)) 13 | ((( super() ))) 14 | \setlength{\parindent}{0em} 15 | \setlength{\parskip}{1em} 16 | 17 | \newlength{\maximagewidth} 18 | \setlength{\maximagewidth}{0.8\textwidth} 19 | \renewcommand{\includegraphics}[1]{\Oldincludegraphics[width=\maximagewidth]{#1}} 20 | \DeclareCaptionLabelFormat{minimal}{#2} 21 | \DeclareCaptionFormat{minimal}{\llap{\textbf{#1#2}}#3\par} 22 | \captionsetup{format=minimal, labelformat=minimal, labelsep=period, singlelinecheck=no, 23 | width=\maximagewidth} 24 | ((* endblock commands *)) 25 | 26 | ## Input necessary details from ipynb metadata 27 | ((* block maketitle *)) 28 | 29 | ((* for item in ['reprefnum', 'title', 'author', 'contributors'] *)) 30 | \((( item ))){((( nb.metadata.get("pop_metadata", {}).get(item, '\color{{red}} pop\_metadata.{} NOT SET'.format(item)) 31 | )))} 32 | ((* endfor *)) 33 | 34 | \date{\today} 35 | \maketitle 36 | ((* endblock maketitle *)) 37 | 38 | ## Set pdfs, pngs, etc. to be included as figures 39 | ((*- block data_pdf *)) 40 | ((( draw_figure_with_caption(output.metadata.filenames['application/pdf'], 41 | cell.metadata.get("caption", ""), 42 | cell.metadata.get("label", "")) ))) 43 | ((* endblock data_pdf *)) 44 | 45 | ((*- block data_png *)) 46 | ((( draw_figure_with_caption(output.metadata.filenames['image/png'], 47 | cell.metadata.get("caption", ""), 48 | cell.metadata.get("label", "")) ))) 49 | ((* endblock data_png *)) 50 | 51 | ((*- block data_svg *)) 52 | ((( draw_figure_with_caption(output.metadata.filenames['image/svg+xml'], 53 | cell.metadata.get("caption", ""), 54 | cell.metadata.get("label", "")) ))) 55 | ((* endblock data_svg *)) 56 | 57 | ((*- block data_html *)) 58 | ((* endblock data_html *)) 59 | 60 | ((*- block data_text *)) 61 | ((* endblock data_text *)) 62 | 63 | 64 | ## macro to encapsulate output files in figures 65 | ((* macro draw_figure_with_caption(filename, caption, label) -*)) 66 | ((* set filename = filename | posix_path *)) 67 | ((*- block figure scoped -*)) 68 | \begin{figure}[!hb] 69 | \begin{center}% 70 | \adjustimage{max size={0.9\linewidth}{0.4\paperheight}}{((( filename )))}% 71 | \end{center} 72 | \caption{((( caption )))} 73 | \label{((( label )))} 74 | \end{figure} 75 | ((*- endblock figure -*)) 76 | ((*- endmacro *)) 77 | 78 | ((* block input_group *)) 79 | ((* endblock input_group *)) 80 | 81 | ((* block output_group *)) 82 | ((*- if cell.metadata.hide_output: -*)) 83 | ((*- else -*)) 84 | ((( super() ))) 85 | ((*- endif *)) 86 | ((* endblock output_group *)) 87 | -------------------------------------------------------------------------------- /presentations/launch_blog/launch_blog.md: -------------------------------------------------------------------------------- 1 | # PyPOP - A new tool for efficient performance analysis 2 | 3 | We are proud to announce the launch of a new tool - PyPOP - designed to support the existing POP 4 | tools with high quality analysis, plotting and report generation functionality. This blog post 5 | will explore the reasons why we wrote a new tool and give an overview of the functionality. 6 | 7 | ## Why we wrote another POP Tool 8 | 9 | There are various profiling tools that are developed by partners in the POP project, including the 10 | [Extrae], [Scalasca] and [Maqao] tools that have been discussed in [previous] [blog] [posts]. 11 | These are extremely powerful tools that can provide huge amounts of detailed information about 12 | application behaviour, however, sometimes the sheer volume of information can be overwhelming both 13 | for the analyst trying to interpret it and also in terms of the size of the trace files being 14 | unable to fit in the memory of a laptop or workstation. 15 | 16 | [Extrae]: https://tools.bsc.es/extrae 17 | [Scalasca]: https://www.scalasca.org/ 18 | [Maqao]: http://www.maqao.org/ 19 | [previous]: https://pop-coe.eu/blog/pop-tool-descriptions-bsc-performance-tools 20 | [blog]: https://pop-coe.eu/blog/pop-tool-descriptions-jsc-performance-tools 21 | [posts]: https://pop-coe.eu/blog/pop-tool-descriptions-uvsq-performance-tools 22 | 23 | The [POP methodology] has been developed in response to the difficulties involved in doing 24 | performance analysis efficiently and effectively, and the [POP Metrics] are at the centre of this. 25 | The idea of the POP methodology is to break down the application performance into different areas 26 | such as parallel communication, use of hardware resources etc, each of which is reflected by one of 27 | the POP metrics. This high-level view informs the next stage of the analysis which can then target 28 | the specific areas with poor performance in an iterative fashion until the root cause(s) of any 29 | inefficiencies are identified and fixed. 30 | 31 | [POP Methodology]: https://pop-coe.eu/further-information/online-training/understanding-application-performance-with-the-pop-metrics 32 | [POP Metrics]: https://pop-coe.eu/node/69 33 | 34 | The development of PyPOP came about as a natural result of the POP Methodology: we needed a tool 35 | that allowed us to efficiently and repeatably analyze profiling results and compute the POP 36 | metrics. As the tool matured it gained additional features such as plotting capabilities, batch 37 | pre-processing of large traces on remote HPC nodes and user interface features to improve ease of 38 | use. 39 | 40 | ## PyPOP Features 41 | 42 | PyPOP has been designed to allow efficient, repeatable performance analysis. It re-uses the 43 | available features of the underlying profiling tools where possible, and is written in Python3 with 44 | the widely used Numpy and Pandas libraries to minimize barrier-to-entry for custom analysis. 45 | 46 | #### Automated calcuation of the POP Metrics 47 | 48 | Trace files are directly supplied as input. PyPOP then processes them to extract the relevant data 49 | and calculate the POP Metrics. As well the original POP MPI Metrics various other metric types 50 | in development by the project can be calculated. 51 | 52 | #### Jupyter notebook interface with interactive "Wizard" 53 | 54 | PyPOP uses a "literate programming" approach that can mix GUI elements, text, figures and Python 55 | code within the Jupyter "electronic notebook" environment. This gives great flexibility to use 56 | PyPOP in either a GUI-style "wizard mode" or to use the full power of the Python language to 57 | rapidly develop custom analyses on top of the PyPOP framework. 58 | 59 | #### High quality table and scaling plot generation 60 | 61 | PyPOP provides routines to plot both formatted metrics tables and scaling plots for parallel 62 | speedup and other metrics. The produced plots are interactive, with mouseover details and 63 | descriptions and can be saved as images for later use. 64 | 65 | #### PDF report generation 66 | 67 | Once the metrics have been calculated and the plots produced, PyPOP supports creation of PDF 68 | formatted reports directly from the Jupyter Notebook. 69 | 70 | 71 | #### Headless pre-analysis mode 72 | 73 | PyPOP also has a command-line interface which can be used to pre-process very large (10GB or more) 74 | traces on remote machines such as HPC nodes or cloud instances. This can be done as a batch job for 75 | example, and the small (less than 100kB) summary file downloaded to analyse locally using the PyPOP 76 | GUI interface. 77 | 78 | 79 | ## Getting Started with PyPOP 80 | 81 | To download a copy of PyPOP visit the [PyPOP Github repository]. The [Readme] and [Documentation] 82 | describe how to install PyPOP along with example analysis files to use with the tutorial. 83 | 84 | We also have an [Introduction to PyPOP] module part of the [POP Online Training Course]. 85 | 86 |

90 | 91 | [PyPOP Github repository]: https://github.com/numericalalgorithmsgroup/pypop/ 92 | [Readme]: https://github.com/numericalalgorithmsgroup/pypop/ 93 | [Documentation]: https://numericalalgorithmsgroup.github.io/pypop/doc.html 94 | 95 | [Introduction to PyPOP]: https://pop-coe.eu/further-information/online-training/computing-the-pop-metrics-with-pypop 96 | [POP Online Training Course]: https://pop-coe.eu/further-information/online-training 97 | -------------------------------------------------------------------------------- /pypop/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """Tools for POP assessments in Python scripts and IPython notebooks 6 | """ 7 | 8 | from . import traceset 9 | from . import metrics 10 | from . import prv 11 | from . import config 12 | 13 | from .version import get_version 14 | 15 | __all__ = ["__version__"] 16 | 17 | __version__ = get_version() 18 | -------------------------------------------------------------------------------- /pypop/_plotsettings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | pagedata = {"textwidth": 6} # width checked for \documentclass[a4paper]{book} 6 | 7 | figparams = { 8 | "single.figsize": (pagedata["textwidth"], pagedata["textwidth"] * 0.66), 9 | "single.width": 0.8, 10 | "single.height": 0.8, 11 | "single.leftpad": 0.1, 12 | "single.bottompad": 0.12, 13 | } 14 | 15 | figparams["single.axlayout"] = ( 16 | ( 17 | figparams["single.leftpad"], 18 | figparams["single.bottompad"], 19 | figparams["single.width"], 20 | figparams["single.height"], 21 | ), 22 | ) 23 | 24 | pypop_mpl_params = { 25 | "text.usetex": False, 26 | "text.latex.preamble": [r"\usepackage{color}", r"\usepackage{amssymb}"], 27 | "savefig.dpi": 300, 28 | "figure.dpi": 300, 29 | "font.size": 10, 30 | "xtick.labelsize": "medium", 31 | "ytick.labelsize": "medium", 32 | "lines.linewidth": 1, 33 | "legend.fontsize": 10, 34 | "legend.frameon": False, 35 | } 36 | -------------------------------------------------------------------------------- /pypop/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/backends/__init__.py -------------------------------------------------------------------------------- /pypop/backends/extrae/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/backends/extrae/__init__.py -------------------------------------------------------------------------------- /pypop/backends/extrae/pcf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 2 | 3 | """\ 4 | PCF File Parser 5 | --------------- 6 | 7 | Provides lightweight support for interrogating the contents of a PRV file, including all 8 | event types and id->string lookups for value types. 9 | """ 10 | 11 | from collections import namedtuple 12 | 13 | import pandas 14 | 15 | Event = namedtuple("Event", ["ID", "Name", "State", "Values"]) 16 | Value = namedtuple("Value", ["ID", "Name"]) 17 | State = namedtuple("State", ["ID", "Name"]) 18 | 19 | class PCF: 20 | 21 | def __init__(self, pcf_path): 22 | 23 | self._pcf_path = pcf_path 24 | 25 | self._state_names = {} 26 | self._event_names = {} 27 | self._event_states = {} 28 | self._event_vals = {} 29 | 30 | self._states = None 31 | self._events = None 32 | self._parse_pcf() 33 | 34 | @property 35 | def filename(self): 36 | return self._pcf_path 37 | 38 | @property 39 | def event_names(self): 40 | return self._event_names 41 | 42 | @property 43 | def event_values(self): 44 | return self._event_vals 45 | 46 | @property 47 | def events(self): 48 | return self._events 49 | 50 | 51 | def _parse_pcf(self): 52 | 53 | states = [] 54 | events = [] 55 | values = [] 56 | 57 | try: 58 | with open(self._pcf_path, "rt") as fh: 59 | block_mode = None 60 | for line in fh: 61 | if not line.strip(): 62 | continue 63 | if not line[0].isdigit(): 64 | block_mode = line.split()[0] 65 | continue 66 | 67 | if block_mode == "EVENT_TYPE": 68 | elem = line.strip().split(maxsplit=2) 69 | event_id = int(elem[1]) 70 | events.append(Event(event_id, elem[2], elem[0], {})) 71 | values[event_id] = [] 72 | continue 73 | 74 | if block_mode == "VALUES": 75 | elem = line.strip().split(maxsplit=1) 76 | value_id = int(elem[0]) 77 | values[event_id].append(Value(value_id, elem[1])) 78 | 79 | if block_mode == "STATES": 80 | elem = line.strip().split(maxsplit=1) 81 | state_id = int(elem[0]) 82 | states.append(State(state_id, elem[1])) 83 | 84 | except FileNotFoundError: 85 | pass 86 | 87 | self._events = pandas.DataFrame(events) 88 | self._states = pandas.DataFrame(states) 89 | self._event_values = {k:pandas.DataFrame(v) for k,v in values.items} 90 | 91 | 92 | -------------------------------------------------------------------------------- /pypop/cfgs/clock_frequency.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 2 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW Cycles per us > 8 | ################################################################################ 9 | window_name Cycles per us 10 | window_type single 11 | window_id 1 12 | window_position_x 384 13 | window_position_y 36 14 | window_width 600 15 | window_height 114 16 | window_comm_lines_enabled false 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_color_mode window_in_null_gradient_mode 20 | window_logical_filtered true 21 | window_physical_filtered false 22 | window_comm_fromto true 23 | window_comm_tagsize true 24 | window_comm_typeval true 25 | window_units Microseconds 26 | window_maximum_y 2200.000000000000 27 | window_minimum_y 0.000000000000 28 | window_compute_y_max false 29 | window_level thread 30 | window_scale_relative 1.000000000000 31 | window_end_time_relative 1.000000000000 32 | window_object appl { 1, { All } } 33 | window_begin_time_relative 0.000000000000 34 | window_open false 35 | window_drawmode draw_maximum 36 | window_drawmode_rows draw_maximum 37 | window_pixel_size 1 38 | window_labels_to_draw 0 39 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Avg Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } 40 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 41 | window_semantic_module thread Avg Next Evt Val { 1, { 1 1.000000000000 } } 42 | window_filter_module evt_type 1 42000059 43 | window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" 44 | 45 | ################################################################################ 46 | < NEW DISPLAYING WINDOW Interval btw. cycles events > 47 | ################################################################################ 48 | window_name Interval btw. cycles events 49 | window_type single 50 | window_id 2 51 | window_position_x 409 52 | window_position_y 88 53 | window_width 600 54 | window_height 114 55 | window_comm_lines_enabled false 56 | window_flags_enabled false 57 | window_noncolor_mode true 58 | window_color_mode window_in_null_gradient_mode 59 | window_logical_filtered true 60 | window_physical_filtered false 61 | window_comm_fromto true 62 | window_comm_tagsize true 63 | window_comm_typeval true 64 | window_units Microseconds 65 | window_maximum_y 2368757786.000000000000 66 | window_minimum_y 0.000000000000 67 | window_compute_y_max false 68 | window_level thread 69 | window_scale_relative 1.000000000000 70 | window_end_time_relative 1.000000000000 71 | window_object appl { 1, { All } } 72 | window_begin_time_relative 0.000000000000 73 | window_open false 74 | window_drawmode draw_maximum 75 | window_drawmode_rows draw_maximum 76 | window_pixel_size 1 77 | window_labels_to_draw 0 78 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } 79 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 80 | window_filter_module evt_type 1 42000059 81 | window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" 82 | 83 | < NEW ANALYZER2D > 84 | Analyzer2D.Name: New Histogram #1 85 | Analyzer2D.X: 1920 86 | Analyzer2D.Y: 25 87 | Analyzer2D.Width: 958 88 | Analyzer2D.Height: 967 89 | Analyzer2D.ControlWindow: 1 90 | Analyzer2D.DataWindow: 2 91 | Analyzer2D.Accumulator: Semantic 92 | Analyzer2D.Statistic: Sum bursts 93 | Analyzer2D.CalculateAll: True 94 | Analyzer2D.HideCols: False 95 | Analyzer2D.HorizVert: Horizontal 96 | Analyzer2D.Color: True 97 | Analyzer2D.SemanticColor: False 98 | Analyzer2D.Zoom: Enabled 99 | Analyzer2D.SortCols: False 100 | Analyzer2D.SortCriteria: Average 101 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 102 | Analyzer2D.AnalysisLimits: Alltrace 103 | Analyzer2D.ComputeYScale: False 104 | Analyzer2D.Minimum: @histlo@ 105 | Analyzer2D.Maximum: @histhi@ 106 | Analyzer2D.Delta: @binwidth@ 107 | Analyzer2D.ComputeGradient: True 108 | Analyzer2D.MinimumGradient: 0.000011738506 109 | Analyzer2D.MaximumGradient: 99.035516451109 110 | Analyzer2D.DrawModeObjects: draw_maximum 111 | Analyzer2D.DrawModeColumns: draw_maximum 112 | Analyzer2D.PixelSize: 1 113 | Analyzer2D.ColorMode: window_in_gradient_mode 114 | Analyzer2D.ShowOnlyTotals: False 115 | Analyzer2D.ShortHeaderLabels: True 116 | 117 | -------------------------------------------------------------------------------- /pypop/cfgs/ideal.collectives: -------------------------------------------------------------------------------- 1 | Machine globalop: 0 0 LOG MAX 0 MAX 2 | Machine globalop: 0 1 LOG MAX 0 MAX 3 | Machine globalop: 0 2 LOG MAX 0 MAX 4 | Machine globalop: 0 3 LOG MAX 0 MAX 5 | Machine globalop: 0 4 LOG MAX 0 MAX 6 | Machine globalop: 0 5 LOG MAX 0 MAX 7 | Machine globalop: 0 6 LOG MAX 0 MAX 8 | Machine globalop: 0 7 LOG MAX 0 MAX 9 | Machine globalop: 0 8 LIN 2MAX LIN 2MAX 10 | Machine globalop: 0 9 LOG MAX 0 MAX 11 | Machine globalop: 0 10 LOG 2MAX 0 MAX 12 | Machine globalop: 0 11 LOG MAX 0 MAX 13 | Machine globalop: 0 12 LOG MAX 0 MAX 14 | Machine globalop: 0 13 LOG MAX 0 MAX 15 | -------------------------------------------------------------------------------- /pypop/cfgs/omp_total_runtime.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW OMP Functions.c1.c1 > 8 | ################################################################################ 9 | window_name OMP Functions.c1.c1 10 | window_type single 11 | window_id 1 12 | window_position_x 3512 13 | window_position_y 96 14 | window_width 1718 15 | window_height 1394 16 | window_comm_lines_enabled false 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 18.000000000000 26 | window_minimum_y 0.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 0.751881260574 30 | window_end_time_relative 0.751881260574 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.741286622858 33 | window_open false 34 | window_drawmode draw_random 35 | window_drawmode_rows draw_random 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Stacked Val}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | window_filter_module evt_type 4 60000001 60000005 60000018 60000023 41 | window_filter_module evt_type_label 4 "Parallel (OMP)" "OpenMP barrier" "Executed OpenMP parallel function" "Executed OpenMP task function" 42 | 43 | < NEW ANALYZER2D > 44 | Analyzer2D.Name: OMP Functions 45 | Analyzer2D.X: 0 46 | Analyzer2D.Y: 22 47 | Analyzer2D.Width: 1718 48 | Analyzer2D.Height: 1333 49 | Analyzer2D.ControlWindow: 1 50 | Analyzer2D.DataWindow: 1 51 | Analyzer2D.Accumulator: Semantic 52 | Analyzer2D.Statistic: Time 53 | Analyzer2D.CalculateAll: True 54 | Analyzer2D.HideCols: False 55 | Analyzer2D.HorizVert: Horizontal 56 | Analyzer2D.Color: True 57 | Analyzer2D.SemanticColor: False 58 | Analyzer2D.Zoom: Disabled 59 | Analyzer2D.SortCols: False 60 | Analyzer2D.SortCriteria: Average 61 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 62 | Analyzer2D.AnalysisLimits: Alltrace 63 | Analyzer2D.ComputeYScale: False 64 | Analyzer2D.Minimum: 1.000000000000 65 | Analyzer2D.Maximum: 100000.000000000000 66 | Analyzer2D.Delta: 100000.000000000000 67 | Analyzer2D.ComputeGradient: True 68 | Analyzer2D.MinimumGradient: 22673805486.108154296875 69 | Analyzer2D.MaximumGradient: 22673805486.108154296875 70 | Analyzer2D.DrawModeObjects: draw_maximum 71 | Analyzer2D.DrawModeColumns: draw_maximum 72 | Analyzer2D.PixelSize: 1 73 | Analyzer2D.ColorMode: window_in_gradient_mode 74 | Analyzer2D.ShowOnlyTotals: False 75 | Analyzer2D.ShortHeaderLabels: True 76 | 77 | -------------------------------------------------------------------------------- /pypop/cfgs/omp_total_runtime_loop.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW OMP Functions.c1.c1 > 8 | ################################################################################ 9 | window_name OMP Functions.c1.c1 10 | window_type single 11 | window_id 1 12 | window_position_x 3512 13 | window_position_y 96 14 | window_width 1718 15 | window_height 1394 16 | window_comm_lines_enabled false 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 18.000000000000 26 | window_minimum_y 0.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 0.751881260574 30 | window_end_time_relative 0.751881260574 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.741286622858 33 | window_open false 34 | window_drawmode draw_random 35 | window_drawmode_rows draw_random 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Stacked Val}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | window_filter_module evt_type 3 60000001 60000005 60000018 41 | window_filter_module evt_type_label 3 "Parallel (OMP)" "OpenMP barrier" "Executed OpenMP parallel function" 42 | 43 | < NEW ANALYZER2D > 44 | Analyzer2D.Name: OMP Functions 45 | Analyzer2D.X: 0 46 | Analyzer2D.Y: 22 47 | Analyzer2D.Width: 1718 48 | Analyzer2D.Height: 1333 49 | Analyzer2D.ControlWindow: 1 50 | Analyzer2D.DataWindow: 1 51 | Analyzer2D.Accumulator: Semantic 52 | Analyzer2D.Statistic: Time 53 | Analyzer2D.CalculateAll: True 54 | Analyzer2D.HideCols: False 55 | Analyzer2D.HorizVert: Horizontal 56 | Analyzer2D.Color: True 57 | Analyzer2D.SemanticColor: False 58 | Analyzer2D.Zoom: Disabled 59 | Analyzer2D.SortCols: False 60 | Analyzer2D.SortCriteria: Average 61 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 62 | Analyzer2D.AnalysisLimits: Alltrace 63 | Analyzer2D.ComputeYScale: False 64 | Analyzer2D.Minimum: 1.000000000000 65 | Analyzer2D.Maximum: 100000.000000000000 66 | Analyzer2D.Delta: 100000.000000000000 67 | Analyzer2D.ComputeGradient: True 68 | Analyzer2D.MinimumGradient: 22673805486.108154296875 69 | Analyzer2D.MaximumGradient: 22673805486.108154296875 70 | Analyzer2D.DrawModeObjects: draw_maximum 71 | Analyzer2D.DrawModeColumns: draw_maximum 72 | Analyzer2D.PixelSize: 1 73 | Analyzer2D.ColorMode: window_in_gradient_mode 74 | Analyzer2D.ShowOnlyTotals: False 75 | Analyzer2D.ShortHeaderLabels: True 76 | 77 | -------------------------------------------------------------------------------- /pypop/cfgs/omp_total_runtime_task.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW OMP Functions.c1.c1 > 8 | ################################################################################ 9 | window_name OMP Functions.c1.c1 10 | window_type single 11 | window_id 1 12 | window_position_x 3512 13 | window_position_y 96 14 | window_width 1718 15 | window_height 1394 16 | window_comm_lines_enabled false 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 18.000000000000 26 | window_minimum_y 0.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 0.751881260574 30 | window_end_time_relative 0.751881260574 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.741286622858 33 | window_open false 34 | window_drawmode draw_random 35 | window_drawmode_rows draw_random 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Stacked Val}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | window_filter_module evt_type 3 60000001 60000005 60000023 41 | window_filter_module evt_type_label 3 "Parallel (OMP)" "OpenMP barrier" "Executed OpenMP task function" 42 | 43 | < NEW ANALYZER2D > 44 | Analyzer2D.Name: OMP Functions 45 | Analyzer2D.X: 0 46 | Analyzer2D.Y: 22 47 | Analyzer2D.Width: 1718 48 | Analyzer2D.Height: 1333 49 | Analyzer2D.ControlWindow: 1 50 | Analyzer2D.DataWindow: 1 51 | Analyzer2D.Accumulator: Semantic 52 | Analyzer2D.Statistic: Time 53 | Analyzer2D.CalculateAll: True 54 | Analyzer2D.HideCols: False 55 | Analyzer2D.HorizVert: Horizontal 56 | Analyzer2D.Color: True 57 | Analyzer2D.SemanticColor: False 58 | Analyzer2D.Zoom: Disabled 59 | Analyzer2D.SortCols: False 60 | Analyzer2D.SortCriteria: Average 61 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 62 | Analyzer2D.AnalysisLimits: Alltrace 63 | Analyzer2D.ComputeYScale: False 64 | Analyzer2D.Minimum: 1.000000000000 65 | Analyzer2D.Maximum: 100000.000000000000 66 | Analyzer2D.Delta: 100000.000000000000 67 | Analyzer2D.ComputeGradient: True 68 | Analyzer2D.MinimumGradient: 22673805486.108154296875 69 | Analyzer2D.MaximumGradient: 22673805486.108154296875 70 | Analyzer2D.DrawModeObjects: draw_maximum 71 | Analyzer2D.DrawModeColumns: draw_maximum 72 | Analyzer2D.PixelSize: 1 73 | Analyzer2D.ColorMode: window_in_gradient_mode 74 | Analyzer2D.ShowOnlyTotals: False 75 | Analyzer2D.ShortHeaderLabels: True 76 | 77 | -------------------------------------------------------------------------------- /pypop/cfgs/serial_useful_computation_no_omp.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW Useful > 8 | ################################################################################ 9 | window_name Useful 10 | window_type single 11 | window_id 1 12 | window_position_x 0 13 | window_position_y 25 14 | window_width 958 15 | window_height 1028 16 | window_comm_lines_enabled true 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 18.000000000000 26 | window_minimum_y 0.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 1.000000000000 30 | window_end_time_relative 1.000000000000 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.000000000000 33 | window_open false 34 | window_drawmode draw_maximum 35 | window_drawmode_rows draw_maximum 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | 41 | < NEW ANALYZER2D > 42 | Analyzer2D.Name: Serial Useful Computation 43 | Analyzer2D.X: 0 44 | Analyzer2D.Y: 25 45 | Analyzer2D.Width: 958 46 | Analyzer2D.Height: 970 47 | Analyzer2D.ControlWindow: 1 48 | Analyzer2D.DataWindow: 1 49 | Analyzer2D.Accumulator: Semantic 50 | Analyzer2D.Statistic: Time 51 | Analyzer2D.CalculateAll: True 52 | Analyzer2D.HideCols: False 53 | Analyzer2D.HorizVert: Horizontal 54 | Analyzer2D.Color: True 55 | Analyzer2D.SemanticColor: False 56 | Analyzer2D.Zoom: Disabled 57 | Analyzer2D.SortCols: False 58 | Analyzer2D.SortCriteria: Average 59 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 60 | Analyzer2D.AnalysisLimits: Alltrace 61 | Analyzer2D.ComputeYScale: True 62 | Analyzer2D.Minimum: 1.000000000000 63 | Analyzer2D.Maximum: 1.000000000000 64 | Analyzer2D.Delta: 1.000000000000 65 | Analyzer2D.ComputeGradient: True 66 | Analyzer2D.MinimumGradient: 29439118981.000000000000 67 | Analyzer2D.MaximumGradient: 46269120190.000000000000 68 | Analyzer2D.DrawModeObjects: draw_maximum 69 | Analyzer2D.DrawModeColumns: draw_maximum 70 | Analyzer2D.PixelSize: 1 71 | Analyzer2D.ColorMode: window_in_gradient_mode 72 | Analyzer2D.ShowOnlyTotals: False 73 | Analyzer2D.ShortHeaderLabels: True 74 | 75 | -------------------------------------------------------------------------------- /pypop/cfgs/total_res_stl_cycles.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW PAPI_RES_STL > 8 | ################################################################################ 9 | window_name PAPI_RES_STL 10 | window_type single 11 | window_id 1 12 | window_position_x 556 13 | window_position_y 540 14 | window_width 1362 15 | window_height 538 16 | window_comm_lines_enabled false 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 41745245515.000000000000 26 | window_minimum_y 12.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 1.000000000000 30 | window_end_time_relative 1.000000000000 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.000000000000 33 | window_open false 34 | window_drawmode draw_maximum 35 | window_drawmode_rows draw_maximum 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | window_filter_module evt_type 1 42001047 41 | window_filter_module evt_type_label 1 "RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason]" 42 | 43 | < NEW ANALYZER2D > 44 | Analyzer2D.Name: SUM(PAPI_RES_STALL) 45 | Analyzer2D.X: 0 46 | Analyzer2D.Y: 0 47 | Analyzer2D.Width: 554 48 | Analyzer2D.Height: 1020 49 | Analyzer2D.ControlWindow: 1 50 | Analyzer2D.DataWindow: 1 51 | Analyzer2D.Accumulator: Semantic 52 | Analyzer2D.Statistic: Sum bursts 53 | Analyzer2D.CalculateAll: True 54 | Analyzer2D.HideCols: False 55 | Analyzer2D.HorizVert: Horizontal 56 | Analyzer2D.Color: False 57 | Analyzer2D.SemanticColor: False 58 | Analyzer2D.Zoom: Disabled 59 | Analyzer2D.SortCols: False 60 | Analyzer2D.SortCriteria: Average 61 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 62 | Analyzer2D.AnalysisLimits: Alltrace 63 | Analyzer2D.ComputeYScale: False 64 | Analyzer2D.Minimum: 12.000000000000 65 | Analyzer2D.Maximum: 10000000000000000.000000000000 66 | Analyzer2D.Delta: 10000000000000000.000000000000 67 | Analyzer2D.ComputeGradient: True 68 | Analyzer2D.MinimumGradient: 36869339789.000000000000 69 | Analyzer2D.MaximumGradient: 60438719596.000000000000 70 | Analyzer2D.DrawModeObjects: draw_maximum 71 | Analyzer2D.DrawModeColumns: draw_maximum 72 | Analyzer2D.PixelSize: 1 73 | Analyzer2D.ColorMode: window_in_gradient_mode 74 | Analyzer2D.ShowOnlyTotals: False 75 | Analyzer2D.ShortHeaderLabels: True 76 | 77 | -------------------------------------------------------------------------------- /pypop/cfgs/total_runtime.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW All states > 8 | ################################################################################ 9 | window_name All states 10 | window_type single 11 | window_id 1 12 | window_position_x 0 13 | window_position_y 25 14 | window_width 958 15 | window_height 1028 16 | window_comm_lines_enabled true 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 18.000000000000 26 | window_minimum_y 0.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 1.000000000000 30 | window_end_time_relative 1.000000000000 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.000000000000 33 | window_open false 34 | window_drawmode draw_maximum 35 | window_drawmode_rows draw_maximum 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State As Is}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | 41 | < NEW ANALYZER2D > 42 | Analyzer2D.Name: Total Runtime 43 | Analyzer2D.X: 0 44 | Analyzer2D.Y: 25 45 | Analyzer2D.Width: 958 46 | Analyzer2D.Height: 970 47 | Analyzer2D.ControlWindow: 1 48 | Analyzer2D.DataWindow: 1 49 | Analyzer2D.Accumulator: Semantic 50 | Analyzer2D.Statistic: Time 51 | Analyzer2D.CalculateAll: True 52 | Analyzer2D.HideCols: False 53 | Analyzer2D.HorizVert: Horizontal 54 | Analyzer2D.Color: True 55 | Analyzer2D.SemanticColor: False 56 | Analyzer2D.Zoom: Disabled 57 | Analyzer2D.SortCols: False 58 | Analyzer2D.SortCriteria: Average 59 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 60 | Analyzer2D.AnalysisLimits: Alltrace 61 | Analyzer2D.ComputeYScale: False 62 | Analyzer2D.Minimum: 0.000000000000 63 | Analyzer2D.Maximum: 100000000000000000000000000000000000000000000000000000000000000.000000000000 64 | Analyzer2D.Delta: 1000000000000000000000000000000000000000000000000000000000000000.000000000000 65 | Analyzer2D.ComputeGradient: True 66 | Analyzer2D.MinimumGradient: 24807461577.000000000000 67 | Analyzer2D.MaximumGradient: 24808211870.000000000000 68 | Analyzer2D.DrawModeObjects: draw_maximum 69 | Analyzer2D.DrawModeColumns: draw_maximum 70 | Analyzer2D.PixelSize: 1 71 | Analyzer2D.ColorMode: window_in_gradient_mode 72 | Analyzer2D.ShowOnlyTotals: False 73 | Analyzer2D.ShortHeaderLabels: True 74 | 75 | -------------------------------------------------------------------------------- /pypop/cfgs/total_useful_computation.cfg: -------------------------------------------------------------------------------- 1 | #ParaverCFG 2 | ConfigFile.Version: 3.4 3 | ConfigFile.NumWindows: 1 4 | 5 | 6 | ################################################################################ 7 | < NEW DISPLAYING WINDOW Total Useful Computation > 8 | ################################################################################ 9 | window_name Total Useful Computation 10 | window_type single 11 | window_id 1 12 | window_position_x 0 13 | window_position_y 25 14 | window_width 958 15 | window_height 1028 16 | window_comm_lines_enabled false 17 | window_flags_enabled false 18 | window_noncolor_mode true 19 | window_logical_filtered true 20 | window_physical_filtered false 21 | window_comm_fromto true 22 | window_comm_tagsize true 23 | window_comm_typeval true 24 | window_units Nanoseconds 25 | window_maximum_y 18.000000000000 26 | window_minimum_y 0.000000000000 27 | window_compute_y_max false 28 | window_level thread 29 | window_scale_relative 1.000000000000 30 | window_end_time_relative 1.000000000000 31 | window_object appl { 1, { All } } 32 | window_begin_time_relative 0.000000000000 33 | window_open false 34 | window_drawmode draw_maximum 35 | window_drawmode_rows draw_maximum 36 | window_pixel_size 1 37 | window_labels_to_draw 1 38 | window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } 39 | window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } 40 | 41 | < NEW ANALYZER2D > 42 | Analyzer2D.Name: Total Useful Computation 43 | Analyzer2D.X: 0 44 | Analyzer2D.Y: 25 45 | Analyzer2D.Width: 958 46 | Analyzer2D.Height: 970 47 | Analyzer2D.ControlWindow: 1 48 | Analyzer2D.DataWindow: 1 49 | Analyzer2D.Accumulator: Semantic 50 | Analyzer2D.Statistic: Time 51 | Analyzer2D.CalculateAll: True 52 | Analyzer2D.HideCols: False 53 | Analyzer2D.HorizVert: Horizontal 54 | Analyzer2D.Color: True 55 | Analyzer2D.SemanticColor: False 56 | Analyzer2D.Zoom: Enabled 57 | Analyzer2D.SortCols: False 58 | Analyzer2D.SortCriteria: Average 59 | Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 60 | Analyzer2D.AnalysisLimits: Alltrace 61 | Analyzer2D.ComputeYScale: True 62 | Analyzer2D.Minimum: 1.000000000000 63 | Analyzer2D.Maximum: 1.000000000000 64 | Analyzer2D.Delta: 1.000000000000 65 | Analyzer2D.ComputeGradient: True 66 | Analyzer2D.MinimumGradient: 3457567262.000000000000 67 | Analyzer2D.MaximumGradient: 6634135832.000000000000 68 | Analyzer2D.DrawModeObjects: draw_maximum 69 | Analyzer2D.DrawModeColumns: draw_maximum 70 | Analyzer2D.PixelSize: 1 71 | Analyzer2D.ColorMode: window_in_gradient_mode 72 | Analyzer2D.ShowOnlyTotals: False 73 | Analyzer2D.ShortHeaderLabels: True 74 | 75 | -------------------------------------------------------------------------------- /pypop/cli/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | -------------------------------------------------------------------------------- /pypop/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """Configuration options for PyPOP 6 | """ 7 | 8 | from os.path import normpath, expanduser 9 | 10 | __all__ = ["set_paramedir_path", "set_dimemas_path", "set_tmpdir_path"] 11 | 12 | _dimemas_path = None 13 | _paramedir_path = None 14 | _tmpdir_path = None 15 | 16 | 17 | def set_dimemas_path(path): 18 | global _dimemas_path 19 | if not isinstance(path, str): 20 | raise ValueError("path must be a string") 21 | 22 | _dimemas_path = normpath(expanduser(path)) 23 | 24 | 25 | def set_paramedir_path(path): 26 | global _paramedir_path 27 | if not isinstance(path, str): 28 | raise ValueError("path must be a string") 29 | 30 | _paramedir_path = normpath(expanduser(path)) 31 | 32 | 33 | def set_tmpdir_path(path): 34 | global _tmpdir_path 35 | if not isinstance(path, str): 36 | raise ValueError("path must be a string") 37 | 38 | _tmpdir_path = normpath(expanduser(path)) 39 | -------------------------------------------------------------------------------- /pypop/cutters/single_region.skel: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 0 5 | 1 6 | @MIN_TIME@ 7 | @MAX_TIME@ 8 | 0 9 | 100 10 | 0 11 | 0 12 | 1 13 | 1 14 | 1 15 | 16 | 17 | -------------------------------------------------------------------------------- /pypop/examples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """Support module for locating the pypop example files 6 | """ 7 | 8 | from pkg_resources import resource_filename 9 | 10 | 11 | def examples_directory(): 12 | """Location of the examples directory 13 | 14 | Returns 15 | ------- 16 | examples_dir: str 17 | Path to examples directory 18 | """ 19 | return resource_filename(__name__, "examples") 20 | 21 | 22 | if __name__ == "__main__": 23 | print(examples_directory()) 24 | -------------------------------------------------------------------------------- /pypop/examples/html_examples/README.md: -------------------------------------------------------------------------------- 1 | # html examples 2 | 3 | These notebooks are designed to be converted to html to demonstrate interactive blog graphics. 4 | 5 | To convert ``notebook.ipynb`` to html using nbconvert: 6 | 7 | jupyter nbconvert --template=../../../ipython_templates/html_nocode.tpl --to=html notebook.ipynb 8 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_16proc.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/mpi/epoch_example_traces/epoch_16proc.prv.gz -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_16proc.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 16 2 | 01.s16r1b20 3 | 02.s16r1b20 4 | 03.s16r1b20 5 | 04.s16r1b20 6 | 05.s16r1b20 7 | 06.s16r1b20 8 | 07.s16r1b20 9 | 08.s16r1b20 10 | 09.s16r1b20 11 | 10.s16r1b20 12 | 11.s16r1b20 13 | 12.s16r1b20 14 | 13.s16r1b20 15 | 14.s16r1b20 16 | 15.s16r1b20 17 | 16.s16r1b20 18 | 19 | LEVEL NODE SIZE 1 20 | s16r1b20 21 | 22 | LEVEL THREAD SIZE 16 23 | THREAD 1.1.1 24 | THREAD 1.2.1 25 | THREAD 1.3.1 26 | THREAD 1.4.1 27 | THREAD 1.5.1 28 | THREAD 1.6.1 29 | THREAD 1.7.1 30 | THREAD 1.8.1 31 | THREAD 1.9.1 32 | THREAD 1.10.1 33 | THREAD 1.11.1 34 | THREAD 1.12.1 35 | THREAD 1.13.1 36 | THREAD 1.14.1 37 | THREAD 1.15.1 38 | THREAD 1.16.1 39 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_1proc.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/mpi/epoch_example_traces/epoch_1proc.prv.gz -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_1proc.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 1 2 | 1.s19r2b64 3 | 4 | LEVEL NODE SIZE 1 5 | s19r2b64 6 | 7 | LEVEL THREAD SIZE 1 8 | THREAD 1.1.1 9 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_2proc.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/mpi/epoch_example_traces/epoch_2proc.prv.gz -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_2proc.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 2 2 | 1.s01r2b02 3 | 2.s01r2b02 4 | 5 | LEVEL NODE SIZE 1 6 | s01r2b02 7 | 8 | LEVEL THREAD SIZE 2 9 | THREAD 1.1.1 10 | THREAD 1.2.1 11 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_4proc.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/mpi/epoch_example_traces/epoch_4proc.prv.gz -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_4proc.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 4 2 | 1.s04r2b68 3 | 2.s04r2b68 4 | 3.s04r2b68 5 | 4.s04r2b68 6 | 7 | LEVEL NODE SIZE 1 8 | s04r2b68 9 | 10 | LEVEL THREAD SIZE 4 11 | THREAD 1.1.1 12 | THREAD 1.2.1 13 | THREAD 1.3.1 14 | THREAD 1.4.1 15 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_8proc.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/mpi/epoch_example_traces/epoch_8proc.prv.gz -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/epoch_8proc.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 8 2 | 1.s05r2b24 3 | 2.s05r2b24 4 | 3.s05r2b24 5 | 4.s05r2b24 6 | 5.s05r2b24 7 | 6.s05r2b24 8 | 7.s05r2b24 9 | 8.s05r2b24 10 | 11 | LEVEL NODE SIZE 1 12 | s05r2b24 13 | 14 | LEVEL THREAD SIZE 8 15 | THREAD 1.1.1 16 | THREAD 1.2.1 17 | THREAD 1.3.1 18 | THREAD 1.4.1 19 | THREAD 1.5.1 20 | THREAD 1.6.1 21 | THREAD 1.7.1 22 | THREAD 1.8.1 23 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/input.deck: -------------------------------------------------------------------------------- 1 | begin:control 2 | nx = 1024 3 | ny = 1024 4 | 5 | # final time of simulation 6 | nsteps = 10 7 | 8 | # size of domain 9 | x_min = 0 10 | x_end = 1 11 | 12 | y_min = 0 13 | y_max = 1 14 | 15 | stdout_frequency = 10 16 | end:control 17 | 18 | 19 | begin:boundaries 20 | bc_x_min = simple_outflow 21 | bc_x_max = simple_outflow 22 | bc_y_min = periodic 23 | bc_y_max = periodic 24 | end:boundaries 25 | 26 | 27 | begin:window 28 | move_window = T 29 | window_start_time = 0 30 | bc_x_min_after_move = simple_outflow 31 | bc_x_max_after_move = simple_outflow 32 | window_v_x = 2e8 33 | end:window 34 | 35 | begin:constant 36 | x0 = 1.31 37 | y0 = 0.5 38 | r0 = 0.4^2 39 | r2 = (x - x0)^2 + (y - y0)^2 40 | end:constant 41 | 42 | begin:species 43 | # electron 44 | name = electron 45 | charge = -1.0 46 | mass = 1.0 47 | nparticles_per_cell = 5 48 | 49 | #number_density = if(r2 lt r0, 2*exp(-((time*5e8)^2)), 1) 50 | number_density = if(abs(x-x0) lt 0.3, 2, 1) 51 | number_density = if(abs(y-y0) lt 0.2, number_density(electron), 1) 52 | #number_density = if(abs(x-x0) lt 0.3, if(abs(y-y0) lt 0.2, 2, 1), 1) 53 | 54 | #temperature_ev = if(r2 lt r0, 2*exp(-((time*5e8)^2)), 1) 55 | temperature_ev = 0 56 | end:species 57 | 58 | 59 | begin:output 60 | name = normal 61 | disabled = T 62 | dt_snapshot = 1e-10 63 | 64 | # Properties at particle positions 65 | particles = always 66 | px = always 67 | py = always 68 | pz = always 69 | vx = always 70 | vy = always 71 | vz = always 72 | particle_weight = always 73 | 74 | # Properties on grid 75 | grid = always 76 | number_density = always + species 77 | temperature = always + species 78 | end:output 79 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/modelfactors.csv: -------------------------------------------------------------------------------- 1 | Number of processes;1;2;4;8;16 2 | Parallel efficiency;99.925583;99.378252;97.349953;95.707736;95.151184 3 | Load balance;100.000000;99.760930;98.687584;98.280683;99.248259 4 | Communication efficiency;99.925583;99.616405;98.644580;97.382042;95.871892 5 | Serialization efficiency;100.000000;99.842359;99.203493;98.282015;97.952863 6 | Transfer efficiency;99.925583;99.773690;99.436599;99.084296;97.875538 7 | Computation scalability;100.000000;95.500319;92.390414;81.340933;61.436394 8 | Global efficiency;99.925583;94.906548;89.942024;77.849566;58.457456 9 | IPC scalability;100.000000;99.916598;99.745711;93.982492;82.262861 10 | Instruction scalability;100.000000;96.748633;95.788940;94.145829;91.017467 11 | Frequency scalability;100.000000;98.792129;96.697961;91.930815;82.053509 12 | Speedup;1.000000;1.899545;3.600360;6.232603;9.360158 13 | Average IPC;1.871993;1.870432;1.867233;1.759346;1.539955 14 | Average frequency (GHz);2.069905;2.044904;2.001556;1.902881;1.698430 15 | # 16 | #Runtime (us);21898659.14;11528373.56;6082352.21;3513565.33;2339560.72 17 | #Runtime (ideal);21884758.97;11504370.09;6051059.08;3486349.43;2310335.96 18 | #Useful duration (average);21882362.82;11456696.18;5921167.02;3362753.84;2226119.73 19 | #Useful duration (maximum);21882362.82;11484151.34;5999910.80;3421581.67;2242981.13 20 | #Useful duration (total);21882362.82;22913392.37;23684668.07;26902030.73;35617915.65 21 | #Useful duration (ideal, max);21884758.97;11486234.50;6002862.00;3426454.46;2263040.22 22 | #Useful instructions (total);84790848422.00;87640358419.00;88518412143.00;90063308642.00;93158874793.00 23 | #Useful cycles (total);45294421893.00;46855679955.00;47406198069.00;51191361663.00;60494538119.00 24 | -------------------------------------------------------------------------------- /pypop/examples/mpi/epoch_example_traces/precomputed.csv: -------------------------------------------------------------------------------- 1 | ,Parallel Efficiency,MPI Load Balance,MPI Communication Efficiency,MPI Serialisation Efficiency,MPI Transfer Efficiency,Computation Scaling,Global Efficiency,IPC Scaling,Instruction Scaling,Frequency Scaling,Speedup,Average IPC,Average Frequency (Ghz),Number of Processes,Threads per Process,Total Threads,Hybrid Layout 2 | 1,0.9992558299999998,1.0,0.9992558299999998,1.0,0.9992558299999998,1.0,0.9992558299999998,1.0,1.0,1.0,1.0,1.871993,2.069905,1,1,1,1x1 3 | 2,0.99378252,0.9976093,0.99616405,0.99842359,0.9977369,0.9550031899999999,0.94906548,0.9991659799999999,0.96748633,0.9879212900000001,1.899545,1.8704319999999999,2.0449040000000003,2,1,2,2x1 4 | 4,0.97349953,0.9868758400000001,0.9864458,0.9920349300000001,0.99436599,0.9239041400000001,0.89942024,0.99745711,0.9578894,0.9669796099999999,3.60036,1.8672330000000001,2.0015560000000003,4,1,4,4x1 5 | 8,0.95707736,0.9828068299999999,0.97382042,0.98282015,0.9908429599999999,0.8134093299999999,0.7784956600000001,0.9398249200000001,0.94145829,0.91930815,6.232603,1.7593459999999999,1.902881,8,1,8,8x1 6 | 16,0.95151184,0.9924825899999999,0.95871892,0.9795286299999999,0.9787553800000001,0.6143639399999999,0.5845745600000001,0.82262861,0.9101746700000001,0.8205350899999999,9.360158,1.539955,1.6984299999999999,16,1,16,16x1 7 | -------------------------------------------------------------------------------- /pypop/examples/mpi/pypop_gui.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "hide_input": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import os\n", 12 | "from pypop.notebook_interface import MetricsWizard\n", 13 | "from pypop.metrics import MPI_Metrics\n", 14 | "\n", 15 | "tracedir=\"./epoch_example_traces\"\n", 16 | "gui = MetricsWizard(MPI_Metrics,\n", 17 | " starting_files=[os.path.join(tracedir, x)\n", 18 | " for x in os.listdir(tracedir)\n", 19 | " if x.endswith(\".prv.gz\")])\n", 20 | "display(gui)" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [] 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Python 3", 34 | "language": "python", 35 | "name": "python3" 36 | }, 37 | "language_info": { 38 | "codemirror_mode": { 39 | "name": "ipython", 40 | "version": 3 41 | }, 42 | "file_extension": ".py", 43 | "mimetype": "text/x-python", 44 | "name": "python", 45 | "nbconvert_exporter": "python", 46 | "pygments_lexer": "ipython3", 47 | "version": "3.8.6" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 4 52 | } 53 | -------------------------------------------------------------------------------- /pypop/examples/openmp/extrae-omp.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | PAPI_TOT_INS,PAPI_TOT_CYC,PAPI_L1_DCM,PAPI_L2_DCM,PAPI_L3_TCM,PAPI_BR_INS,PAPI_BR_MSP,RESOURCE_STALLS 24 | 25 | 26 | PAPI_TOT_INS,PAPI_TOT_CYC,PAPI_VEC_SP,PAPI_SR_INS,PAPI_LD_INS,PAPI_FP_INS 27 | PAPI_TOT_CYC 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | TRACE 40 | 5 41 | /scratch 42 | /gpfs/scratch/bsc41/bsc41273 43 | 44 | 45 | 46 | 5000000 47 | 48 | 49 | 50 | 51 | /gpfs/scratch/bsc41/bsc41273/control 52 | 53 | 54 | 55 | 56 | 10M 57 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp1.pcf: -------------------------------------------------------------------------------- 1 | DEFAULT_OPTIONS 2 | 3 | LEVEL THREAD 4 | UNITS NANOSEC 5 | LOOK_BACK 100 6 | SPEED 1 7 | FLAG_ICONS ENABLED 8 | NUM_OF_STATE_COLORS 1000 9 | YMAX_SCALE 37 10 | 11 | 12 | DEFAULT_SEMANTIC 13 | 14 | THREAD_FUNC State As Is 15 | 16 | 17 | STATES 18 | 0 Idle 19 | 1 Running 20 | 2 Not created 21 | 3 Waiting a message 22 | 4 Blocking Send 23 | 5 Synchronization 24 | 6 Test/Probe 25 | 7 Scheduling and Fork/Join 26 | 8 Wait/WaitAll 27 | 9 Blocked 28 | 10 Immediate Send 29 | 11 Immediate Receive 30 | 12 I/O 31 | 13 Group Communication 32 | 14 Tracing Disabled 33 | 15 Others 34 | 16 Send Receive 35 | 17 Memory transfer 36 | 18 Profiling 37 | 19 On-line analysis 38 | 20 Remote memory access 39 | 21 Atomic memory operation 40 | 22 Memory ordering operation 41 | 23 Distributed locking 42 | 24 Overhead 43 | 25 One-sided op 44 | 26 Startup latency 45 | 27 Waiting links 46 | 28 Data copy 47 | 29 RTT 48 | 30 Allocating memory 49 | 31 Freeing memory 50 | 51 | 52 | STATES_COLOR 53 | 0 {117,195,255} 54 | 1 {0,0,255} 55 | 2 {255,255,255} 56 | 3 {255,0,0} 57 | 4 {255,0,174} 58 | 5 {179,0,0} 59 | 6 {0,255,0} 60 | 7 {255,255,0} 61 | 8 {235,0,0} 62 | 9 {0,162,0} 63 | 10 {255,0,255} 64 | 11 {100,100,177} 65 | 12 {172,174,41} 66 | 13 {255,144,26} 67 | 14 {2,255,177} 68 | 15 {192,224,0} 69 | 16 {66,66,66} 70 | 17 {255,0,96} 71 | 18 {169,169,169} 72 | 19 {169,0,0} 73 | 20 {0,109,255} 74 | 21 {200,61,68} 75 | 22 {200,66,0} 76 | 23 {0,41,0} 77 | 24 {139,121,177} 78 | 25 {116,116,116} 79 | 26 {200,50,89} 80 | 27 {255,171,98} 81 | 28 {0,68,189} 82 | 29 {52,43,0} 83 | 30 {255,46,0} 84 | 31 {100,216,32} 85 | 86 | 87 | EVENT_TYPE 88 | 0 60000001 Parallel (OMP) 89 | VALUES 90 | 0 close 91 | 1 DO (open) 92 | 2 SECTIONS (open) 93 | 3 REGION (open) 94 | 95 | EVENT_TYPE 96 | 0 60000018 Executed OpenMP parallel function 97 | VALUES 98 | 0 End 99 | 1 Unresolved 100 | 2 _NOT_Found 101 | 3 GetPixelBlue 102 | 4 GetPixelGreen 103 | 104 | 105 | EVENT_TYPE 106 | 0 60000118 Executed OpenMP parallel function line and file 107 | VALUES 108 | 0 End 109 | 1 0 (Unresolved) 110 | 2 0 (_NOT_Found) 111 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 112 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 113 | 114 | 115 | EVENT_TYPE 116 | 0 60000023 Executed OpenMP task function 117 | VALUES 118 | 0 End 119 | 1 Unresolved 120 | 2 _NOT_Found 121 | 3 GetPixelBlue 122 | 4 GetPixelGreen 123 | 124 | 125 | EVENT_TYPE 126 | 0 60000123 Executed OpenMP task function line and file 127 | VALUES 128 | 0 End 129 | 1 0 (Unresolved) 130 | 2 0 (_NOT_Found) 131 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 132 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 133 | 134 | 135 | EVENT_TYPE 136 | 0 60000024 Instantiated OpenMP task function 137 | VALUES 138 | 0 End 139 | 1 Unresolved 140 | 2 _NOT_Found 141 | 3 GetPixelBlue 142 | 4 GetPixelGreen 143 | 144 | 145 | EVENT_TYPE 146 | 0 60000124 Instantiated OpenMP task function line and file 147 | VALUES 148 | 0 End 149 | 1 0 (Unresolved) 150 | 2 0 (_NOT_Found) 151 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 152 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 153 | 154 | 155 | EVENT_TYPE 156 | 0 60000006 OpenMP named-Lock 157 | VALUES 158 | 0 Unlocked status 159 | 3 Lock 160 | 5 Unlock 161 | 6 Locked status 162 | 163 | EVENT_TYPE 164 | 0 60000032 OpenMP named-Lock address name 165 | EVENT_TYPE 166 | 0 65000000 Number of OpenMP instantiated tasks 167 | 0 65000001 Number of OpenMP executed tasks 168 | 169 | EVENT_TYPE 170 | 0 60000020 pthread function 171 | VALUES 172 | 0 End 173 | 1 Unresolved 174 | 2 _NOT_Found 175 | 3 GetPixelBlue 176 | 4 GetPixelGreen 177 | 178 | 179 | EVENT_TYPE 180 | 0 60000120 pthread function line and file 181 | VALUES 182 | 0 End 183 | 1 0 (Unresolved) 184 | 2 0 (_NOT_Found) 185 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 186 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 187 | 188 | 189 | EVENT_TYPE 190 | 6 40000033 Executing CPU 191 | 6 40000133 CPU-Event sampling interval 192 | 193 | 194 | EVENT_TYPE 195 | 6 40000001 Application 196 | VALUES 197 | 0 End 198 | 1 Begin 199 | 200 | 201 | EVENT_TYPE 202 | 6 40000050 RAW clock() value from system 203 | 204 | 205 | EVENT_TYPE 206 | 6 40000003 Flushing Traces 207 | VALUES 208 | 0 End 209 | 1 Begin 210 | 211 | 212 | EVENT_TYPE 213 | 6 40000002 Trace initialization 214 | VALUES 215 | 0 End 216 | 1 Begin 217 | 218 | 219 | EVENT_TYPE 220 | 6 40000036 Process IDentifier 221 | 6 40000037 Parent Process IDentifier 222 | 6 40000038 fork() depth 223 | 224 | 225 | EVENT_TYPE 226 | 7 42000050 PAPI_TOT_INS [Instr completed] 227 | 7 42000059 PAPI_TOT_CYC [Total cycles] 228 | 7 42000000 PAPI_L1_DCM [L1D cache misses] 229 | 7 42000002 PAPI_L2_DCM [L2D cache misses] 230 | 7 42000008 PAPI_L3_TCM [L3 cache misses] 231 | 7 42000055 PAPI_BR_INS [Branches] 232 | 7 42000046 PAPI_BR_MSP [Cond br mspredictd] 233 | 7 42001047 RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason] 234 | 7 41999999 Active hardware counter set 235 | 236 | 237 | GRADIENT_COLOR 238 | 0 {0,255,2} 239 | 1 {0,244,13} 240 | 2 {0,232,25} 241 | 3 {0,220,37} 242 | 4 {0,209,48} 243 | 5 {0,197,60} 244 | 6 {0,185,72} 245 | 7 {0,173,84} 246 | 8 {0,162,95} 247 | 9 {0,150,107} 248 | 10 {0,138,119} 249 | 11 {0,127,130} 250 | 12 {0,115,142} 251 | 13 {0,103,154} 252 | 14 {0,91,166} 253 | 254 | 255 | GRADIENT_NAMES 256 | 0 Gradient 0 257 | 1 Grad. 1/MPI Events 258 | 2 Grad. 2/OMP Events 259 | 3 Grad. 3/OMP locks 260 | 4 Grad. 4/User func 261 | 5 Grad. 5/User Events 262 | 6 Grad. 6/General Events 263 | 7 Grad. 7/Hardware Counters 264 | 8 Gradient 8 265 | 9 Gradient 9 266 | 10 Gradient 10 267 | 11 Gradient 11 268 | 12 Gradient 12 269 | 13 Gradient 13 270 | 14 Gradient 14 271 | 272 | 273 | EVENT_TYPE 274 | 0 41000000 Object addresses for task 1.1 275 | VALUES 276 | 1 completed.8406 [0x00004248-0x00004248] 277 | 2 __FRAME_END__ [0x00002218-0x00002217] 278 | 3 __GNU_EH_FRAME_HDR [0x00002078-0x00002077] 279 | 4 _IO_stdin_used [0x00002000-0x00002003] 280 | 5 __bss_start [0x00004240-0x0000423f] 281 | 6 _end [0x00004250-0x0000424f] 282 | 7 stdout@@GLIBC_2.2.5 [0x00004240-0x00004247] 283 | 284 | EVENT_TYPE 285 | 9 40000018 Tracing mode: 286 | VALUES 287 | 1 Detailed 288 | 2 CPU Bursts 289 | 290 | 291 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp1.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/openmp/imagemagick_example_traces/omp1.prv.gz -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp1.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 1 2 | 1.vetinari 3 | 4 | LEVEL NODE SIZE 1 5 | vetinari 6 | 7 | LEVEL THREAD SIZE 1 8 | THREAD 1.1.1 9 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp2.pcf: -------------------------------------------------------------------------------- 1 | DEFAULT_OPTIONS 2 | 3 | LEVEL THREAD 4 | UNITS NANOSEC 5 | LOOK_BACK 100 6 | SPEED 1 7 | FLAG_ICONS ENABLED 8 | NUM_OF_STATE_COLORS 1000 9 | YMAX_SCALE 37 10 | 11 | 12 | DEFAULT_SEMANTIC 13 | 14 | THREAD_FUNC State As Is 15 | 16 | 17 | STATES 18 | 0 Idle 19 | 1 Running 20 | 2 Not created 21 | 3 Waiting a message 22 | 4 Blocking Send 23 | 5 Synchronization 24 | 6 Test/Probe 25 | 7 Scheduling and Fork/Join 26 | 8 Wait/WaitAll 27 | 9 Blocked 28 | 10 Immediate Send 29 | 11 Immediate Receive 30 | 12 I/O 31 | 13 Group Communication 32 | 14 Tracing Disabled 33 | 15 Others 34 | 16 Send Receive 35 | 17 Memory transfer 36 | 18 Profiling 37 | 19 On-line analysis 38 | 20 Remote memory access 39 | 21 Atomic memory operation 40 | 22 Memory ordering operation 41 | 23 Distributed locking 42 | 24 Overhead 43 | 25 One-sided op 44 | 26 Startup latency 45 | 27 Waiting links 46 | 28 Data copy 47 | 29 RTT 48 | 30 Allocating memory 49 | 31 Freeing memory 50 | 51 | 52 | STATES_COLOR 53 | 0 {117,195,255} 54 | 1 {0,0,255} 55 | 2 {255,255,255} 56 | 3 {255,0,0} 57 | 4 {255,0,174} 58 | 5 {179,0,0} 59 | 6 {0,255,0} 60 | 7 {255,255,0} 61 | 8 {235,0,0} 62 | 9 {0,162,0} 63 | 10 {255,0,255} 64 | 11 {100,100,177} 65 | 12 {172,174,41} 66 | 13 {255,144,26} 67 | 14 {2,255,177} 68 | 15 {192,224,0} 69 | 16 {66,66,66} 70 | 17 {255,0,96} 71 | 18 {169,169,169} 72 | 19 {169,0,0} 73 | 20 {0,109,255} 74 | 21 {200,61,68} 75 | 22 {200,66,0} 76 | 23 {0,41,0} 77 | 24 {139,121,177} 78 | 25 {116,116,116} 79 | 26 {200,50,89} 80 | 27 {255,171,98} 81 | 28 {0,68,189} 82 | 29 {52,43,0} 83 | 30 {255,46,0} 84 | 31 {100,216,32} 85 | 86 | 87 | EVENT_TYPE 88 | 0 60000001 Parallel (OMP) 89 | VALUES 90 | 0 close 91 | 1 DO (open) 92 | 2 SECTIONS (open) 93 | 3 REGION (open) 94 | 95 | EVENT_TYPE 96 | 0 60000018 Executed OpenMP parallel function 97 | VALUES 98 | 0 End 99 | 1 Unresolved 100 | 2 _NOT_Found 101 | 3 GetPixelBlue 102 | 4 GetPixelGreen 103 | 104 | 105 | EVENT_TYPE 106 | 0 60000118 Executed OpenMP parallel function line and file 107 | VALUES 108 | 0 End 109 | 1 0 (Unresolved) 110 | 2 0 (_NOT_Found) 111 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 112 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 113 | 114 | 115 | EVENT_TYPE 116 | 0 60000023 Executed OpenMP task function 117 | VALUES 118 | 0 End 119 | 1 Unresolved 120 | 2 _NOT_Found 121 | 3 GetPixelBlue 122 | 4 GetPixelGreen 123 | 124 | 125 | EVENT_TYPE 126 | 0 60000123 Executed OpenMP task function line and file 127 | VALUES 128 | 0 End 129 | 1 0 (Unresolved) 130 | 2 0 (_NOT_Found) 131 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 132 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 133 | 134 | 135 | EVENT_TYPE 136 | 0 60000024 Instantiated OpenMP task function 137 | VALUES 138 | 0 End 139 | 1 Unresolved 140 | 2 _NOT_Found 141 | 3 GetPixelBlue 142 | 4 GetPixelGreen 143 | 144 | 145 | EVENT_TYPE 146 | 0 60000124 Instantiated OpenMP task function line and file 147 | VALUES 148 | 0 End 149 | 1 0 (Unresolved) 150 | 2 0 (_NOT_Found) 151 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 152 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 153 | 154 | 155 | EVENT_TYPE 156 | 0 60000006 OpenMP named-Lock 157 | VALUES 158 | 0 Unlocked status 159 | 3 Lock 160 | 5 Unlock 161 | 6 Locked status 162 | 163 | EVENT_TYPE 164 | 0 60000032 OpenMP named-Lock address name 165 | EVENT_TYPE 166 | 0 65000000 Number of OpenMP instantiated tasks 167 | 0 65000001 Number of OpenMP executed tasks 168 | 169 | EVENT_TYPE 170 | 0 60000020 pthread function 171 | VALUES 172 | 0 End 173 | 1 Unresolved 174 | 2 _NOT_Found 175 | 3 GetPixelBlue 176 | 4 GetPixelGreen 177 | 178 | 179 | EVENT_TYPE 180 | 0 60000120 pthread function line and file 181 | VALUES 182 | 0 End 183 | 1 0 (Unresolved) 184 | 2 0 (_NOT_Found) 185 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 186 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 187 | 188 | 189 | EVENT_TYPE 190 | 6 40000033 Executing CPU 191 | 6 40000133 CPU-Event sampling interval 192 | 193 | 194 | EVENT_TYPE 195 | 6 40000001 Application 196 | VALUES 197 | 0 End 198 | 1 Begin 199 | 200 | 201 | EVENT_TYPE 202 | 6 40000050 RAW clock() value from system 203 | 204 | 205 | EVENT_TYPE 206 | 6 40000003 Flushing Traces 207 | VALUES 208 | 0 End 209 | 1 Begin 210 | 211 | 212 | EVENT_TYPE 213 | 6 40000002 Trace initialization 214 | VALUES 215 | 0 End 216 | 1 Begin 217 | 218 | 219 | EVENT_TYPE 220 | 6 40000036 Process IDentifier 221 | 6 40000037 Parent Process IDentifier 222 | 6 40000038 fork() depth 223 | 224 | 225 | EVENT_TYPE 226 | 7 42000050 PAPI_TOT_INS [Instr completed] 227 | 7 42000059 PAPI_TOT_CYC [Total cycles] 228 | 7 42000000 PAPI_L1_DCM [L1D cache misses] 229 | 7 42000002 PAPI_L2_DCM [L2D cache misses] 230 | 7 42000008 PAPI_L3_TCM [L3 cache misses] 231 | 7 42000055 PAPI_BR_INS [Branches] 232 | 7 42000046 PAPI_BR_MSP [Cond br mspredictd] 233 | 7 42001047 RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason] 234 | 7 41999999 Active hardware counter set 235 | 236 | 237 | GRADIENT_COLOR 238 | 0 {0,255,2} 239 | 1 {0,244,13} 240 | 2 {0,232,25} 241 | 3 {0,220,37} 242 | 4 {0,209,48} 243 | 5 {0,197,60} 244 | 6 {0,185,72} 245 | 7 {0,173,84} 246 | 8 {0,162,95} 247 | 9 {0,150,107} 248 | 10 {0,138,119} 249 | 11 {0,127,130} 250 | 12 {0,115,142} 251 | 13 {0,103,154} 252 | 14 {0,91,166} 253 | 254 | 255 | GRADIENT_NAMES 256 | 0 Gradient 0 257 | 1 Grad. 1/MPI Events 258 | 2 Grad. 2/OMP Events 259 | 3 Grad. 3/OMP locks 260 | 4 Grad. 4/User func 261 | 5 Grad. 5/User Events 262 | 6 Grad. 6/General Events 263 | 7 Grad. 7/Hardware Counters 264 | 8 Gradient 8 265 | 9 Gradient 9 266 | 10 Gradient 10 267 | 11 Gradient 11 268 | 12 Gradient 12 269 | 13 Gradient 13 270 | 14 Gradient 14 271 | 272 | 273 | EVENT_TYPE 274 | 0 41000000 Object addresses for task 1.1 275 | VALUES 276 | 1 completed.8406 [0x00004248-0x00004248] 277 | 2 __FRAME_END__ [0x00002218-0x00002217] 278 | 3 __GNU_EH_FRAME_HDR [0x00002078-0x00002077] 279 | 4 _IO_stdin_used [0x00002000-0x00002003] 280 | 5 __bss_start [0x00004240-0x0000423f] 281 | 6 _end [0x00004250-0x0000424f] 282 | 7 stdout@@GLIBC_2.2.5 [0x00004240-0x00004247] 283 | 284 | EVENT_TYPE 285 | 9 40000018 Tracing mode: 286 | VALUES 287 | 1 Detailed 288 | 2 CPU Bursts 289 | 290 | 291 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp2.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/openmp/imagemagick_example_traces/omp2.prv.gz -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp2.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 2 2 | 1.vetinari 3 | 2.vetinari 4 | 5 | LEVEL NODE SIZE 1 6 | vetinari 7 | 8 | LEVEL THREAD SIZE 2 9 | THREAD 1.1.1 10 | THREAD 1.1.2 11 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp4.pcf: -------------------------------------------------------------------------------- 1 | DEFAULT_OPTIONS 2 | 3 | LEVEL THREAD 4 | UNITS NANOSEC 5 | LOOK_BACK 100 6 | SPEED 1 7 | FLAG_ICONS ENABLED 8 | NUM_OF_STATE_COLORS 1000 9 | YMAX_SCALE 37 10 | 11 | 12 | DEFAULT_SEMANTIC 13 | 14 | THREAD_FUNC State As Is 15 | 16 | 17 | STATES 18 | 0 Idle 19 | 1 Running 20 | 2 Not created 21 | 3 Waiting a message 22 | 4 Blocking Send 23 | 5 Synchronization 24 | 6 Test/Probe 25 | 7 Scheduling and Fork/Join 26 | 8 Wait/WaitAll 27 | 9 Blocked 28 | 10 Immediate Send 29 | 11 Immediate Receive 30 | 12 I/O 31 | 13 Group Communication 32 | 14 Tracing Disabled 33 | 15 Others 34 | 16 Send Receive 35 | 17 Memory transfer 36 | 18 Profiling 37 | 19 On-line analysis 38 | 20 Remote memory access 39 | 21 Atomic memory operation 40 | 22 Memory ordering operation 41 | 23 Distributed locking 42 | 24 Overhead 43 | 25 One-sided op 44 | 26 Startup latency 45 | 27 Waiting links 46 | 28 Data copy 47 | 29 RTT 48 | 30 Allocating memory 49 | 31 Freeing memory 50 | 51 | 52 | STATES_COLOR 53 | 0 {117,195,255} 54 | 1 {0,0,255} 55 | 2 {255,255,255} 56 | 3 {255,0,0} 57 | 4 {255,0,174} 58 | 5 {179,0,0} 59 | 6 {0,255,0} 60 | 7 {255,255,0} 61 | 8 {235,0,0} 62 | 9 {0,162,0} 63 | 10 {255,0,255} 64 | 11 {100,100,177} 65 | 12 {172,174,41} 66 | 13 {255,144,26} 67 | 14 {2,255,177} 68 | 15 {192,224,0} 69 | 16 {66,66,66} 70 | 17 {255,0,96} 71 | 18 {169,169,169} 72 | 19 {169,0,0} 73 | 20 {0,109,255} 74 | 21 {200,61,68} 75 | 22 {200,66,0} 76 | 23 {0,41,0} 77 | 24 {139,121,177} 78 | 25 {116,116,116} 79 | 26 {200,50,89} 80 | 27 {255,171,98} 81 | 28 {0,68,189} 82 | 29 {52,43,0} 83 | 30 {255,46,0} 84 | 31 {100,216,32} 85 | 86 | 87 | EVENT_TYPE 88 | 0 60000001 Parallel (OMP) 89 | VALUES 90 | 0 close 91 | 1 DO (open) 92 | 2 SECTIONS (open) 93 | 3 REGION (open) 94 | 95 | EVENT_TYPE 96 | 0 60000018 Executed OpenMP parallel function 97 | VALUES 98 | 0 End 99 | 1 Unresolved 100 | 2 _NOT_Found 101 | 3 GetPixelBlue 102 | 4 GetPixelGreen 103 | 104 | 105 | EVENT_TYPE 106 | 0 60000118 Executed OpenMP parallel function line and file 107 | VALUES 108 | 0 End 109 | 1 0 (Unresolved) 110 | 2 0 (_NOT_Found) 111 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 112 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 113 | 114 | 115 | EVENT_TYPE 116 | 0 60000023 Executed OpenMP task function 117 | VALUES 118 | 0 End 119 | 1 Unresolved 120 | 2 _NOT_Found 121 | 3 GetPixelBlue 122 | 4 GetPixelGreen 123 | 124 | 125 | EVENT_TYPE 126 | 0 60000123 Executed OpenMP task function line and file 127 | VALUES 128 | 0 End 129 | 1 0 (Unresolved) 130 | 2 0 (_NOT_Found) 131 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 132 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 133 | 134 | 135 | EVENT_TYPE 136 | 0 60000024 Instantiated OpenMP task function 137 | VALUES 138 | 0 End 139 | 1 Unresolved 140 | 2 _NOT_Found 141 | 3 GetPixelBlue 142 | 4 GetPixelGreen 143 | 144 | 145 | EVENT_TYPE 146 | 0 60000124 Instantiated OpenMP task function line and file 147 | VALUES 148 | 0 End 149 | 1 0 (Unresolved) 150 | 2 0 (_NOT_Found) 151 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 152 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 153 | 154 | 155 | EVENT_TYPE 156 | 0 60000006 OpenMP named-Lock 157 | VALUES 158 | 0 Unlocked status 159 | 3 Lock 160 | 5 Unlock 161 | 6 Locked status 162 | 163 | EVENT_TYPE 164 | 0 60000032 OpenMP named-Lock address name 165 | EVENT_TYPE 166 | 0 65000000 Number of OpenMP instantiated tasks 167 | 0 65000001 Number of OpenMP executed tasks 168 | 169 | EVENT_TYPE 170 | 0 60000020 pthread function 171 | VALUES 172 | 0 End 173 | 1 Unresolved 174 | 2 _NOT_Found 175 | 3 GetPixelBlue 176 | 4 GetPixelGreen 177 | 178 | 179 | EVENT_TYPE 180 | 0 60000120 pthread function line and file 181 | VALUES 182 | 0 End 183 | 1 0 (Unresolved) 184 | 2 0 (_NOT_Found) 185 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 186 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 187 | 188 | 189 | EVENT_TYPE 190 | 6 40000033 Executing CPU 191 | 6 40000133 CPU-Event sampling interval 192 | 193 | 194 | EVENT_TYPE 195 | 6 40000001 Application 196 | VALUES 197 | 0 End 198 | 1 Begin 199 | 200 | 201 | EVENT_TYPE 202 | 6 40000050 RAW clock() value from system 203 | 204 | 205 | EVENT_TYPE 206 | 6 40000003 Flushing Traces 207 | VALUES 208 | 0 End 209 | 1 Begin 210 | 211 | 212 | EVENT_TYPE 213 | 6 40000002 Trace initialization 214 | VALUES 215 | 0 End 216 | 1 Begin 217 | 218 | 219 | EVENT_TYPE 220 | 6 40000036 Process IDentifier 221 | 6 40000037 Parent Process IDentifier 222 | 6 40000038 fork() depth 223 | 224 | 225 | EVENT_TYPE 226 | 7 42000050 PAPI_TOT_INS [Instr completed] 227 | 7 42000059 PAPI_TOT_CYC [Total cycles] 228 | 7 42000000 PAPI_L1_DCM [L1D cache misses] 229 | 7 42000002 PAPI_L2_DCM [L2D cache misses] 230 | 7 42000008 PAPI_L3_TCM [L3 cache misses] 231 | 7 42000055 PAPI_BR_INS [Branches] 232 | 7 42000046 PAPI_BR_MSP [Cond br mspredictd] 233 | 7 42001047 RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason] 234 | 7 41999999 Active hardware counter set 235 | 236 | 237 | GRADIENT_COLOR 238 | 0 {0,255,2} 239 | 1 {0,244,13} 240 | 2 {0,232,25} 241 | 3 {0,220,37} 242 | 4 {0,209,48} 243 | 5 {0,197,60} 244 | 6 {0,185,72} 245 | 7 {0,173,84} 246 | 8 {0,162,95} 247 | 9 {0,150,107} 248 | 10 {0,138,119} 249 | 11 {0,127,130} 250 | 12 {0,115,142} 251 | 13 {0,103,154} 252 | 14 {0,91,166} 253 | 254 | 255 | GRADIENT_NAMES 256 | 0 Gradient 0 257 | 1 Grad. 1/MPI Events 258 | 2 Grad. 2/OMP Events 259 | 3 Grad. 3/OMP locks 260 | 4 Grad. 4/User func 261 | 5 Grad. 5/User Events 262 | 6 Grad. 6/General Events 263 | 7 Grad. 7/Hardware Counters 264 | 8 Gradient 8 265 | 9 Gradient 9 266 | 10 Gradient 10 267 | 11 Gradient 11 268 | 12 Gradient 12 269 | 13 Gradient 13 270 | 14 Gradient 14 271 | 272 | 273 | EVENT_TYPE 274 | 0 41000000 Object addresses for task 1.1 275 | VALUES 276 | 1 completed.8406 [0x00004248-0x00004248] 277 | 2 __FRAME_END__ [0x00002218-0x00002217] 278 | 3 __GNU_EH_FRAME_HDR [0x00002078-0x00002077] 279 | 4 _IO_stdin_used [0x00002000-0x00002003] 280 | 5 __bss_start [0x00004240-0x0000423f] 281 | 6 _end [0x00004250-0x0000424f] 282 | 7 stdout@@GLIBC_2.2.5 [0x00004240-0x00004247] 283 | 284 | EVENT_TYPE 285 | 9 40000018 Tracing mode: 286 | VALUES 287 | 1 Detailed 288 | 2 CPU Bursts 289 | 290 | 291 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp4.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/openmp/imagemagick_example_traces/omp4.prv.gz -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp4.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 4 2 | 1.vetinari 3 | 2.vetinari 4 | 3.vetinari 5 | 4.vetinari 6 | 7 | LEVEL NODE SIZE 1 8 | vetinari 9 | 10 | LEVEL THREAD SIZE 4 11 | THREAD 1.1.1 12 | THREAD 1.1.2 13 | THREAD 1.1.3 14 | THREAD 1.1.4 15 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp6.pcf: -------------------------------------------------------------------------------- 1 | DEFAULT_OPTIONS 2 | 3 | LEVEL THREAD 4 | UNITS NANOSEC 5 | LOOK_BACK 100 6 | SPEED 1 7 | FLAG_ICONS ENABLED 8 | NUM_OF_STATE_COLORS 1000 9 | YMAX_SCALE 37 10 | 11 | 12 | DEFAULT_SEMANTIC 13 | 14 | THREAD_FUNC State As Is 15 | 16 | 17 | STATES 18 | 0 Idle 19 | 1 Running 20 | 2 Not created 21 | 3 Waiting a message 22 | 4 Blocking Send 23 | 5 Synchronization 24 | 6 Test/Probe 25 | 7 Scheduling and Fork/Join 26 | 8 Wait/WaitAll 27 | 9 Blocked 28 | 10 Immediate Send 29 | 11 Immediate Receive 30 | 12 I/O 31 | 13 Group Communication 32 | 14 Tracing Disabled 33 | 15 Others 34 | 16 Send Receive 35 | 17 Memory transfer 36 | 18 Profiling 37 | 19 On-line analysis 38 | 20 Remote memory access 39 | 21 Atomic memory operation 40 | 22 Memory ordering operation 41 | 23 Distributed locking 42 | 24 Overhead 43 | 25 One-sided op 44 | 26 Startup latency 45 | 27 Waiting links 46 | 28 Data copy 47 | 29 RTT 48 | 30 Allocating memory 49 | 31 Freeing memory 50 | 51 | 52 | STATES_COLOR 53 | 0 {117,195,255} 54 | 1 {0,0,255} 55 | 2 {255,255,255} 56 | 3 {255,0,0} 57 | 4 {255,0,174} 58 | 5 {179,0,0} 59 | 6 {0,255,0} 60 | 7 {255,255,0} 61 | 8 {235,0,0} 62 | 9 {0,162,0} 63 | 10 {255,0,255} 64 | 11 {100,100,177} 65 | 12 {172,174,41} 66 | 13 {255,144,26} 67 | 14 {2,255,177} 68 | 15 {192,224,0} 69 | 16 {66,66,66} 70 | 17 {255,0,96} 71 | 18 {169,169,169} 72 | 19 {169,0,0} 73 | 20 {0,109,255} 74 | 21 {200,61,68} 75 | 22 {200,66,0} 76 | 23 {0,41,0} 77 | 24 {139,121,177} 78 | 25 {116,116,116} 79 | 26 {200,50,89} 80 | 27 {255,171,98} 81 | 28 {0,68,189} 82 | 29 {52,43,0} 83 | 30 {255,46,0} 84 | 31 {100,216,32} 85 | 86 | 87 | EVENT_TYPE 88 | 0 60000001 Parallel (OMP) 89 | VALUES 90 | 0 close 91 | 1 DO (open) 92 | 2 SECTIONS (open) 93 | 3 REGION (open) 94 | 95 | EVENT_TYPE 96 | 0 60000018 Executed OpenMP parallel function 97 | VALUES 98 | 0 End 99 | 1 Unresolved 100 | 2 _NOT_Found 101 | 3 GetPixelBlue 102 | 4 GetPixelGreen 103 | 104 | 105 | EVENT_TYPE 106 | 0 60000118 Executed OpenMP parallel function line and file 107 | VALUES 108 | 0 End 109 | 1 0 (Unresolved) 110 | 2 0 (_NOT_Found) 111 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 112 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 113 | 114 | 115 | EVENT_TYPE 116 | 0 60000023 Executed OpenMP task function 117 | VALUES 118 | 0 End 119 | 1 Unresolved 120 | 2 _NOT_Found 121 | 3 GetPixelBlue 122 | 4 GetPixelGreen 123 | 124 | 125 | EVENT_TYPE 126 | 0 60000123 Executed OpenMP task function line and file 127 | VALUES 128 | 0 End 129 | 1 0 (Unresolved) 130 | 2 0 (_NOT_Found) 131 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 132 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 133 | 134 | 135 | EVENT_TYPE 136 | 0 60000024 Instantiated OpenMP task function 137 | VALUES 138 | 0 End 139 | 1 Unresolved 140 | 2 _NOT_Found 141 | 3 GetPixelBlue 142 | 4 GetPixelGreen 143 | 144 | 145 | EVENT_TYPE 146 | 0 60000124 Instantiated OpenMP task function line and file 147 | VALUES 148 | 0 End 149 | 1 0 (Unresolved) 150 | 2 0 (_NOT_Found) 151 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 152 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 153 | 154 | 155 | EVENT_TYPE 156 | 0 60000006 OpenMP named-Lock 157 | VALUES 158 | 0 Unlocked status 159 | 3 Lock 160 | 5 Unlock 161 | 6 Locked status 162 | 163 | EVENT_TYPE 164 | 0 60000032 OpenMP named-Lock address name 165 | EVENT_TYPE 166 | 0 65000000 Number of OpenMP instantiated tasks 167 | 0 65000001 Number of OpenMP executed tasks 168 | 169 | EVENT_TYPE 170 | 0 60000020 pthread function 171 | VALUES 172 | 0 End 173 | 1 Unresolved 174 | 2 _NOT_Found 175 | 3 GetPixelBlue 176 | 4 GetPixelGreen 177 | 178 | 179 | EVENT_TYPE 180 | 0 60000120 pthread function line and file 181 | VALUES 182 | 0 End 183 | 1 0 (Unresolved) 184 | 2 0 (_NOT_Found) 185 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 186 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 187 | 188 | 189 | EVENT_TYPE 190 | 6 40000033 Executing CPU 191 | 6 40000133 CPU-Event sampling interval 192 | 193 | 194 | EVENT_TYPE 195 | 6 40000001 Application 196 | VALUES 197 | 0 End 198 | 1 Begin 199 | 200 | 201 | EVENT_TYPE 202 | 6 40000050 RAW clock() value from system 203 | 204 | 205 | EVENT_TYPE 206 | 6 40000003 Flushing Traces 207 | VALUES 208 | 0 End 209 | 1 Begin 210 | 211 | 212 | EVENT_TYPE 213 | 6 40000002 Trace initialization 214 | VALUES 215 | 0 End 216 | 1 Begin 217 | 218 | 219 | EVENT_TYPE 220 | 6 40000036 Process IDentifier 221 | 6 40000037 Parent Process IDentifier 222 | 6 40000038 fork() depth 223 | 224 | 225 | EVENT_TYPE 226 | 7 42000050 PAPI_TOT_INS [Instr completed] 227 | 7 42000059 PAPI_TOT_CYC [Total cycles] 228 | 7 42000000 PAPI_L1_DCM [L1D cache misses] 229 | 7 42000002 PAPI_L2_DCM [L2D cache misses] 230 | 7 42000008 PAPI_L3_TCM [L3 cache misses] 231 | 7 42000055 PAPI_BR_INS [Branches] 232 | 7 42000046 PAPI_BR_MSP [Cond br mspredictd] 233 | 7 42001047 RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason] 234 | 7 41999999 Active hardware counter set 235 | 236 | 237 | GRADIENT_COLOR 238 | 0 {0,255,2} 239 | 1 {0,244,13} 240 | 2 {0,232,25} 241 | 3 {0,220,37} 242 | 4 {0,209,48} 243 | 5 {0,197,60} 244 | 6 {0,185,72} 245 | 7 {0,173,84} 246 | 8 {0,162,95} 247 | 9 {0,150,107} 248 | 10 {0,138,119} 249 | 11 {0,127,130} 250 | 12 {0,115,142} 251 | 13 {0,103,154} 252 | 14 {0,91,166} 253 | 254 | 255 | GRADIENT_NAMES 256 | 0 Gradient 0 257 | 1 Grad. 1/MPI Events 258 | 2 Grad. 2/OMP Events 259 | 3 Grad. 3/OMP locks 260 | 4 Grad. 4/User func 261 | 5 Grad. 5/User Events 262 | 6 Grad. 6/General Events 263 | 7 Grad. 7/Hardware Counters 264 | 8 Gradient 8 265 | 9 Gradient 9 266 | 10 Gradient 10 267 | 11 Gradient 11 268 | 12 Gradient 12 269 | 13 Gradient 13 270 | 14 Gradient 14 271 | 272 | 273 | EVENT_TYPE 274 | 0 41000000 Object addresses for task 1.1 275 | VALUES 276 | 1 completed.8406 [0x00004248-0x00004248] 277 | 2 __FRAME_END__ [0x00002218-0x00002217] 278 | 3 __GNU_EH_FRAME_HDR [0x00002078-0x00002077] 279 | 4 _IO_stdin_used [0x00002000-0x00002003] 280 | 5 __bss_start [0x00004240-0x0000423f] 281 | 6 _end [0x00004250-0x0000424f] 282 | 7 stdout@@GLIBC_2.2.5 [0x00004240-0x00004247] 283 | 284 | EVENT_TYPE 285 | 9 40000018 Tracing mode: 286 | VALUES 287 | 1 Detailed 288 | 2 CPU Bursts 289 | 290 | 291 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp6.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/openmp/imagemagick_example_traces/omp6.prv.gz -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp6.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 6 2 | 1.vetinari 3 | 2.vetinari 4 | 3.vetinari 5 | 4.vetinari 6 | 5.vetinari 7 | 6.vetinari 8 | 9 | LEVEL NODE SIZE 1 10 | vetinari 11 | 12 | LEVEL THREAD SIZE 6 13 | THREAD 1.1.1 14 | THREAD 1.1.2 15 | THREAD 1.1.3 16 | THREAD 1.1.4 17 | THREAD 1.1.5 18 | THREAD 1.1.6 19 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp8.pcf: -------------------------------------------------------------------------------- 1 | DEFAULT_OPTIONS 2 | 3 | LEVEL THREAD 4 | UNITS NANOSEC 5 | LOOK_BACK 100 6 | SPEED 1 7 | FLAG_ICONS ENABLED 8 | NUM_OF_STATE_COLORS 1000 9 | YMAX_SCALE 37 10 | 11 | 12 | DEFAULT_SEMANTIC 13 | 14 | THREAD_FUNC State As Is 15 | 16 | 17 | STATES 18 | 0 Idle 19 | 1 Running 20 | 2 Not created 21 | 3 Waiting a message 22 | 4 Blocking Send 23 | 5 Synchronization 24 | 6 Test/Probe 25 | 7 Scheduling and Fork/Join 26 | 8 Wait/WaitAll 27 | 9 Blocked 28 | 10 Immediate Send 29 | 11 Immediate Receive 30 | 12 I/O 31 | 13 Group Communication 32 | 14 Tracing Disabled 33 | 15 Others 34 | 16 Send Receive 35 | 17 Memory transfer 36 | 18 Profiling 37 | 19 On-line analysis 38 | 20 Remote memory access 39 | 21 Atomic memory operation 40 | 22 Memory ordering operation 41 | 23 Distributed locking 42 | 24 Overhead 43 | 25 One-sided op 44 | 26 Startup latency 45 | 27 Waiting links 46 | 28 Data copy 47 | 29 RTT 48 | 30 Allocating memory 49 | 31 Freeing memory 50 | 51 | 52 | STATES_COLOR 53 | 0 {117,195,255} 54 | 1 {0,0,255} 55 | 2 {255,255,255} 56 | 3 {255,0,0} 57 | 4 {255,0,174} 58 | 5 {179,0,0} 59 | 6 {0,255,0} 60 | 7 {255,255,0} 61 | 8 {235,0,0} 62 | 9 {0,162,0} 63 | 10 {255,0,255} 64 | 11 {100,100,177} 65 | 12 {172,174,41} 66 | 13 {255,144,26} 67 | 14 {2,255,177} 68 | 15 {192,224,0} 69 | 16 {66,66,66} 70 | 17 {255,0,96} 71 | 18 {169,169,169} 72 | 19 {169,0,0} 73 | 20 {0,109,255} 74 | 21 {200,61,68} 75 | 22 {200,66,0} 76 | 23 {0,41,0} 77 | 24 {139,121,177} 78 | 25 {116,116,116} 79 | 26 {200,50,89} 80 | 27 {255,171,98} 81 | 28 {0,68,189} 82 | 29 {52,43,0} 83 | 30 {255,46,0} 84 | 31 {100,216,32} 85 | 86 | 87 | EVENT_TYPE 88 | 0 60000001 Parallel (OMP) 89 | VALUES 90 | 0 close 91 | 1 DO (open) 92 | 2 SECTIONS (open) 93 | 3 REGION (open) 94 | 95 | EVENT_TYPE 96 | 0 60000018 Executed OpenMP parallel function 97 | VALUES 98 | 0 End 99 | 1 Unresolved 100 | 2 _NOT_Found 101 | 3 GetPixelBlue 102 | 4 GetPixelGreen 103 | 104 | 105 | EVENT_TYPE 106 | 0 60000118 Executed OpenMP parallel function line and file 107 | VALUES 108 | 0 End 109 | 1 0 (Unresolved) 110 | 2 0 (_NOT_Found) 111 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 112 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 113 | 114 | 115 | EVENT_TYPE 116 | 0 60000023 Executed OpenMP task function 117 | VALUES 118 | 0 End 119 | 1 Unresolved 120 | 2 _NOT_Found 121 | 3 GetPixelBlue 122 | 4 GetPixelGreen 123 | 124 | 125 | EVENT_TYPE 126 | 0 60000123 Executed OpenMP task function line and file 127 | VALUES 128 | 0 End 129 | 1 0 (Unresolved) 130 | 2 0 (_NOT_Found) 131 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 132 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 133 | 134 | 135 | EVENT_TYPE 136 | 0 60000024 Instantiated OpenMP task function 137 | VALUES 138 | 0 End 139 | 1 Unresolved 140 | 2 _NOT_Found 141 | 3 GetPixelBlue 142 | 4 GetPixelGreen 143 | 144 | 145 | EVENT_TYPE 146 | 0 60000124 Instantiated OpenMP task function line and file 147 | VALUES 148 | 0 End 149 | 1 0 (Unresolved) 150 | 2 0 (_NOT_Found) 151 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 152 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 153 | 154 | 155 | EVENT_TYPE 156 | 0 60000006 OpenMP named-Lock 157 | VALUES 158 | 0 Unlocked status 159 | 3 Lock 160 | 5 Unlock 161 | 6 Locked status 162 | 163 | EVENT_TYPE 164 | 0 60000032 OpenMP named-Lock address name 165 | EVENT_TYPE 166 | 0 65000000 Number of OpenMP instantiated tasks 167 | 0 65000001 Number of OpenMP executed tasks 168 | 169 | EVENT_TYPE 170 | 0 60000020 pthread function 171 | VALUES 172 | 0 End 173 | 1 Unresolved 174 | 2 _NOT_Found 175 | 3 GetPixelBlue 176 | 4 GetPixelGreen 177 | 178 | 179 | EVENT_TYPE 180 | 0 60000120 pthread function line and file 181 | VALUES 182 | 0 End 183 | 1 0 (Unresolved) 184 | 2 0 (_NOT_Found) 185 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 186 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 187 | 188 | 189 | EVENT_TYPE 190 | 6 40000033 Executing CPU 191 | 6 40000133 CPU-Event sampling interval 192 | 193 | 194 | EVENT_TYPE 195 | 6 40000001 Application 196 | VALUES 197 | 0 End 198 | 1 Begin 199 | 200 | 201 | EVENT_TYPE 202 | 6 40000050 RAW clock() value from system 203 | 204 | 205 | EVENT_TYPE 206 | 6 40000003 Flushing Traces 207 | VALUES 208 | 0 End 209 | 1 Begin 210 | 211 | 212 | EVENT_TYPE 213 | 6 40000002 Trace initialization 214 | VALUES 215 | 0 End 216 | 1 Begin 217 | 218 | 219 | EVENT_TYPE 220 | 6 40000036 Process IDentifier 221 | 6 40000037 Parent Process IDentifier 222 | 6 40000038 fork() depth 223 | 224 | 225 | EVENT_TYPE 226 | 7 42000050 PAPI_TOT_INS [Instr completed] 227 | 7 42000059 PAPI_TOT_CYC [Total cycles] 228 | 7 42000000 PAPI_L1_DCM [L1D cache misses] 229 | 7 42000002 PAPI_L2_DCM [L2D cache misses] 230 | 7 42000008 PAPI_L3_TCM [L3 cache misses] 231 | 7 42000055 PAPI_BR_INS [Branches] 232 | 7 42000046 PAPI_BR_MSP [Cond br mspredictd] 233 | 7 42001047 RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason] 234 | 7 41999999 Active hardware counter set 235 | 236 | 237 | GRADIENT_COLOR 238 | 0 {0,255,2} 239 | 1 {0,244,13} 240 | 2 {0,232,25} 241 | 3 {0,220,37} 242 | 4 {0,209,48} 243 | 5 {0,197,60} 244 | 6 {0,185,72} 245 | 7 {0,173,84} 246 | 8 {0,162,95} 247 | 9 {0,150,107} 248 | 10 {0,138,119} 249 | 11 {0,127,130} 250 | 12 {0,115,142} 251 | 13 {0,103,154} 252 | 14 {0,91,166} 253 | 254 | 255 | GRADIENT_NAMES 256 | 0 Gradient 0 257 | 1 Grad. 1/MPI Events 258 | 2 Grad. 2/OMP Events 259 | 3 Grad. 3/OMP locks 260 | 4 Grad. 4/User func 261 | 5 Grad. 5/User Events 262 | 6 Grad. 6/General Events 263 | 7 Grad. 7/Hardware Counters 264 | 8 Gradient 8 265 | 9 Gradient 9 266 | 10 Gradient 10 267 | 11 Gradient 11 268 | 12 Gradient 12 269 | 13 Gradient 13 270 | 14 Gradient 14 271 | 272 | 273 | EVENT_TYPE 274 | 0 41000000 Object addresses for task 1.1 275 | VALUES 276 | 1 completed.8406 [0x00004248-0x00004248] 277 | 2 __FRAME_END__ [0x00002218-0x00002217] 278 | 3 __GNU_EH_FRAME_HDR [0x00002078-0x00002077] 279 | 4 _IO_stdin_used [0x00002000-0x00002003] 280 | 5 __bss_start [0x00004240-0x0000423f] 281 | 6 _end [0x00004250-0x0000424f] 282 | 7 stdout@@GLIBC_2.2.5 [0x00004240-0x00004247] 283 | 284 | EVENT_TYPE 285 | 9 40000018 Tracing mode: 286 | VALUES 287 | 1 Detailed 288 | 2 CPU Bursts 289 | 290 | 291 | -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp8.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/openmp/imagemagick_example_traces/omp8.prv.gz -------------------------------------------------------------------------------- /pypop/examples/openmp/imagemagick_example_traces/omp8.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 8 2 | 1.vetinari 3 | 2.vetinari 4 | 3.vetinari 5 | 4.vetinari 6 | 5.vetinari 7 | 6.vetinari 8 | 7.vetinari 9 | 8.vetinari 10 | 11 | LEVEL NODE SIZE 1 12 | vetinari 13 | 14 | LEVEL THREAD SIZE 8 15 | THREAD 1.1.1 16 | THREAD 1.1.2 17 | THREAD 1.1.3 18 | THREAD 1.1.4 19 | THREAD 1.1.5 20 | THREAD 1.1.6 21 | THREAD 1.1.7 22 | THREAD 1.1.8 23 | -------------------------------------------------------------------------------- /pypop/examples/openmp/omp_detail.pcf: -------------------------------------------------------------------------------- 1 | DEFAULT_OPTIONS 2 | 3 | LEVEL THREAD 4 | UNITS NANOSEC 5 | LOOK_BACK 100 6 | SPEED 1 7 | FLAG_ICONS ENABLED 8 | NUM_OF_STATE_COLORS 1000 9 | YMAX_SCALE 37 10 | 11 | 12 | DEFAULT_SEMANTIC 13 | 14 | THREAD_FUNC State As Is 15 | 16 | 17 | STATES 18 | 0 Idle 19 | 1 Running 20 | 2 Not created 21 | 3 Waiting a message 22 | 4 Blocking Send 23 | 5 Synchronization 24 | 6 Test/Probe 25 | 7 Scheduling and Fork/Join 26 | 8 Wait/WaitAll 27 | 9 Blocked 28 | 10 Immediate Send 29 | 11 Immediate Receive 30 | 12 I/O 31 | 13 Group Communication 32 | 14 Tracing Disabled 33 | 15 Others 34 | 16 Send Receive 35 | 17 Memory transfer 36 | 18 Profiling 37 | 19 On-line analysis 38 | 20 Remote memory access 39 | 21 Atomic memory operation 40 | 22 Memory ordering operation 41 | 23 Distributed locking 42 | 24 Overhead 43 | 25 One-sided op 44 | 26 Startup latency 45 | 27 Waiting links 46 | 28 Data copy 47 | 29 RTT 48 | 30 Allocating memory 49 | 31 Freeing memory 50 | 51 | 52 | STATES_COLOR 53 | 0 {117,195,255} 54 | 1 {0,0,255} 55 | 2 {255,255,255} 56 | 3 {255,0,0} 57 | 4 {255,0,174} 58 | 5 {179,0,0} 59 | 6 {0,255,0} 60 | 7 {255,255,0} 61 | 8 {235,0,0} 62 | 9 {0,162,0} 63 | 10 {255,0,255} 64 | 11 {100,100,177} 65 | 12 {172,174,41} 66 | 13 {255,144,26} 67 | 14 {2,255,177} 68 | 15 {192,224,0} 69 | 16 {66,66,66} 70 | 17 {255,0,96} 71 | 18 {169,169,169} 72 | 19 {169,0,0} 73 | 20 {0,109,255} 74 | 21 {200,61,68} 75 | 22 {200,66,0} 76 | 23 {0,41,0} 77 | 24 {139,121,177} 78 | 25 {116,116,116} 79 | 26 {200,50,89} 80 | 27 {255,171,98} 81 | 28 {0,68,189} 82 | 29 {52,43,0} 83 | 30 {255,46,0} 84 | 31 {100,216,32} 85 | 86 | 87 | EVENT_TYPE 88 | 0 60000001 Parallel (OMP) 89 | VALUES 90 | 0 close 91 | 1 DO (open) 92 | 2 SECTIONS (open) 93 | 3 REGION (open) 94 | 95 | EVENT_TYPE 96 | 0 60000018 Executed OpenMP parallel function 97 | VALUES 98 | 0 End 99 | 1 Unresolved 100 | 2 _NOT_Found 101 | 3 GetPixelBlue 102 | 4 GetPixelGreen 103 | 104 | 105 | EVENT_TYPE 106 | 0 60000118 Executed OpenMP parallel function line and file 107 | VALUES 108 | 0 End 109 | 1 0 (Unresolved) 110 | 2 0 (_NOT_Found) 111 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 112 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 113 | 114 | 115 | EVENT_TYPE 116 | 0 60000023 Executed OpenMP task function 117 | VALUES 118 | 0 End 119 | 1 Unresolved 120 | 2 _NOT_Found 121 | 3 GetPixelBlue 122 | 4 GetPixelGreen 123 | 124 | 125 | EVENT_TYPE 126 | 0 60000123 Executed OpenMP task function line and file 127 | VALUES 128 | 0 End 129 | 1 0 (Unresolved) 130 | 2 0 (_NOT_Found) 131 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 132 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 133 | 134 | 135 | EVENT_TYPE 136 | 0 60000024 Instantiated OpenMP task function 137 | VALUES 138 | 0 End 139 | 1 Unresolved 140 | 2 _NOT_Found 141 | 3 GetPixelBlue 142 | 4 GetPixelGreen 143 | 144 | 145 | EVENT_TYPE 146 | 0 60000124 Instantiated OpenMP task function line and file 147 | VALUES 148 | 0 End 149 | 1 0 (Unresolved) 150 | 2 0 (_NOT_Found) 151 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 152 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 153 | 154 | 155 | EVENT_TYPE 156 | 0 60000006 OpenMP named-Lock 157 | VALUES 158 | 0 Unlocked status 159 | 3 Lock 160 | 5 Unlock 161 | 6 Locked status 162 | 163 | EVENT_TYPE 164 | 0 60000032 OpenMP named-Lock address name 165 | EVENT_TYPE 166 | 0 65000000 Number of OpenMP instantiated tasks 167 | 0 65000001 Number of OpenMP executed tasks 168 | 169 | EVENT_TYPE 170 | 0 60000020 pthread function 171 | VALUES 172 | 0 End 173 | 1 Unresolved 174 | 2 _NOT_Found 175 | 3 GetPixelBlue 176 | 4 GetPixelGreen 177 | 178 | 179 | EVENT_TYPE 180 | 0 60000120 pthread function line and file 181 | VALUES 182 | 0 End 183 | 1 0 (Unresolved) 184 | 2 0 (_NOT_Found) 185 | 3 92 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 186 | 4 186 (pixel-accessor.h, libMagickCore-7.Q16.so.6.0.0) 187 | 188 | 189 | EVENT_TYPE 190 | 6 40000033 Executing CPU 191 | 6 40000133 CPU-Event sampling interval 192 | 193 | 194 | EVENT_TYPE 195 | 6 40000001 Application 196 | VALUES 197 | 0 End 198 | 1 Begin 199 | 200 | 201 | EVENT_TYPE 202 | 6 40000050 RAW clock() value from system 203 | 204 | 205 | EVENT_TYPE 206 | 6 40000003 Flushing Traces 207 | VALUES 208 | 0 End 209 | 1 Begin 210 | 211 | 212 | EVENT_TYPE 213 | 6 40000002 Trace initialization 214 | VALUES 215 | 0 End 216 | 1 Begin 217 | 218 | 219 | EVENT_TYPE 220 | 6 40000036 Process IDentifier 221 | 6 40000037 Parent Process IDentifier 222 | 6 40000038 fork() depth 223 | 224 | 225 | EVENT_TYPE 226 | 7 42000050 PAPI_TOT_INS [Instr completed] 227 | 7 42000059 PAPI_TOT_CYC [Total cycles] 228 | 7 42000000 PAPI_L1_DCM [L1D cache misses] 229 | 7 42000002 PAPI_L2_DCM [L2D cache misses] 230 | 7 42000008 PAPI_L3_TCM [L3 cache misses] 231 | 7 42000055 PAPI_BR_INS [Branches] 232 | 7 42000046 PAPI_BR_MSP [Cond br mspredictd] 233 | 7 42001047 RESOURCE_STALLS [Cycles Allocation is stalled due to Resource Related reason] 234 | 7 41999999 Active hardware counter set 235 | 236 | 237 | GRADIENT_COLOR 238 | 0 {0,255,2} 239 | 1 {0,244,13} 240 | 2 {0,232,25} 241 | 3 {0,220,37} 242 | 4 {0,209,48} 243 | 5 {0,197,60} 244 | 6 {0,185,72} 245 | 7 {0,173,84} 246 | 8 {0,162,95} 247 | 9 {0,150,107} 248 | 10 {0,138,119} 249 | 11 {0,127,130} 250 | 12 {0,115,142} 251 | 13 {0,103,154} 252 | 14 {0,91,166} 253 | 254 | 255 | GRADIENT_NAMES 256 | 0 Gradient 0 257 | 1 Grad. 1/MPI Events 258 | 2 Grad. 2/OMP Events 259 | 3 Grad. 3/OMP locks 260 | 4 Grad. 4/User func 261 | 5 Grad. 5/User Events 262 | 6 Grad. 6/General Events 263 | 7 Grad. 7/Hardware Counters 264 | 8 Gradient 8 265 | 9 Gradient 9 266 | 10 Gradient 10 267 | 11 Gradient 11 268 | 12 Gradient 12 269 | 13 Gradient 13 270 | 14 Gradient 14 271 | 272 | 273 | EVENT_TYPE 274 | 0 41000000 Object addresses for task 1.1 275 | VALUES 276 | 1 completed.8406 [0x00004248-0x00004248] 277 | 2 __FRAME_END__ [0x00002218-0x00002217] 278 | 3 __GNU_EH_FRAME_HDR [0x00002078-0x00002077] 279 | 4 _IO_stdin_used [0x00002000-0x00002003] 280 | 5 __bss_start [0x00004240-0x0000423f] 281 | 6 _end [0x00004250-0x0000424f] 282 | 7 stdout@@GLIBC_2.2.5 [0x00004240-0x00004247] 283 | 284 | EVENT_TYPE 285 | 9 40000018 Tracing mode: 286 | VALUES 287 | 1 Detailed 288 | 2 CPU Bursts 289 | 290 | 291 | -------------------------------------------------------------------------------- /pypop/examples/openmp/omp_detail.prv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/examples/openmp/omp_detail.prv.gz -------------------------------------------------------------------------------- /pypop/examples/openmp/omp_detail.row: -------------------------------------------------------------------------------- 1 | LEVEL CPU SIZE 8 2 | 1.vetinari 3 | 2.vetinari 4 | 3.vetinari 5 | 4.vetinari 6 | 5.vetinari 7 | 6.vetinari 8 | 7.vetinari 9 | 8.vetinari 10 | 11 | LEVEL NODE SIZE 1 12 | vetinari 13 | 14 | LEVEL THREAD SIZE 8 15 | THREAD 1.1.1 16 | THREAD 1.1.2 17 | THREAD 1.1.3 18 | THREAD 1.1.4 19 | THREAD 1.1.5 20 | THREAD 1.1.6 21 | THREAD 1.1.7 22 | THREAD 1.1.8 23 | -------------------------------------------------------------------------------- /pypop/examples/openmp/profile_imagemagick.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ ! -f "./hubble_orion.tiff" ]; then 4 | echo "Fetching test image" 5 | wget https://imgsrc.hubblesite.org/hvi/uploads/image_file/image_attachment/12728/full_tif.tif \ 6 | --no-check-certificate \ 7 | -O hubble_orion.tiff 8 | fi 9 | 10 | if [ -z "${EXTRAE_HOME}" ]; then 11 | echo "Error: \$EXTRAE_HOME is not set." 12 | exit -1 13 | fi 14 | 15 | mkdir -p imagemagick_example_traces 16 | 17 | for num_threads in 1 $(seq 2 2 8); do 18 | export EXTRAE_CONFIG_FILE="./extrae-omp.xml" 19 | export OMP_NUM_THREADS=${num_threads} 20 | LD_PRELOAD="${EXTRAE_HOME}/libomptrace.so" magick hubble_orion.tiff -resize 25% hubble_orion_small.tiff 21 | mpi2prv -f TRACE.mpits -o imagemagick_example_traces/omp${num_threads}.prv -no-keep-mpits 22 | rm -r set-* 23 | rm hubble_orion_small.tiff 24 | done 25 | 26 | export EXTRAE_CONFIG_FILE="./extrae-omp.xml" 27 | export OMP_NUM_THREADS=8 28 | LD_PRELOAD="${EXTRAE_HOME}/libomptrace.so" magick hubble_orion.tiff -bench 2 -resize 25% hubble_orion_small.tiff 29 | mpi2prv -f TRACE.mpits -o omp_detail.prv -no-keep-mpits 30 | -------------------------------------------------------------------------------- /pypop/filters/tracing_state.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 1 5 | 0 6 | 1 7 | Running 8 | 0 9 | 10 | 11 | 40000012 12 | 13 | 0 14 | 15 | 16 | -------------------------------------------------------------------------------- /pypop/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """\ 6 | Metric Calculator Classes 7 | ------------------------- 8 | 9 | PyPOP provides calculator classes for generating the POP Metrics for different 10 | application types. 11 | 12 | Currently PyPOP supports calculation of the following Metric types: 13 | 14 | .. autosummary:: 15 | :nosignatures: 16 | 17 | MPI_Metrics 18 | MPI_OpenMP_Metrics 19 | MPI_OpenMP_Ineff_Metrics 20 | MPI_OpenMP_Multiplicative_Metrics 21 | OpenMP_Metrics 22 | Judit_Hybrid_Metrics 23 | """ 24 | 25 | from .mpi import MPI_Metrics, MPI_Multiplicative_Metrics 26 | from .hybrid import ( 27 | MPI_OpenMP_Metrics, 28 | MPI_OpenMP_Multiplicative_Metrics, 29 | ) 30 | from .mpi_openmp_ineff import MPI_OpenMP_Ineff_Metrics 31 | from .openmp import OpenMP_Metrics 32 | from .judit import Judit_Hybrid_Metrics 33 | 34 | __all__ = [ 35 | "MPI_Metrics", 36 | "MPI_Multiplicative_Metrics", 37 | "MPI_OpenMP_Metrics", 38 | "MPI_OpenMP_Ineff_Metrics", 39 | "MPI_OpenMP_Multiplicative_Metrics", 40 | "OpenMP_Metrics", 41 | "Judit_Hybrid_Metrics", 42 | ] 43 | -------------------------------------------------------------------------------- /pypop/metrics/judit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """Judit Hybrid metrics 6 | """ 7 | 8 | import pandas 9 | 10 | from .metricset import MetricSet, Metric 11 | 12 | __all__ = ["Judit_Hybrid_Metrics"] 13 | 14 | 15 | class Judit_Hybrid_Metrics(MetricSet): 16 | """Judit Multiplicative MPI+OpenMP Metrics. 17 | """ 18 | 19 | _metric_list = [ 20 | Metric("Global Efficiency", 0), 21 | Metric("Hybrid Parallel Efficiency", 1), 22 | Metric("MPI Parallel Efficiency", 2), 23 | Metric("MPI Load Balance", 3), 24 | Metric("MPI Communication Efficiency", 3), 25 | Metric("OpenMP Parallel Efficiency", 2), 26 | Metric("OpenMP Communication Efficiency", 3), 27 | Metric("OpenMP Load Balance", 3), 28 | Metric("Hybrid Communication Efficiency", 2), 29 | Metric("Hybrid Load Balance", 2), 30 | Metric("Computational Scaling", 1), 31 | Metric("Instruction Scaling", 2), 32 | Metric("IPC Scaling", 2, "IPC Scaling"), 33 | ] 34 | 35 | _programming_model = "MPI + OpenMP" 36 | 37 | _default_metric_key = "Number of Processes" 38 | _default_group_key = "Threads per Process" 39 | 40 | def _calculate_metrics(self, ref_key=None, sort_keys=True): 41 | if not ref_key: 42 | ref_key = min(self._stats_dict.keys()) 43 | 44 | metrics_by_key = {} 45 | 46 | if sort_keys: 47 | keys = sorted(self._stats_dict.keys()) 48 | else: 49 | key = self._stats_dict.keys() 50 | 51 | for key in keys: 52 | metadata = self._stats_dict[key].metadata 53 | stats = self._stats_dict[key].statistics 54 | metrics = self._create_subdataframe(metadata, key) 55 | 56 | try: 57 | metrics["MPI Communication Efficiency"] = ( 58 | stats["Total Non-MPI Runtime"].loc[:, 1].max() 59 | / stats["Total Runtime"].max() 60 | ) 61 | 62 | metrics["MPI Load Balance"] = ( 63 | stats["Total Non-MPI Runtime"].loc[:, 1].mean() 64 | / stats["Total Non-MPI Runtime"].loc[:, 1].max() 65 | ) 66 | 67 | metrics["MPI Parallel Efficiency"] = ( 68 | stats["Total Non-MPI Runtime"].loc[:, 1].mean() 69 | ) / stats["Total Runtime"].max() 70 | 71 | metrics["Hybrid Parallel Efficiency"] = ( 72 | stats["Total Useful Computation"].mean() 73 | / stats["Total Runtime"].max() # avg all threads to include Amdahl 74 | ) 75 | 76 | metrics["Hybrid Load Balance"] = ( 77 | stats["Total Useful Computation"].mean() 78 | / stats["Total Useful Computation"].max() 79 | ) 80 | 81 | metrics["Hybrid Communication Efficiency"] = ( 82 | stats["Total Useful Computation"].max() 83 | / stats["Total Runtime"].max() 84 | ) 85 | 86 | metrics["OpenMP Parallel Efficiency"] = ( 87 | metrics["Hybrid Parallel Efficiency"] 88 | / metrics["MPI Parallel Efficiency"] 89 | ) 90 | 91 | metrics["OpenMP Load Balance"] = ( 92 | metrics["Hybrid Load Balance"] / metrics["MPI Load Balance"] 93 | ) 94 | 95 | metrics["OpenMP Communication Efficiency"] = ( 96 | metrics["Hybrid Communication Efficiency"] 97 | / metrics["MPI Communication Efficiency"] 98 | ) 99 | 100 | metrics["IPC Scaling"] = ( 101 | stats["Useful Instructions"].sum() / stats["Useful Cycles"].sum() 102 | ) / ( 103 | self._stats_dict[ref_key].statistics["Useful Instructions"].sum() 104 | / self._stats_dict[ref_key].statistics["Useful Cycles"].sum() 105 | ) 106 | 107 | metrics["Instruction Scaling"] = ( 108 | self._stats_dict[ref_key].statistics["Useful Instructions"].sum() 109 | / stats["Useful Instructions"].sum() 110 | ) 111 | 112 | metrics["Frequency Scaling"] = ( 113 | stats["Useful Cycles"].sum() 114 | / stats["Total Useful Computation"].sum() 115 | ) / ( 116 | self._stats_dict[ref_key].statistics["Useful Cycles"].sum() 117 | / self._stats_dict[ref_key] 118 | .statistics["Total Useful Computation"] 119 | .sum() 120 | ) 121 | 122 | metrics["Computational Scaling"] = ( 123 | self._stats_dict[ref_key] 124 | .statistics["Total Useful Computation"] 125 | .sum() 126 | / stats["Total Useful Computation"].sum() 127 | ) 128 | 129 | metrics["Global Efficiency"] = ( 130 | metrics["Computational Scaling"] 131 | * metrics["Hybrid Parallel Efficiency"] 132 | ) 133 | 134 | metrics["Speedup"] = ( 135 | self._stats_dict[ref_key].statistics["Total Runtime"].max() 136 | / stats["Total Runtime"].max() 137 | ) 138 | 139 | metrics["Runtime"] = stats["Total Runtime"].max() 140 | 141 | except KeyError as err: 142 | raise ValueError( 143 | "No '{}' statistic. (Wrong analysis type?)" "".format(err.args[0]) 144 | ) 145 | 146 | metrics_by_key[key] = metrics 147 | 148 | self._metric_data = pandas.concat(metrics_by_key.values()) 149 | -------------------------------------------------------------------------------- /pypop/metrics/multiplot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """Shared plot routines for multiple Metric Sets 6 | """ 7 | 8 | import numpy 9 | import pandas 10 | import matplotlib as mpl 11 | import matplotlib.pyplot as plt 12 | import matplotlib.ticker as mtick 13 | 14 | from .._plotsettings import pypop_mpl_params, figparams 15 | 16 | from .metricset import MetricSet 17 | from ..traceset import TraceSet 18 | 19 | 20 | def _get_dataframe(obj): 21 | if isinstance(obj, (TraceSet)): 22 | return obj.statistics 23 | 24 | if isinstance(obj, MetricSet): 25 | return obj.metric_data 26 | 27 | return obj 28 | 29 | 30 | def plot_scalings( 31 | scalings, 32 | x_key="Number of Processes", 33 | y_key="Speedup", 34 | x_label=None, 35 | y_label=None, 36 | series_labels=None, 37 | title=None, 38 | ): 39 | """Plot scaling graph with region shading. 40 | 41 | Plots scaling data from pandas dataframe(s). The 0-80% and 80-100% scaling 42 | regions are shaded for visual identification. Multiple scaling lines may be 43 | plotted by passing a dict of dataframes. 44 | 45 | Parameters 46 | ---------- 47 | scalings_dict: (list of) MetricSet, TraceSet, or pandas.DataFrame 48 | (List of) Pandas DataFrames containing scaling data. 49 | 50 | x_key: scalar 51 | Key of Dataframe column to use as x-axis. 52 | 53 | y_key: scalar 54 | key of Dataframe column to use as y-axis. 55 | 56 | x_label: str or None 57 | Label to be used for x-axis and data series. Defaults to `x_key`. 58 | 59 | y_label: str or None 60 | Label to be used for y-axis and data series. Defaults to `y_key`. 61 | 62 | series_labels: list of str or None 63 | Lables to be used for series, defaults to `y_label` for a single series, 64 | enumeration for multiple series. 65 | 66 | title: str or None 67 | Optional title for plot. 68 | 69 | Returns 70 | ------- 71 | figure: matplotlib.figure.Figure 72 | Figure containing complete scaling plot. 73 | """ 74 | with mpl.rc_context(pypop_mpl_params): 75 | return _plot_scalings_multiple( 76 | scalings, x_key, y_key, x_label, y_label, series_labels, title 77 | ) 78 | 79 | 80 | def _plot_scalings_multiple( 81 | scalings, x_key, y_key, x_label, y_label, series_labels, title, 82 | ): 83 | 84 | # Wrap single instance in list if needed 85 | if isinstance(scalings, (pandas.DataFrame, TraceSet, MetricSet)): 86 | scalings = [scalings] 87 | 88 | # Make sure we are dealing with all pd.Dataframe 89 | scalings = [_get_dataframe(dataset) for dataset in scalings] 90 | 91 | # Sort out labels as needed 92 | if x_label is None: 93 | x_label = x_key 94 | 95 | if y_label is None: 96 | y_label = y_key 97 | 98 | if series_labels is None: 99 | if len(scalings) == 1: 100 | series_labels = [y_label] 101 | else: 102 | series_labels = range(len(scalings)) 103 | 104 | # Calculate x and y ranges 105 | cores_min = numpy.nan 106 | cores_max = numpy.nan 107 | y_max = numpy.nan 108 | 109 | for dataset in scalings: 110 | cores_min = numpy.nanmin([cores_min, dataset[x_key].min()]) 111 | cores_max = numpy.nanmax([cores_max, dataset[x_key].max()]) 112 | y_max = numpy.nanmax((y_max, dataset[y_key].max())) 113 | 114 | # And pad appropriately 115 | y_max *= 1.2 116 | x_margin = 0.02 * cores_max 117 | 118 | # Calculate ideal scalings 119 | ideal_scaling_cores = numpy.linspace(cores_min - x_margin, cores_max + x_margin) 120 | ideal_scaling = ideal_scaling_cores / cores_min 121 | ideal_scaling_80pc = 0.2 + 0.8 * ideal_scaling 122 | 123 | # Finally, plot 124 | fig = plt.figure(figsize=figparams["single.figsize"]) 125 | ax = fig.add_axes(figparams["single.axlayout"][0]) 126 | 127 | ax.fill_between( 128 | ideal_scaling_cores, 129 | 0, 130 | ideal_scaling_80pc, 131 | label="80% Scaling", 132 | alpha=0.1, 133 | color="g", 134 | linestyle="-", 135 | ) 136 | ax.fill_between( 137 | ideal_scaling_cores, 138 | ideal_scaling_80pc, 139 | ideal_scaling, 140 | label="Ideal Scaling", 141 | alpha=0.2, 142 | color="g", 143 | linestyle="-", 144 | ) 145 | 146 | for dataset, label in zip(scalings, series_labels): 147 | ax.plot( 148 | dataset[x_key], 149 | dataset[y_key], 150 | label=label, 151 | marker="x", 152 | linestyle="-", 153 | alpha=0.8, 154 | ) 155 | 156 | ax.set_xlim(ideal_scaling_cores.min(), ideal_scaling_cores.max()) 157 | ax.set_ylim(0, y_max) 158 | ax.set_xlabel(x_label) 159 | ax.set_ylabel(y_label) 160 | ax.xaxis.set_major_locator(mtick.FixedLocator(scalings[0][x_key], 6)) 161 | ax.legend() 162 | 163 | if title: 164 | ax.set_title(title) 165 | 166 | return fig 167 | -------------------------------------------------------------------------------- /pypop/mplplotting/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | from .builtin import * 6 | 7 | __all__ = [] 8 | -------------------------------------------------------------------------------- /pypop/mplplotting/mplplotbase.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | from io import BytesIO 6 | 7 | 8 | class MPLPlotBase(object): 9 | def __init__(self): 10 | self._figure = None 11 | 12 | def _build_plot(self): 13 | raise NotImplementedError("MPLPlotBase should not be used directly!") 14 | 15 | @property 16 | def figure(self): 17 | if not self._figure: 18 | self._build_plot() 19 | return self._figure 20 | 21 | def _repr_png_(self): 22 | imgbuffer = BytesIO() 23 | self.figure.savefig(imgbuffer, format="png") 24 | return imgbuffer.getvalue() 25 | 26 | def save_png(self, filename): 27 | self.figure.savefig(filename, format="png") 28 | -------------------------------------------------------------------------------- /pypop/notebook_interface/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | from .wizard import MetricsWizard 6 | 7 | __all__ = ["MetricsWizard"] 8 | -------------------------------------------------------------------------------- /pypop/notebook_interface/bokeh_widget.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from ipywidgets import Output 4 | 5 | from bokeh.io import output_notebook, push_notebook 6 | from bokeh.plotting import show 7 | from bokeh.core.validation import silence 8 | from bokeh.core.validation.warnings import MISSING_RENDERERS 9 | 10 | silence(MISSING_RENDERERS) 11 | 12 | 13 | class BokehWidgetWrapper(Output): 14 | def __init__(self, plot_object): 15 | output_notebook(hide_banner=True) 16 | super().__init__() 17 | self._plot_object = plot_object 18 | self._figure = plot_object.figure 19 | self._plot_object.install_update_callback(self.update) 20 | with self: 21 | self._handle = show(self._figure, notebook_handle=True) 22 | 23 | def update(self): 24 | if self._handle is None: 25 | with self: 26 | self._handle = show(self.figure, notebook_handle=True) 27 | push_notebook(handle=self._handle) 28 | -------------------------------------------------------------------------------- /pypop/notebook_interface/omp_explorer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | import numpy 6 | 7 | from bokeh.plotting import figure 8 | from bokeh.transform import linear_cmap 9 | 10 | from pypop.prv import PRV 11 | 12 | from .plotting import BokehBase 13 | from .palettes import efficiency_red_green 14 | 15 | 16 | class OMPRegionExplorer(BokehBase): 17 | 18 | _omp_tooltip_template = [ 19 | ("Function(s)", "@{Region Function Fingerprint}"), 20 | ("Location(s)", "@{Region Location Fingerprint}"), 21 | ("Load Bal.", "@{Load Balance}{0.00}"), 22 | ("Length", "@{Region Length}{0.000 a}s"), 23 | ("Avg. Comp.", "@{Average Computation Time}{0.000 a}s"), 24 | ("Max. Comp.", "@{Maximum Computation Time}{0.000 a}s"), 25 | ("Sum Comp.", "@{Region Total Computation}{0.000 a}s"), 26 | ] 27 | 28 | def __init__(self, prv: PRV, fontsize=14): 29 | super().__init__() 30 | self._prv = prv 31 | self.fontsize = fontsize 32 | 33 | def _build_plot(self): 34 | 35 | omp_region_stats = self._prv.profile_openmp_regions().copy() 36 | 37 | for tm in [ 38 | "Region Start", 39 | "Region End", 40 | "Region Length", 41 | "Average Computation Time", 42 | "Maximum Computation Time", 43 | "Computation Delay Time", 44 | "Region Total Computation", 45 | "Region Delay Time", 46 | ]: 47 | omp_region_stats[tm] /= 1e9 48 | 49 | # Geometry calculations - start at 48 em width and 2em plus 2em per bar height up 50 | # to a maximum of 15 bars, then start shrinking the bars 51 | pt_to_px = 96 / 72 52 | font_px = self.fontsize * pt_to_px 53 | width = int(48 * font_px) 54 | height = int(32 * font_px) 55 | 56 | self._plot_dims = (width, height) 57 | 58 | self._figure = figure( 59 | plot_width=900, 60 | plot_height=600, 61 | tools="xwheel_zoom,zoom_in,zoom_out,pan,reset,save", 62 | tooltips=self._omp_tooltip_template, 63 | ) 64 | 65 | for rank, rankdata in omp_region_stats.groupby(level="rank"): 66 | self._figure.hbar( 67 | y=rank, 68 | left="Region Start", 69 | right="Region End", 70 | height=0.9, 71 | color=linear_cmap("Load Balance", efficiency_red_green, 0, 1), 72 | source=rankdata, 73 | ) 74 | 75 | self._figure.outline_line_color = None 76 | 77 | for ax in (self._figure.xaxis, self._figure.yaxis): 78 | ax.minor_tick_line_color = None 79 | 80 | n_ranks = len(omp_region_stats.index.unique(level="rank")) 81 | n_rankticks = n_ranks if n_ranks < 10 else 10 82 | rankticks = [int(x) for x in numpy.linspace(1, n_ranks, n_rankticks)] 83 | 84 | self._figure.yaxis.ticker = rankticks 85 | 86 | self._figure.ygrid.visible = False 87 | self._figure.yaxis.major_tick_line_color = None 88 | self._figure.yaxis.axis_line_color = None 89 | 90 | self._figure.xaxis.axis_label = "Time (s)" 91 | self._figure.xaxis.axis_label_text_font_size = "16pt" 92 | self._figure.xaxis.major_label_text_font_size = "16pt" 93 | self._figure.yaxis.axis_label = "Processes" 94 | self._figure.yaxis.axis_label_text_font_size = "16pt" 95 | self._figure.yaxis.major_label_text_font_size = "16pt" 96 | 97 | self.update() 98 | -------------------------------------------------------------------------------- /pypop/notebook_interface/palettes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | from bokeh.palettes import diverging_palette, all_palettes 6 | 7 | __all__ = ["efficiency_red_green", "inefficiency_green_red"] 8 | 9 | k_breakover_threshold = 0.8 10 | 11 | 12 | def build_efficiency_red_green(): 13 | 14 | return diverging_palette( 15 | all_palettes["Reds"][256], 16 | all_palettes["Greens"][256], 17 | 256, 18 | k_breakover_threshold, 19 | ) 20 | 21 | 22 | def build_inefficiency_green_red(): 23 | 24 | return diverging_palette( 25 | all_palettes["Greens"][256], 26 | all_palettes["Reds"][256], 27 | 256, 28 | 1 - k_breakover_threshold, 29 | ) 30 | 31 | 32 | efficiency_red_green = build_efficiency_red_green() 33 | inefficiency_green_red = build_inefficiency_green_red() 34 | -------------------------------------------------------------------------------- /pypop/notebook_interface/reporting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | from string import Formatter 6 | 7 | from ipywidgets import Button, Text, VBox 8 | 9 | from ipyfilechooser import FileChooser 10 | 11 | import nbformat 12 | 13 | latest_nbformat = getattr(nbformat, "v{}".format(nbformat.current_nbformat)) 14 | new_nb = latest_nbformat.new_notebook 15 | code_cell = latest_nbformat.new_code_cell 16 | md_cell = latest_nbformat.new_markdown_cell 17 | 18 | 19 | def quiet_code_cell(*args, **kwargs): 20 | 21 | quiet_cell = {"hide_output": True} 22 | 23 | if "metadata" in kwargs: 24 | kwargs["metadata"].update(quiet_cell) 25 | else: 26 | kwargs["metadata"] = quiet_cell 27 | 28 | return code_cell(*args, **kwargs) 29 | 30 | 31 | class ReportGenerator(VBox): 32 | 33 | _text_header = """ 34 | """ 35 | 36 | _import_header = """ 37 | from pypop.traceset import TraceSet 38 | from pypop.metrics import {metric_class} 39 | from pypop.notebook_interface.plotting import MetricTable, ScalingPlot 40 | """ 41 | 42 | _file_load = """ 43 | files_list = [{trace_file_list}] 44 | statistics = TraceSet(files_list) 45 | metrics = {metric_class}(statistics) 46 | """ 47 | 48 | _appl_info = """ 49 | # Application Information 50 | * Application Name: 51 | * Applicant: 52 | * Language: 53 | * Programming Model: {programming_model} 54 | * Application description: 55 | * Run Parameters: 56 | * Machine Environment: 57 | """ 58 | 59 | _scaling_header = """ 60 | # Application Scaling 61 | """ 62 | 63 | _scaling_plot = """ 64 | scaling_plot = ScalingPlot(metrics) 65 | display(scaling_plot) 66 | """ 67 | 68 | _scaling_discussion = """ 69 | _Discussion of scaling results._ 70 | """ 71 | 72 | _metrics_header = """ 73 | # Application Efficiency Metrics 74 | """ 75 | 76 | _metrics_table = """ 77 | metric_table = MetricTable(metrics) 78 | display(metric_table) 79 | """ 80 | 81 | _metrics_discussion = """ 82 | _Discussion of metrics._ 83 | """ 84 | 85 | _conclusion_cell = """ 86 | # Conclusions 87 | 88 | _Key findings and recommendations._ 89 | """ 90 | 91 | _notebook_layout = [ 92 | (_text_header, md_cell), 93 | (_import_header, quiet_code_cell), 94 | (_file_load, quiet_code_cell), 95 | (_appl_info, md_cell), 96 | (_scaling_header, md_cell), 97 | (_scaling_plot, code_cell), 98 | (_scaling_discussion, md_cell), 99 | (_metrics_header, md_cell), 100 | (_metrics_table, code_cell), 101 | (_metrics_discussion, md_cell), 102 | (_conclusion_cell, md_cell), 103 | ] 104 | 105 | def __init__(self, analysis_state, **kwargs): 106 | 107 | self._analysis_state = analysis_state 108 | 109 | self._filechooser = FileChooser( 110 | title="Report Notebook Filename", 111 | filename="report.ipynb", 112 | select_default=True, 113 | ) 114 | 115 | self._codename = Text(description="Application:", placeholder="") 116 | self._author = Text(description="Report Author:", placeholder="") 117 | self._contributors = Text(description="Contributors:", placeholder="") 118 | self._report_id = Text(description="Report ID:", value="DEMO_001") 119 | 120 | self._button_generate_report = Button( 121 | description="Generate", button_style="success" 122 | ) 123 | self._button_generate_report.on_click(self._generate_report) 124 | 125 | super().__init__( 126 | children=[ 127 | self._filechooser, 128 | self._codename, 129 | self._author, 130 | self._contributors, 131 | self._report_id, 132 | self._button_generate_report, 133 | ], 134 | **kwargs 135 | ) 136 | 137 | def _generate_subst_dict(self): 138 | 139 | keys = set() 140 | for cell_text, _ in self._notebook_layout: 141 | keys.update([k[1] for k in Formatter().parse(cell_text) if k[1] is not None]) 142 | 143 | # Debug condition 144 | # if "" in keys or any(x.isnumeric() for x in keys): 145 | # raise ValueError("") 146 | 147 | return {key: getattr(self, "_get_{}".format(key))() for key in keys} 148 | 149 | def _generate_report_nb_metadata(self): 150 | 151 | return { 152 | "reprefnum": self._report_id.value, 153 | "title": "Performance Report --- {}".format(self._codename.value), 154 | "author": self._author.value, 155 | "contributors": self._contributors.value, 156 | } 157 | 158 | def _generate_report(self, callback_reference=None): 159 | 160 | subst_dict = self._generate_subst_dict() 161 | 162 | report_nb = new_nb( 163 | metadata={ 164 | "pop_metadata": self._generate_report_nb_metadata(), 165 | "celltoolbar": "Edit Metadata", 166 | } 167 | ) 168 | 169 | for cell_text, cell_ctr in self._notebook_layout: 170 | report_nb.cells.append(cell_ctr(cell_text.strip().format(**subst_dict))) 171 | 172 | nbformat.write(report_nb, self._filechooser.selected) 173 | 174 | def _get_metric_class(self): 175 | return self._analysis_state["metrics_object"].__class__.__name__ 176 | 177 | def _get_trace_file_list(self): 178 | return ", ".join('"{}"'.format(x) for x in self._analysis_state["trace_files"]) 179 | 180 | def _get_programming_model(self): 181 | return self._analysis_state["metrics_object"]._programming_model 182 | -------------------------------------------------------------------------------- /pypop/notebook_interface/tqdm_widget.py: -------------------------------------------------------------------------------- 1 | 2 | from ipywidgets import HBox, HTML, Output, FloatProgress 3 | from tqdm.notebook import tqdm_notebook 4 | 5 | class tqdm_notebook_noauto(tqdm_notebook): 6 | 7 | @staticmethod 8 | def status_printer(_, total=None, desc=None, ncols=None): 9 | """ 10 | Manage the printing of an IPython/Jupyter Notebook progress bar widget. 11 | """ 12 | # Fallback to text bar if there's no total 13 | # DEPRECATED: replaced with an 'info' style bar 14 | # if not total: 15 | # return super(tqdm_notebook, tqdm_notebook).status_printer(file) 16 | 17 | # fp = file 18 | 19 | if total: 20 | pbar = FloatProgress(min=0, max=total) 21 | else: # No total? Show info style bar with no progress tqdm status 22 | pbar = FloatProgress(min=0, max=1) 23 | pbar.value = 1 24 | pbar.bar_style = 'info' 25 | if ncols is None: 26 | pbar.layout.width = "20px" 27 | 28 | ltext = HTML() 29 | rtext = HTML() 30 | if desc: 31 | ltext.value = desc 32 | container = HBox(children=[ltext, pbar, rtext]) 33 | # Prepare layout 34 | if ncols is not None: # use default style of ipywidgets 35 | # ncols could be 100, "100px", "100%" 36 | ncols = str(ncols) # ipywidgets only accepts string 37 | try: 38 | if int(ncols) > 0: # isnumeric and positive 39 | ncols += 'px' 40 | except ValueError: 41 | pass 42 | pbar.layout.flex = '2' 43 | container.layout.width = ncols 44 | container.layout.display = 'inline-flex' 45 | container.layout.flex_flow = 'row wrap' 46 | 47 | return container 48 | -------------------------------------------------------------------------------- /pypop/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/enb python3 2 | 3 | """A minimal interface for managing IPython servers 4 | """ 5 | 6 | import os 7 | import subprocess 8 | from time import sleep 9 | 10 | from notebook.notebookapp import list_running_servers 11 | 12 | 13 | def get_cache_pid(): 14 | 15 | try: 16 | with open(os.path.expanduser("~/.cache/pypopserver.pid")) as fh: 17 | pid = int(fh.readline().strip()) 18 | return pid 19 | except (FileNotFoundError, ValueError): 20 | return None 21 | 22 | 23 | def get_notebook_server_instance(try_use_existing=False): 24 | """Create a notebook server instance to use. Optionally attempting to re-use existing 25 | instances. 26 | """ 27 | 28 | pid = get_cache_pid() 29 | 30 | servers = list_running_servers() 31 | 32 | # If we already have a server, use that 33 | for server in servers: 34 | if server["pid"] == pid: 35 | return (server, None) 36 | 37 | # Otherwise, if we are allowed, try to piggyback on another session 38 | if try_use_existing and servers: 39 | return (servers[0], None) 40 | 41 | # Fine, I'll make my own server, with blackjack, and userhooks! 42 | try: 43 | server_process = subprocess.Popen(["jupyter", "notebook", "--no-browser"]) 44 | except OSError as err: 45 | raise RuntimeError("Failed to start server: {}".format(err)) 46 | print("Started Jupyter Notebook server pid {}".format(server_process.pid)) 47 | # wait for 1 second for server to come up 48 | sleep(1) 49 | 50 | server = None 51 | for retry in range(5): 52 | try: 53 | server = {s["pid"]: s for s in list_running_servers()}[server_process.pid] 54 | break 55 | except KeyError: 56 | # Sleep for increasing times to give server a chance to come up 57 | sleep(5) 58 | 59 | if server: 60 | return (server, server_process) 61 | 62 | # Don't leave orphans! 63 | server_process.kill() 64 | raise RuntimeError("Failed to acquire server instance after 25s") 65 | 66 | 67 | def construct_nb_url(server_info, nb_path): 68 | 69 | nb_dir = server_info["notebook_dir"] 70 | 71 | if os.path.isabs(nb_dir): 72 | nb_relpath = os.path.relpath(nb_path, nb_dir) 73 | if nb_relpath.startswith(".."): 74 | raise ValueError( 75 | "Requested notebook file path is not in the notebook server " 76 | "directory ({}).".format(nb_path, nb_dir) 77 | ) 78 | else: 79 | nb_relpath = nb_path.lstrip(".").lstrip("/") 80 | 81 | if server_info["token"]: 82 | return "{}notebooks/{}?token={}".format( 83 | server_info["url"], nb_relpath, server_info["token"] 84 | ) 85 | else: 86 | return "{}notebooks/{}".format(server_info["url"], nb_relpath) 87 | -------------------------------------------------------------------------------- /pypop/trace/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from .trace import Trace 4 | from .prvtrace import PRVTrace 5 | 6 | __all__ = ["Trace", "PRVTrace"] 7 | -------------------------------------------------------------------------------- /pypop/trace/tracemetadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from collections import namedtuple 4 | 5 | from warnings import warn 6 | 7 | import numpy 8 | import pandas 9 | 10 | ProcessLayout = namedtuple("ProcessLayout", ["node", "process", "num_threads"]) 11 | 12 | 13 | class CompatibilityApplicationLayout: 14 | def __init__(self, commsize, threads): 15 | 16 | self.commsize = commsize 17 | 18 | thread_modulus = sum(threads) / commsize 19 | 20 | self.rank_threads = [(j, 1 + i // thread_modulus) for i, j in enumerate(threads)] 21 | 22 | 23 | class TraceMetadata: 24 | 25 | _tracesubclasskey = "PyPOPTraceSubclass" 26 | 27 | _datavars = { 28 | _tracesubclasskey: str, 29 | "capture_time": str, 30 | "elapsed_seconds": numpy.float64, 31 | "num_nodes": int, 32 | "cores_per_node": int, 33 | "num_processes": int, 34 | "threads_per_process": int, 35 | # "layout": str, 36 | "tracefile_name": str, 37 | "fingerprint": str, 38 | "tag": str, 39 | } 40 | 41 | _optional = {"tag"} 42 | 43 | _needs_packing = ["layout", "threads_per_process"] 44 | 45 | def __init__(self, parent): 46 | 47 | for attr in self._datavars.keys(): 48 | setattr(self, attr, None) 49 | 50 | if parent is not None: 51 | setattr(self, TraceMetadata._tracesubclasskey, parent.__class__.__name__) 52 | else: 53 | setattr(self, TraceMetadata._tracesubclasskey, None) 54 | 55 | @staticmethod 56 | def unpack_dataframe(dataframe): 57 | 58 | metadata = TraceMetadata(None) 59 | 60 | for var, vartype in metadata._datavars.items(): 61 | try: 62 | packed_data = dataframe[var] 63 | except KeyError as err: 64 | if var not in metadata._optional: 65 | warn("Missing metadata entry: {}".format(err)) 66 | setattr(metadata, var, None) 67 | continue 68 | if var in metadata._needs_packing: 69 | unpack = getattr(metadata, "unpack_{}".format(var)) 70 | setattr(metadata, var, unpack(packed_data, vartype)) 71 | else: 72 | setattr(metadata, var, vartype(packed_data[0])) 73 | 74 | return metadata 75 | 76 | def pack_dataframe(self): 77 | 78 | data = {} 79 | 80 | for var, vartype in self._datavars.items(): 81 | if getattr(self, var) is None: 82 | continue 83 | if var in self._needs_packing: 84 | pack = getattr(self, "pack_{}".format(var)) 85 | packed = pack(getattr(self, var), vartype) 86 | assert isinstance(packed, (pandas.Series, pandas.DataFrame)) 87 | data[var] = packed 88 | else: 89 | data[var] = pandas.Series(data=getattr(self, var), dtype=vartype) 90 | 91 | return pandas.DataFrame(data) 92 | 93 | @staticmethod 94 | def pack_layout(layout, dtype): 95 | return pandas.Series( 96 | data="|".join(["{},{},{}".format(*dat) for dat in layout]), dtype=dtype 97 | ) 98 | 99 | @staticmethod 100 | def unpack_layout(packed, vartype): 101 | layoutstring = packed[0] 102 | return [ 103 | ProcessLayout(*(int(x) for x in proc)) for proc in layoutstring.split("|") 104 | ] 105 | 106 | @staticmethod 107 | def pack_threads_per_process(data, dtype): 108 | data = "|".join(str(x) for x in data) 109 | return pandas.Series(data=data, dtype=str) 110 | 111 | @staticmethod 112 | def unpack_threads_per_process(data, dtype): 113 | try: 114 | return [dtype(x) for x in data[0].split("|")] 115 | except AttributeError: 116 | return list(data) 117 | 118 | def __bool__(self): 119 | return self.fingerprint is not None 120 | 121 | @property 122 | def captured(self): 123 | warn("captured is deprecated. Please use capture_time.", FutureWarning) 124 | return self.capture_time 125 | 126 | @property 127 | def ns_elapsed(self): 128 | warn("ns_elapsed is deprecated. Please use elapsed_seconds.", FutureWarning) 129 | return self.elapsed_seconds 130 | 131 | @property 132 | def nodes(self): 133 | warn("nodes is deprecated. Please use num_nodes.", FutureWarning) 134 | return self.num_nodes 135 | 136 | @property 137 | def procs_per_node(self): 138 | warn("procs_per_node is deprecated. Please use cores_per_node.", FutureWarning) 139 | return self.cores_per_node 140 | 141 | @property 142 | def application_layout(self): 143 | warn( 144 | "application_layout is deprecated.\n" 145 | "Please use num_processes for application_layout.commsize, and " 146 | "threads_per_process for application_layout.rank_threads", 147 | FutureWarning, 148 | ) 149 | 150 | return CompatibilityApplicationLayout( 151 | self.num_processes, self.threads_per_process 152 | ) 153 | 154 | @property 155 | def trace_subclass_name(self): 156 | return getattr(self, TraceMetadata._tracesubclasskey) 157 | 158 | def __repr__(self): 159 | 160 | return "\n".join( 161 | [ 162 | "{}: {}".format(key, str(getattr(self, key))) 163 | for key in self._datavars.keys() 164 | ] 165 | ) 166 | -------------------------------------------------------------------------------- /pypop/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | -------------------------------------------------------------------------------- /pypop/utils/asserters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | __all__ = ["SimpleAsserter", "ListofStrAsserter"] 6 | 7 | 8 | class BaseAsserter: 9 | """Parent class for type asserter objects 10 | 11 | Asserter classes provide a callable which can be used to assert the validity of an 12 | object. 13 | 14 | Subclasses should implement a check_type function returning true/false as needed. 15 | """ 16 | 17 | _expected = None 18 | 19 | def __call__(self, testobj): 20 | if not self.check_type(testobj): 21 | raise TypeError( 22 | "Expected {} not {}".format(self._expected, testobj.__class__.__name__) 23 | ) 24 | return True 25 | 26 | def check_type(self, testobj): 27 | """Return true if object is valid 28 | 29 | Parameters 30 | ---------- 31 | testobj: object 32 | Object to be validated. 33 | """ 34 | raise NotImplementedError() 35 | 36 | 37 | class SimpleAsserter(BaseAsserter): 38 | """Assert that an object is of a given type 39 | 40 | Provides a callable object which asserts the provided object is of the required type. 41 | """ 42 | 43 | def __init__(self, assert_class): 44 | """Define class to be validated 45 | 46 | Parameters 47 | ---------- 48 | assert_class: class 49 | Reference class to be compared to. 50 | """ 51 | self._assert_cls = assert_class 52 | self._expected = assert_class.__name__ 53 | 54 | def check_type(self, testobj): 55 | if isinstance(testobj, self._assert_cls): 56 | return True 57 | 58 | 59 | class ListofStrAsserter(BaseAsserter): 60 | """Assert that an object is a list of strings 61 | 62 | Provides a callable object which asserts a provided object is a list of strings. 63 | """ 64 | 65 | _expected = "list of str" 66 | 67 | def check_type(self, testobj): 68 | if isinstance(testobj, list) and all(isinstance(x, str) for x in testobj): 69 | return True 70 | -------------------------------------------------------------------------------- /pypop/utils/exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | class WrongLoaderError(RuntimeError): 5 | pass 6 | 7 | 8 | class UnknownLoaderError(RuntimeError): 9 | pass 10 | 11 | 12 | class ExtraePRVNoOnOffEventsError(RuntimeError): 13 | pass 14 | 15 | 16 | class NoWebDriverError(RuntimeError): 17 | pass 18 | -------------------------------------------------------------------------------- /pypop/utils/io.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """\ 6 | Miscellaneous utility routines 7 | ------------------------------ 8 | 9 | General utility and helper routines not specific to a particular tool. 10 | """ 11 | 12 | from hashlib import md5 13 | import gzip 14 | 15 | 16 | def zipopen(path, mode="rt"): 17 | 18 | # Default modes of open() and gzip.open() are not the same ('t' and 'b' respectively) 19 | # so ensure a consistent default of text mode 20 | if not any(x in mode for x in ("t", "b")): 21 | mode = "{}t".format(mode) 22 | 23 | try: 24 | if gzip.open(path, mode=mode).read(1): 25 | return gzip.open(path, mode=mode) 26 | except OSError: 27 | return open(path, mode=mode) 28 | 29 | 30 | def chunked_md5sum(filename, blocksize=8388608): 31 | """ Chunked md5 checksum using hashlib 32 | 33 | Parameters 34 | ---------- 35 | filename: str 36 | File to calculate checksum. 37 | blocksize: int 38 | Blocksize in bytes to iterate with, (default 8MB). 39 | 40 | Returns 41 | ------- 42 | hexdigest: str 43 | String containing hexadecimal representation of the checksum. 44 | """ 45 | hashobj = md5() 46 | with open(filename, "rb") as f: 47 | for block in iter(lambda: f.read(blocksize), b""): 48 | hashobj.update(block) 49 | return hashobj.hexdigest() 50 | 51 | 52 | def return_first_arg(arg, *args, **kwargs): 53 | """Return first argument and disregard remaining, intended for silently replacing 54 | tqdm when unavailable 55 | 56 | Parameters 57 | ---------- 58 | arg: * 59 | Argument to be returned 60 | 61 | *args: Will be ignored 62 | 63 | **kwargs: Will be ignored 64 | 65 | Returns 66 | ------- 67 | arg: 68 | First argument passed to function 69 | """ 70 | 71 | return arg 72 | -------------------------------------------------------------------------------- /pypop/utils/pandas.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pandas 4 | 5 | 6 | class HDFStoreContext: 7 | """A simple contextmanager that manages opening and closing of a pandas HDFStore 8 | object including closing correctly in the case that exceptions are thrown during 9 | read/write. 10 | """ 11 | 12 | def __init__(self, filename, **kwargs): 13 | self._filename = filename 14 | self._kwargs = kwargs 15 | 16 | def __enter__(self): 17 | try: 18 | self._hdfstore = pandas.HDFStore(self._filename, **self._kwargs) 19 | except OSError as err: 20 | # Try to interpret and reraise the OSError as something actionable to the 21 | # despatcher 22 | if "does not exist" in str(err): 23 | raise FileNotFoundError('"{}" does not exist'.format(self._filename)) 24 | if "HDF5 error" in str(err) or "not a regular file" in str(err): 25 | raise ValueError('"{}" is not a valid HDF5 file'.format(self._filename)) 26 | # Reraise anything we don't catch explicitly 27 | raise 28 | return self._hdfstore 29 | 30 | def __exit__(self, exc_type, exc_val, exc_tb): 31 | self._hdfstore.close() 32 | -------------------------------------------------------------------------------- /pypop/utils/plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | import numpy 6 | from pkg_resources import resource_filename 7 | 8 | __all__ = ["approx_string_em_width", "get_pop_logo_data"] 9 | 10 | _POP_LOGO_PNG = "pop_logo.npy" 11 | 12 | 13 | def approx_string_em_width(string): 14 | """Give the approximate string width in 'em' 15 | 16 | Basically assume 'w' and 'm' have em width and everything else has 17 | a width of 0.6*em 18 | """ 19 | 20 | return 0.6 * len(string) + 0.4 * sum(string.count(x) for x in ["w", "m", "~"]) 21 | 22 | 23 | def get_pop_logo_data(): 24 | """Return numpy array containing POP logo pixels 25 | """ 26 | 27 | return numpy.load(resource_filename(__name__, _POP_LOGO_PNG)) 28 | -------------------------------------------------------------------------------- /pypop/utils/pop_logo.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numericalalgorithmsgroup/pypop/830913dd1ad4c7e854f408c991f4171bdc69b18b/pypop/utils/pop_logo.npy -------------------------------------------------------------------------------- /pypop/utils/tex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | """CLI Analysis scripts""" 6 | 7 | import re 8 | 9 | 10 | def tex_escape(text): 11 | """ 12 | :param text: a plain text message 13 | :return: the message escaped to appear correctly in LaTeX 14 | """ 15 | conv = { 16 | "&": r"\&", 17 | "%": r"\%", 18 | "$": r"\$", 19 | "#": r"\#", 20 | "_": r"\_", 21 | "{": r"\{", 22 | "}": r"\}", 23 | "~": r"\textasciitilde{}", 24 | "^": r"\^{}", 25 | "\\": r"\textbackslash{}", 26 | "<": r"\textless{}", 27 | ">": r"\textgreater{}", 28 | } 29 | regex = re.compile( 30 | "|".join( 31 | re.escape(str(key)) 32 | for key in sorted(conv.keys(), key=lambda item: -len(item)) 33 | ) 34 | ) 35 | return regex.sub(lambda match: conv[match.group()], text) 36 | -------------------------------------------------------------------------------- /pypop/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from warnings import warn 4 | from pkg_resources import resource_filename 5 | from subprocess import run, PIPE 6 | 7 | _k_unknown_ver = "Unknown" 8 | 9 | 10 | def get_git_verstring(): 11 | git_result = run( 12 | ["git", "describe", "--tags", "--dirty", "--long"], stdout=PIPE, stderr=PIPE, 13 | ) 14 | if git_result.returncode != 0: 15 | return _k_unknown_ver 16 | 17 | ver_string = git_result.stdout.decode().strip() 18 | ver_tokens = ver_string.split("-") 19 | 20 | dirty = None 21 | if ver_tokens[-1] == "dirty": 22 | dirty = ver_tokens.pop() 23 | 24 | sha = ver_tokens.pop() 25 | commitcount = ver_tokens.pop() 26 | tag = "-".join(ver_tokens) 27 | 28 | if commitcount == "0": 29 | return "-".join([tag, dirty]) if dirty else tag 30 | 31 | return ver_string 32 | 33 | 34 | def get_version(): 35 | 36 | gitver = get_git_verstring() 37 | if gitver != _k_unknown_ver: 38 | return gitver 39 | 40 | try: 41 | with open(resource_filename(__name__, "version"), "rt") as fh: 42 | fileversion = fh.readlines()[0].strip() 43 | return fileversion 44 | except FileNotFoundError: 45 | pass 46 | 47 | return _k_unknown_ver 48 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore:.*imp module.*:DeprecationWarning:IPython 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause-Clear 3 | # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved. 4 | 5 | from setuptools import setup, find_packages 6 | from versionate import versionate 7 | 8 | with open("README.md", "r") as fh: 9 | long_desc = fh.read() 10 | 11 | setup( 12 | name="NAG-PyPOP", 13 | version=versionate(), 14 | url="https://github.com/numericalalgorithmsgroup/pypop.git", 15 | author="Numerical Algorithms Group", 16 | author_email="phil.tooley@nag.co.uk", 17 | description="Python notebook support for POP metrics and reports", 18 | long_description=long_desc, 19 | long_description_content_type="text/markdown", 20 | classifiers=[ 21 | "Programming Language :: Python :: 3", 22 | "License :: OSI Approved :: BSD License", 23 | "Operating System :: OS Independent", 24 | ], 25 | packages=find_packages(), 26 | python_requires=">=3.6", 27 | install_requires=[ 28 | "numpy", 29 | "matplotlib >= 2", 30 | "pandas >= 0.24", 31 | "jupyter", 32 | "bokeh >= 1", 33 | "tables", 34 | "tqdm", 35 | "ipyfilechooser", 36 | "selenium", 37 | "nbformat", 38 | ], 39 | zip_safe=True, 40 | include_package_data=True, 41 | entry_points={ 42 | "console_scripts": [ 43 | "pypop-mpi-metrics = pypop.cli.cli:mpi_cli_metrics", 44 | "pypop-hybrid-metrics = pypop.cli.cli:hybrid_cli_metrics", 45 | "pypop-openmp-metrics = pypop.cli.cli:openmp_cli_metrics", 46 | "pypop-preprocess = pypop.cli.cli:preprocess_traces", 47 | "pypop-idealise-prv = pypop.cli.cli:dimemas_idealise_cli", 48 | "pypop-copy-examples = pypop.cli.cli:copy_examples", 49 | "pypop-gui = pypop.cli.cli:pypop_gui", 50 | ] 51 | }, 52 | ) 53 | -------------------------------------------------------------------------------- /tests/integration/test_mpi_epoch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pytest 4 | 5 | import os 6 | from collections import namedtuple 7 | 8 | import pandas 9 | import numpy 10 | 11 | import pypop.examples 12 | from pypop.extrae import is_extrae_tracefile 13 | from pypop.traceset import TraceSet 14 | from pypop.metrics import MPI_Metrics 15 | from pypop.trace import PRVTrace 16 | 17 | MetricData = namedtuple("MetricData", ["testfiles", "metric_data"]) 18 | 19 | 20 | @pytest.fixture(scope="class") 21 | def testdata(): 22 | extrae_mpi_tracedir = os.path.join( 23 | pypop.examples.examples_directory(), "mpi", "epoch_example_traces" 24 | ) 25 | 26 | extrae_mpi_traces = [ 27 | os.path.join(extrae_mpi_tracedir, f) 28 | for f in os.listdir(extrae_mpi_tracedir) 29 | if is_extrae_tracefile(f) 30 | ] 31 | 32 | metrics = pandas.read_csv(os.path.join(extrae_mpi_tracedir, "precomputed.csv")) 33 | 34 | return MetricData(testfiles=extrae_mpi_traces, metric_data=metrics) 35 | 36 | 37 | class TestMPI: 38 | def test_mpi_trace_analysis(self, testdata): 39 | 40 | statistics = TraceSet(testdata.testfiles, force_recalculation=True) 41 | 42 | metrics = MPI_Metrics(statistics.by_commsize()) 43 | 44 | for metric in metrics.metrics: 45 | print(metric.key) 46 | assert numpy.all( 47 | numpy.isclose( 48 | metrics.metric_data[metric.key].values.astype(numpy.float64), 49 | testdata.metric_data[metric.key].values.astype(numpy.float64), 50 | rtol=0, 51 | atol=0.01, 52 | ) 53 | ) 54 | -------------------------------------------------------------------------------- /tests/unit/test_traceset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import pypop.traceset 5 | 6 | -------------------------------------------------------------------------------- /util/publish_doc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ( 4 | scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | basedir="$(readlink -f ${scriptdir}/..)" 6 | docdir="${basedir}/doc" 7 | htmldir="${docdir}/build/html/" 8 | ghpdir="${basedir}/gh-pages/" 9 | 10 | cd $docdir 11 | make clean; make html 12 | 13 | cd $ghpdir 14 | rsync -a --delete ${htmldir} ${ghpdir} 15 | git init 16 | git checkout -b gh-pages 17 | git remote add upstream git@github.com:numericalalgorithmsgroup/pypop.git 18 | git add -A 19 | git commit -m "docs build @ $(date)" 20 | git push --force upstream gh-pages 21 | ) 22 | -------------------------------------------------------------------------------- /versionate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from sys import hexversion 4 | from warnings import warn 5 | from subprocess import check_output, CalledProcessError, PIPE 6 | 7 | _k_unknown_ver = "Unknown" 8 | 9 | 10 | def get_git_verstring(silent=True): 11 | try: 12 | git_result = check_output( 13 | ['git', 'describe', '--tags', '--dirty', '--long'], 14 | stderr=PIPE, 15 | ) 16 | except CalledProcessError as err: 17 | if not silent: 18 | warn("Git failed with error: {}".format(str(err.stderr))) 19 | return _k_unknown_ver 20 | 21 | ver_string = git_result.decode().strip() 22 | ver_tokens = ver_string.split('-') 23 | 24 | dirty = None 25 | if ver_tokens[-1] == "dirty": 26 | dirty = ver_tokens.pop() 27 | 28 | sha = ver_tokens.pop() 29 | commitcount = ver_tokens.pop() 30 | tag = '-'.join(ver_tokens) 31 | 32 | if commitcount == '0': 33 | return "+".join([tag, dirty]) if dirty else tag 34 | 35 | localver = "+".join([tag,sha]) 36 | 37 | return ".".join([localver, dirty]) if dirty else localver 38 | 39 | 40 | def versionate(): 41 | 42 | ver_string = get_git_verstring() 43 | 44 | if ver_string == _k_unknown_ver: 45 | try: 46 | with open('pypop/version', 'rt') as fh: 47 | ver_string = fh.readlines()[0].strip() 48 | except: 49 | pass 50 | finally: 51 | return ver_string 52 | 53 | with open('pypop/version', 'wt') as fh: 54 | fh.write(ver_string) 55 | 56 | return ver_string 57 | 58 | 59 | if __name__ == "__main__": 60 | print(get_git_verstring()) 61 | --------------------------------------------------------------------------------