├── .gitignore ├── CHANGELOG ├── docs ├── tacctitle.jpg ├── tacclogonew.jpg ├── Makefile ├── commonmacs.tex ├── presentation.tex ├── acromacs.tex ├── body.tex ├── snippetmacs.tex ├── coursemacs.tex └── listingmacs.tex ├── programs └── example_script.sh ├── spawnrc.example ├── conf ├── conf.paw-col ├── conf.stress.p2p ├── osu_ls6.conf ├── osu.conf ├── osu_rp.conf ├── conf.ERROR-no-queue ├── conf.shell-script ├── conf.paw.frontera.p2p ├── conf.paw.frontera ├── conf.paw.frontera.coll ├── conf.shell-script-multiple ├── conf.paw.rtx.p2p ├── conf.osu.frontera ├── conf.mpistress.frontera ├── conf.paw.frontera-daos └── conf.paw.frontera.all ├── compare.sh ├── TODO ├── Makefile ├── spawn.py ├── readme.md └── jobsuite.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | __pycache__ 3 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | See the Readme file. 2 | -------------------------------------------------------------------------------- /docs/tacctitle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TACC/demonspawn/HEAD/docs/tacctitle.jpg -------------------------------------------------------------------------------- /docs/tacclogonew.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TACC/demonspawn/HEAD/docs/tacclogonew.jpg -------------------------------------------------------------------------------- /programs/example_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## Example shell script 4 | ## 5 | 6 | hostname 7 | 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | info :: 2 | @echo "make presentation" 3 | presentation : 4 | pdflatex presentation.tex 5 | clean : 6 | @/bin/rm -f \ 7 | *~ \#* \ 8 | *.{aux,blg,idx,ilg,log,mtc,maf,mw,nav,out,snm,svn,toc,vrb} 9 | -------------------------------------------------------------------------------- /spawnrc.example: -------------------------------------------------------------------------------- 1 | ## 2 | ## global demonspawn settings for Victor 3 | ## 4 | system frontera 5 | user eijkhout 6 | account A-ccsc 7 | 8 | queue development limit:1 9 | queue rtx limit:4 10 | queue normal limit:10 11 | queue small limit:10 12 | -------------------------------------------------------------------------------- /conf/conf.paw-col: -------------------------------------------------------------------------------- 1 | user eijkhout 2 | modules intel/19.0.5 impi/19.0.7 3 | account A-ccsc 4 | queue normal 5 | scriptdir /work2/00434/eijkhout/demonspawn/paw.scripts-col 6 | outputdir /work2/00434/eijkhout/demonspawn/paw.output-col 7 | nodes 2,5,10,50 8 | ppn 40 9 | suite name:paw-mpi type:mpi dir:/work2/00434/eijkhout/mpi-regression/paw/installation-frontera col_* 10 | -------------------------------------------------------------------------------- /compare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 3 ] ; then 4 | echo "Usage: $0 dirone dirtwo pattern" 5 | echo " where pattern is used to select file names" 6 | exit 1 7 | fi 8 | 9 | dirone=$1 10 | dirtwo=$2 11 | pat=$3 12 | 13 | for t in ` ( cd $dirone ; ls *${pat}* ) ` ; do 14 | r=${t%%.out*} 15 | echo 16 | echo $r 17 | diff paw.output/${r}* paw.output-daos/${r}* 18 | done 19 | -------------------------------------------------------------------------------- /conf/conf.stress.p2p: -------------------------------------------------------------------------------- 1 | system fontera 2 | user eijkhout 3 | modules intel/18.0.2 impi/18.0.2 4 | account A-ccsc 5 | queue normal 6 | rootdir /work2/00434/eijkhout/demonspawn 7 | stressdir /work2/00434/eijkhout/parallel-programming-private/mpi-stresstest 8 | scriptdir %[rootdir]/stress.scripts.frontera 9 | outputdir %[rootdir]/stress.output.frontera 10 | nodes 2 11 | ppn 1 12 | suite name:mpi-stress-p2p type:mpi dir:%[stressdir]/intel-18.0.2_impi-18.0.2 tags 13 | -------------------------------------------------------------------------------- /conf/osu_ls6.conf: -------------------------------------------------------------------------------- 1 | jobname osubenchmark 2 | 3 | osudir %[STOCKYARD]/osubenchmark/installation-6.0-%[TACC_SYSTEM]-%[TACC_FAMILY_COMPILER]-%[TACC_FAMILY_MPI]/libexec/osu-micro-benchmarks/mpi/ 4 | 5 | # 6 | # point to point test 7 | # 8 | queue normal 9 | nodes 2 10 | ppn 1 11 | regression line:last field:2 margin:30percent 12 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 13 | 14 | # 15 | # collective test 16 | # 17 | nodes 3,15,50 18 | regression line:last field:2 margin:30percent 19 | suite name:osu-col type:mpi dir:%[osudir]/collective * 20 | -------------------------------------------------------------------------------- /conf/osu.conf: -------------------------------------------------------------------------------- 1 | jobname osubenchmark 2 | 3 | osudir %[STOCKYARD]/osubenchmark/installation-6.0-%[TACC_SYSTEM]-intel-%[TACC_FAMILY_MPI]/libexec/osu-micro-benchmarks/mpi/ 4 | 5 | # 6 | # point to point test 7 | # 8 | queue small 9 | nodes 2 10 | ppn 1 11 | regression line:last field:2 margin:20percent 12 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 13 | 14 | # 15 | # collective test 16 | # 17 | queue normal 18 | nodes 3,10,30,100 19 | ppn 1,20,56 20 | regression line:last field:2 margin:20percent 21 | suite name:osu-col type:mpi dir:%[osudir]/collective * 22 | -------------------------------------------------------------------------------- /conf/osu_rp.conf: -------------------------------------------------------------------------------- 1 | jobname osubenchmark 2 | 3 | osudir %[STOCKYARD]/osubenchmark/installation-6.0-%[TACC_SYSTEM]-%[TACC_FAMILY_COMPILER]-%[TACC_FAMILY_MPI]/libexec/osu-micro-benchmarks/mpi/ 4 | queue RP 5 | env UCX_NET_DEVICES mlx5_1:1 6 | 7 | # 8 | # point to point test 9 | # 10 | nodes 2 11 | ppn 1 12 | regression line:last field:2 margin:20percent 13 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 14 | 15 | # 16 | # collective test 17 | # 18 | nodes 3,10,30,100 19 | regression line:last field:2 margin:20percent 20 | suite name:osu-col type:mpi dir:%[osudir]/collective * 21 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | python3 spawn.py conf/conf.ERROR-no-queue 2 | Open > at <<./None>> new: <> 3 | 4 | This should list the queue: 5 | ################################################################ 6 | Test suite: testsuite 7 | modules: default 8 | nodes/cores/threads: [[1, 1, 0]] 9 | regression: grep:BW field:4 take:avg 10 | suites: [{'name': 'script', 'runner': '', 'dir': '/work2/00434/eijkhout/demonspawn/conf', 'apps': ['example_script.sh']}] 11 | ################################################################ 12 | 13 | shell-script-multiple : better name handling 14 | 15 | -------------------------------------------------------------------------------- /conf/conf.ERROR-no-queue: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running a shell script 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | account A-ccsc 12 | 13 | # 14 | # describe this run 15 | # 16 | jobname script 17 | rootdir %[pwd] 18 | outputdir %[rootdir]/spawn-%[jobname]-%[date] 19 | 20 | # verbose output 21 | trace 1 22 | 23 | # number of nodes 24 | nodes 1 25 | # processes per node 26 | ppn 1 27 | 28 | scriptdir %[pwd]/conf 29 | regression grep:BW field:4 take:avg 30 | suite name:script type:seq dir:%[scriptdir] example_script.sh 31 | 32 | -------------------------------------------------------------------------------- /conf/conf.shell-script: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running a shell script 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | account A-ccsc 12 | queue small limit:1 13 | 14 | # 15 | # describe this run 16 | # 17 | jobname script 18 | rootdir %[pwd] 19 | outputdir %[rootdir]/spawn-%[jobname]-%[date] 20 | 21 | # verbose output 22 | trace 1 23 | 24 | # number of nodes 25 | nodes 1 26 | # processes per node 27 | ppn 1 28 | 29 | scriptdir %[pwd]/programs 30 | regression grep:BW field:4 take:avg 31 | 32 | time 0:5:0 33 | suite name:short type:seq dir:%[scriptdir] example_script.sh 34 | 35 | -------------------------------------------------------------------------------- /conf/conf.paw.frontera.p2p: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running the Carlos Rosales PAW suite 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | modules intel/19.1.1 impi/19.0.7 10 | queue development 11 | sbatch --mail-user=%[user] 12 | sbatch --mail-type=end 13 | 14 | # 15 | # describe this run 16 | # 17 | jobname paw-p2p 18 | 19 | # verbose output 20 | trace 1 21 | 22 | # 23 | # p2p benchmark 24 | # 25 | 26 | # number of nodes 27 | nodes 2 28 | # processes per node 29 | ppn 1 30 | 31 | pawdir /work2/00434/eijkhout/mpi-regression/paw/installation-git-clx 32 | regression grep:BW field:4 take:avg 33 | suite name:paw-mpi-p2p type:mpi dir:%[pawdir] p2p_* 34 | 35 | -------------------------------------------------------------------------------- /conf/conf.paw.frontera: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running Carlos' paw tests 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | modules intel/19.1.1 impi/19.0.7 12 | account A-ccsc 13 | queue normal 14 | 15 | # 16 | # global settings for this run 17 | # 18 | jobname paw 19 | rootdir /work2/00434/eijkhout/demonspawn 20 | pawdir /work2/00434/eijkhout/mpi-regression/paw/installation_git_%[system]_%[modules] 21 | 22 | # 23 | # point-to-point tests 24 | # 25 | nodes 3 26 | ppn 1 27 | suite name:paw-mpi-p2p type:mpi dir:%[pawdir] p2p_* 28 | 29 | # 30 | # collective tests 31 | # 32 | ppn 56 33 | nodes 3,5,10,50,200 34 | suite name:paw-mpi-col type:mpi dir:%[pawdir] col_* 35 | -------------------------------------------------------------------------------- /conf/conf.paw.frontera.coll: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running the Carlos Rosales PAW suite 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | modules intel/19.1.1 impi/19.0.7 10 | ## all the rest in .spawnrc ! 11 | 12 | # 13 | # describe this run 14 | # 15 | rootdir %[pwd] 16 | outputdir %[rootdir]/spawn-%[jobname]-%[date] 17 | 18 | # verbose output 19 | trace 1 20 | # data for regression 21 | regression grep:5.242880E+05 field:2 take:avg 22 | 23 | # 24 | # collective benchmark 25 | # 26 | # number of nodes 27 | nodes 5,10,50,200 28 | # processes per node 29 | ppn 2,20,48 30 | pawdir /work2/00434/eijkhout/mpi-regression/paw/installation_git_%[system]_%[modules] 31 | suite name:paw-mpi-col type:mpi dir:%[pawdir] col_* 32 | -------------------------------------------------------------------------------- /conf/conf.shell-script-multiple: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running a shell script 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | account A-ccsc 12 | queue small 1 13 | 14 | # 15 | # describe this run 16 | # 17 | jobname script 18 | rootdir %[pwd] 19 | outputdir %[rootdir]/spawn-%[jobname]-%[date] 20 | 21 | # verbose output 22 | trace 1 23 | 24 | # number of nodes 25 | nodes 1 26 | # processes per node 27 | ppn 1 28 | 29 | scriptdir %[pwd]/programs 30 | regression grep:BW field:4 take:avg 31 | 32 | # do a short run 33 | time 0:5:0 34 | suite name:short type:seq dir:%[scriptdir] example_script.sh 35 | # do a long run 36 | time 0:15:0 37 | suite name:long type:seq dir:%[scriptdir] example_script.sh 38 | 39 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ################################################################ 2 | #### 3 | #### Makefile for the Demonspawn tool 4 | #### Victor Eijkhout eijkhout@tacc.utexas.edu 5 | #### 2020-2022 6 | #### 7 | ################################################################ 8 | 9 | info :: 10 | 11 | info :: 12 | @echo "make localclean clean totalclean" 13 | .PHONY: localclean clean totalclean 14 | clean :: localclean 15 | @for d in * ; do \ 16 | if [ -d "$$d" ] ; then \ 17 | ( cd "$$d" && make --no-print-directory localclean -f ../Makefile ) \ 18 | ; \ 19 | fi ; \ 20 | done 21 | localclean :: 22 | @/bin/rm -rf *~ slurm*.out *.pyc __pycache__ \ 23 | *.out[0-9]* *.slurm-out 24 | totalclean :: clean 25 | @/bin/rm -rf log-* regress-* regression-* output-* script-* scripts-* \ 26 | logfile-* spawn-* testsuite-* spawn_output* 27 | -------------------------------------------------------------------------------- /conf/conf.paw.rtx.p2p: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running the Carlos Rosales PAW suite 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | modules intel/19.1.1 impi/19.0.7 12 | account A-ccsc 13 | queue rtx 14 | 15 | # 16 | # describe this run 17 | # 18 | rootdir /work2/00434/eijkhout/demonspawn 19 | # slurm scripts 20 | scriptdir %[rootdir]/paw.scripts.%[system]-rtx.%[date] 21 | # all output 22 | outputdir %[rootdir]/paw.output.%[system]-rtx.%[date] 23 | # number of nodes 24 | nodes 2 25 | # processes per node 26 | ppn 1 27 | # verbose output 28 | trace 1 29 | # data for regression 30 | regression grep:Maximum field:4 take:avg 31 | 32 | # 33 | # benchmark to run 34 | # 35 | pawdir /work2/00434/eijkhout/mpi-regression/paw/installation_git_%[system]_%[modules] 36 | suite name:paw-mpi-p2p type:mpi dir:%[pawdir] p2p_* 37 | -------------------------------------------------------------------------------- /conf/conf.osu.frontera: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running the OSU benchmarks 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | modules intel/19.1.1 impi/19.0.7 12 | account A-ccsc 13 | queue normal 14 | 15 | # 16 | # describe this run 17 | # 18 | rootdir /work2/00434/eijkhout/demonspawn 19 | benchmark osu 20 | # slurm scripts 21 | scriptdir %[rootdir]/paw.scripts.%[benchmark].%[system].%[date] 22 | # all output 23 | outputdir %[rootdir]/paw.output.%[benchmark].%[system].%[date] 24 | 25 | # verbose output 26 | trace 1 27 | # data for regression 28 | #regression grep:Maximum field:4 take:avg 29 | 30 | # 31 | # collective benchmark 32 | # 33 | nodes 2,5,10 34 | ppn 20 35 | osudir /scratch1/05231/aruhela/libs/mvapich2-2.3.5/buildintel/libexec/osu-micro-benchmarks/mpi/collective 36 | suite name:osu-collective type:mpi dir:%[osudir] * 37 | -------------------------------------------------------------------------------- /conf/conf.mpistress.frontera: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running my own MPI stress test 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | modules intel/19.1.1 impi/19.0.9 12 | account A-ccsc 13 | queue development 14 | 15 | # 16 | # describe this run 17 | # 18 | rootdir /work2/00434/eijkhout/demonspawn 19 | # number of nodes 20 | nodes 2 21 | # processes per node 22 | ppn 1 23 | # verbose output 24 | trace 1 25 | # data for regression 26 | #regression grep:Maximum field:4 take:avg 27 | 28 | # 29 | # where is this set of suites located? 30 | # 31 | stressdir /work2/00434/eijkhout/parallel-programming-private/mpi-stresstest/%[system]_%[modules] 32 | 33 | # 34 | # p2p benchmark 35 | # 36 | suite name:stress-p2p type:mpi dir:%[stressdir] tags 37 | 38 | # 39 | # halfbandwidth 40 | # 41 | nodes 1,4,16 42 | ppn 20,30,40,50 43 | regression grep:Strategy 44 | suite name:stress-hbw type:mpi dir:%[stressdir] halfbandwidth 45 | -------------------------------------------------------------------------------- /conf/conf.paw.frontera-daos: -------------------------------------------------------------------------------- 1 | # 2 | # demonspawn configuration for point-to-point testing 3 | # 4 | system frontera 5 | user eijkhout 6 | modules intel/19.0.5 impi/19.0.7 7 | account A-ccsc 8 | # 9 | ppn 52 10 | # 11 | queue normal 12 | scriptdir /work2/00434/eijkhout/demonspawn/paw.scripts 13 | outputdir /work2/00434/eijkhout/demonspawn/paw.output 14 | nodes 2 15 | suite name:paw-mpi-p2p type:mpi dir:/work2/00434/eijkhout/mpi-regression/paw/installation-frontera p2p_* 16 | nodes 2,5,10,42 17 | suite name:paw-mpi-col type:mpi dir:/work2/00434/eijkhout/mpi-regression/paw/installation-frontera col_* 18 | # 19 | queue daos 20 | scriptdir /work2/00434/eijkhout/demonspawn/paw.scripts-daos 21 | outputdir /work2/00434/eijkhout/demonspawn/paw.output-daos 22 | nodes 2 23 | suite name:paw-daos-p2p type:mpi dir:/work2/00434/eijkhout/mpi-regression/paw/installation-frontera-daos p2p_* 24 | nodes 2,5,10,42 25 | suite name:paw-mpi-col type:mpi dir:/work2/00434/eijkhout/mpi-regression/paw/installation-frontera-daos col_* 26 | -------------------------------------------------------------------------------- /conf/conf.paw.frontera.all: -------------------------------------------------------------------------------- 1 | ## 2 | ## Demonspawn configuration 3 | ## for running the Carlos Rosales PAW suite 4 | ## 5 | 6 | # 7 | # some global setup 8 | # 9 | system frontera 10 | user eijkhout 11 | modules intel/19.1.1 impi/19.0.7 12 | account A-ccsc 13 | queue normal 14 | 15 | # 16 | # describe this run 17 | # 18 | rootdir /work2/00434/eijkhout/demonspawn 19 | # slurm scripts 20 | scriptdir %[rootdir]/paw.scripts.%[system].%[date] 21 | # all output 22 | outputdir %[rootdir]/paw.output.%[system].%[date] 23 | # verbose output 24 | trace 1 25 | # data for regression 26 | regression grep:Maximum field:4 take:avg 27 | 28 | # 29 | # p2p benchmark 30 | # 31 | pawdir /work2/00434/eijkhout/mpi-regression/paw/installation_git_%[system]_%[modules] 32 | # number of nodes 33 | nodes 2 34 | # processes per node 35 | ppn 1 36 | suite name:paw-mpi-p2p type:mpi dir:%[pawdir] p2p_* 37 | 38 | # 39 | # collective benchmark 40 | # 41 | # number of nodes 42 | nodes 2,5,10,50,200 43 | # processes per node 44 | ppn 2,20,48 45 | suite name:paw-mpi-col type:mpi dir:%[pawdir] col_* 46 | -------------------------------------------------------------------------------- /docs/commonmacs.tex: -------------------------------------------------------------------------------- 1 | % -*- latex -*- 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %%%% 5 | %%%% This LaTeX file is part of the source of 6 | %%%% `Parallel Computing' 7 | %%%% by Victor Eijkhout, copyright 2012-2021 8 | %%%% 9 | %%%% commonmacs.tex : common macros for book and slides and all 10 | %%%% 11 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 12 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | 14 | %% 15 | %% verbatim 16 | %% 17 | \newcommand\InnocentChars{% 18 | \catcode`\$=12 \catcode`\#=12 19 | \catcode`\_=12 \catcode`\>=12 \catcode`\<=12\relax 20 | \catcode`\&=12 \catcode`\^=12 \catcode`\~=12\relax 21 | \def\\{\char`\\}\relax 22 | } 23 | 24 | \def\n#{\bgroup \InnocentChars \tt \let\next=} 25 | 26 | %% 27 | %% math stuff 28 | %% 29 | \newcommand\inv{^{-1}}\newcommand\invt{^{-t}} 30 | \newcommand\setspan[1]{[\![#1]\!]} 31 | \newcommand\fp[2]{#1\cdot10^{#2}} 32 | \newcommand\fxp[2]{\langle #1,#2\rangle} 33 | 34 | \newcommand\diag{\mathop{\mathrm {diag}}} 35 | \newcommand\argmin{\mathop{\mathrm {argmin}}} 36 | \newcommand\defined{ 37 | \mathrel{\lower 5pt \hbox{${\equiv\atop\mathrm{\scriptstyle D}}$}}} 38 | 39 | \newcommand\bbP{\mathbb{P}} 40 | \newcommand\bbR{\mathbb{R}} 41 | 42 | -------------------------------------------------------------------------------- /docs/presentation.tex: -------------------------------------------------------------------------------- 1 | % -*- latex -*- 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %%%% 5 | %%%% Driver file for presentation about demonspawn 6 | %%%% 7 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 8 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 9 | 10 | \documentclass[11pt,headernav]{beamer} 11 | 12 | \beamertemplatenavigationsymbolsempty 13 | \usetheme{Madrid}%{Montpellier} 14 | \usecolortheme{seahorse} 15 | \setcounter{tocdepth}{1} 16 | 17 | \setbeamertemplate{footline}{\hskip1em Eijkhout: demonspawn\hfill 18 | \hbox to 0in {\hss \includegraphics[scale=.1]{tacclogonew}}% 19 | \hbox to 0in {\hss \arabic{page}\hskip 1in}} 20 | 21 | \usepackage{morewrites} 22 | \usepackage{array} %,multicol,multirow} 23 | \newcolumntype{R}{>{\hbox to 1.2em\bgroup\hss}{r}<{\egroup}} 24 | \newcolumntype{T}{>{\hbox to 8em\bgroup}{c}<{\hss\egroup}} 25 | 26 | \input commonmacs 27 | \input acromacs 28 | \input coursemacs 29 | \input listingmacs 30 | \input snippetmacs 31 | 32 | \lstset{basicstyle=\ttfamily, 33 | showstringspaces=false, 34 | commentstyle=\color{black}, 35 | keywords={nothing}, 36 | keywordstyle=\color{black} 37 | } 38 | 39 | %%%% 40 | %%%% Where is this course? 41 | %%%% 42 | 43 | \def\Location{}% redefine in the inex file 44 | \def\courseyear{2022} 45 | \def\Location{TACC HPC \courseyear} 46 | 47 | %%%%%%%%%%%%%%%% 48 | %%%%%%%%%%%%%%%% Document 49 | %%%%%%%%%%%%%%%% 50 | 51 | \begin{document} 52 | \parskip=10pt plus 5pt minus 3pt 53 | 54 | \title{Rapid Regression Testing with Demonspawn} 55 | \author{ Victor Eijkhout} 56 | \date{2022} 57 | 58 | \begin{frame} 59 | \titlepage 60 | \end{frame} 61 | 62 | \input body 63 | 64 | \end{document} 65 | 66 | -------------------------------------------------------------------------------- /docs/acromacs.tex: -------------------------------------------------------------------------------- 1 | % -*- latex 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %%%% 5 | %%%% This text file is part of the source of 6 | %%%% `Introduction to High-Performance Scientific Computing' 7 | %%%% by Victor Eijkhout, copyright 2021 8 | %%%% 9 | %%%% This book is distributed under a Creative Commons Attribution 3.0 10 | %%%% Unported (CC BY 3.0) license and made possible by funding from 11 | %%%% The Saylor Foundation \url{http://www.saylor.org}. 12 | %%%% 13 | %%%% acromacs : acronym definitions 14 | %%%% 15 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 16 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 17 | 18 | \usepackage{acronym} 19 | \newwrite\acrowrite 20 | \openout\acrowrite=acronyms.tex 21 | \def\acroitem#1#2{\acrodef{#1}{#2} 22 | \write\acrowrite{\message{defining #1}\noexpand\acitem{#1}{#2}} 23 | } 24 | \acroitem{API}{Application Programmer Interface} 25 | \acroitem{AMG}{Algebraic MultiGrid} 26 | \acroitem{AVX}{Advanced Vector Extensions} 27 | \acroitem{BLAS}{Basic Linear Algebra Subprograms} 28 | \acroitem{BSP}{Bulk Synchronous Parallel} 29 | \acroitem{CAF}{Co-array Fortran} 30 | \acroitem{CRS}{Compressed Row Storage} 31 | \acroitem{CSP}{Communicating Sequential Processes} 32 | \acroitem{CG}{Conjugate Gradients} 33 | \acroitem{CUDA}{Compute-Unified Device Architecture} 34 | \acroitem{DAG}{Directed Acyclic Graph} 35 | \acroitem{DFS}{Depth First Search} 36 | \acroitem{DPCPP}{Data Parallel C++} 37 | \acroitem{DSP}{Digital Signal Processing} 38 | \acroitem{FEM}{Finite Element Method} 39 | \acroitem{FIFO}{First-In~/ First-Out} 40 | \acroitem{FPU}{Floating Point Unit} 41 | \acroitem{FFT}{Fast Fourier Transform} 42 | \acroitem{FSA}{Finite State Automaton} 43 | \acroitem{GPU}{Graphics Processing Unit} 44 | \acroitem{HPC}{High-Performance Computing} 45 | \acroitem{HPF}{High Performance Fortran} 46 | \acroitem{HPL}{High Performance Linpack} 47 | \acroitem{ICV}{Internal Control Variable} 48 | \acroitem{LAPACK}{Linear Algebra Package} 49 | \acroitem{MG}{Multi-Grid} 50 | \acroitem{MIC}{Many Integrated Cores} 51 | \acroitem{MIMD}{Multiple Instruction Multiple Data} 52 | \acroitem{MPI}{Message Passing Interface} 53 | \acroitem{MPL}{Message Passing Layer} 54 | \acroitem{MPMD}{Multiple Program Multiple Data} 55 | \acroitem{MTA}{Multi-Threaded Architecture} 56 | \acroitem{NIC}{Network Interface Card} 57 | \acroitem{NUMA}{Non-Uniform Memory Access} 58 | \acroitem{OO}{Object-Oriented} 59 | \acroitem{OOP}{Object-Oriented Programming} 60 | \acroitem{OS}{Operating System} 61 | \acroitem{PGAS}{Partitioned Global Address Space} 62 | \acroitem{PDE}{Partial Diffential Equation} 63 | \acroitem{PRAM}{Parallel Random Access Machine} 64 | \acroitem{RDMA}{Remote Direct Memory Access} 65 | \acroitem{RMA}{Remote Memory Access} 66 | \acroitem{SAN}{Storage Area Network} 67 | \acroitem{SaaS}{Software as-a Service} 68 | \acroitem{SFC}{Space-Filling Curve} 69 | \acroitem{SIMD}{Single Instruction Multiple Data} 70 | \acroitem{SIMT}{Single Instruction Multiple Thread} 71 | \acroitem{SLURM}{Simple Linux Utility for Resource Management} 72 | \acroitem{SM}{Streaming Multiprocessor} 73 | \acroitem{SMP}{Symmetric Multi Processing} 74 | \acroitem{SOR}{Successive Over-Relaxation} 75 | \acroitem{SP}{Streaming Processor} 76 | \acroitem{SPMD}{Single Program Multiple Data} 77 | \acroitem{SPD}{symmetric positive definite} 78 | \acroitem{SSE}{SIMD Streaming Extensions} 79 | \acroitem{STL}{Standard Template Library} 80 | \acroitem{TACC}{Texas Advanced Computing Center} 81 | \acroitem{TLB}{Translation Look-aside Buffer} 82 | \acroitem{UMA}{Uniform Memory Access} 83 | \acroitem{UPC}{Unified Parallel C} 84 | \acroitem{URI}{Uniform Resource Identifier} 85 | \acroitem{WAN}{Wide Area Network} 86 | \acresetall 87 | \closeout\acrowrite 88 | 89 | -------------------------------------------------------------------------------- /docs/body.tex: -------------------------------------------------------------------------------- 1 | \begin{numberedframe}{Basic design} 2 | \begin{itemize} 3 | \item Benchmark suite software has been compiled 4 | \item You want to fire off many SLURM jobs 5 | \item Regression testing on output 6 | \item Regression compare to previous run 7 | \end{itemize} 8 | \url{https://github.com/TACC/demonspawn} 9 | \end{numberedframe} 10 | 11 | \begin{numberedframe}{Example: OSU point-to-point suite} 12 | Key-value specification: 13 | \begin{verbatim} 14 | jobname osubenchmark 15 | osudir %[STOCKYARD]/osubenchmark/mpi/ 16 | 17 | # point-to-point 18 | queue small 19 | nodes 2 20 | ppn 1 21 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 22 | 23 | # collective 24 | queue normal 25 | nodes 3,10,30,100 26 | regression line:last field:2 margin:30percent 27 | suite name:osu-col type:mpi dir:%[osudir]/collective * 28 | \end{verbatim} 29 | \end{numberedframe} 30 | 31 | \begin{numberedframe}{Macros} 32 | Macros: 33 | \begin{itemize} 34 | \item keys from this configuration file, or 35 | \item environment variables; 36 | \item can be redefined; 37 | \item using a new syntax because why not 38 | \end{itemize} 39 | \begin{verbatim} 40 | osudir %[STOCKYARD]/osubenchmark/mpi/ 41 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 42 | \end{verbatim} 43 | \end{numberedframe} 44 | 45 | \begin{numberedframe}{Benchmark suite} 46 | Multiple suites per configuration; 47 | each \n{suite} line uses definitions in effect at that point 48 | \begin{verbatim} 49 | queue small 50 | nodes 2 51 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 52 | 53 | queue normal 54 | nodes 3,10,30,100 55 | suite name:osu-col type:mpi dir:%[osudir]/collective * 56 | \end{verbatim} 57 | \end{numberedframe} 58 | 59 | \begin{numberedframe}{SLURM job specification} 60 | \begin{verbatim} 61 | queue small 62 | nodes 2 63 | ppn 1 64 | \end{verbatim} 65 | \end{numberedframe} 66 | 67 | \begin{numberedframe}{Queue spec in rc file} 68 | Machine-dependent settings 69 | \begin{verbatim} 70 | system frontera 71 | user eijkhout 72 | account A-ccsc 73 | 74 | queue development limit:1 75 | queue rtx limit:4 76 | queue normal limit:10 77 | queue small limit:10 78 | \end{verbatim} 79 | \end{numberedframe} 80 | 81 | \begin{numberedframe}{Collective suite} 82 | Loop over node counts: 83 | \begin{verbatim} 84 | queue normal 85 | nodes 3,10,30,100 86 | suite name:osu-col type:mpi dir:%[osudir]/collective * 87 | \end{verbatim} 88 | \end{numberedframe} 89 | 90 | \begin{numberedframe}{Regression specification} 91 | Each slurm job outputs to its own file,\\ 92 | regression spec applies to that. 93 | 94 | Keywords still under design. Let me know what you need~\ldots 95 | \begin{verbatim} 96 | regression line:last field:2 margin:30percent 97 | suite name:osu-ptp type:mpi dir:%[osudir]/pt2pt * 98 | \end{verbatim} 99 | \end{numberedframe} 100 | 101 | \begin{numberedframe}{Invocation with regression} 102 | Option \n{-o outputdir}: 103 | \begin{verbatim} 104 | python3 spawn.py \ 105 | -o spawn_osu_60_mvap \ 106 | conf/osu.conf 107 | \end{verbatim} 108 | \end{numberedframe} 109 | 110 | \begin{numberedframe}{Regression comparison} 111 | Option \n{-r outputdir}: only regression, no runs\\ 112 | option \n{-c prevdir}: compare regression to previous run 113 | \begin{verbatim} 114 | python3 spawn.py \ 115 | -r spawn_osu_60_mvap \ 116 | -c spawn_osu_60_impi/ \ 117 | conf/osu.conf 118 | \end{verbatim} 119 | \end{numberedframe} 120 | 121 | \begin{numberedframe}{Output structure} 122 | \begin{verbatim} 123 | [staff demonspawn:625] ls spawn_osu_60_impi 124 | logfile-spawn-2022829-10.22 125 | output/ 126 | regression/ 127 | regression-osu-col.txt 128 | regression-osu-ptp.txt 129 | scripts/ 130 | \end{verbatim} 131 | \end{numberedframe} 132 | 133 | \begin{numberedframe}{Comparison output} 134 | \tiny 135 | \begin{verbatim} 136 | Comparing: output=spawn_osu_60_mvap/regression/osu-col-osu_ialltoall-3-1-0.txt compare=spawn_osu_60_impi//regression/osu-col-osu_ialltoall-3-1-0.txt 137 | Output: 502.89, compare: 518.98, inside 0.2 margin 138 | 139 | ================ Major violations ================ 140 | 141 | spawn_osu_60_mvap/regression/osu-col-osu_ialltoall-10-1-0.txt 142 | Output: 1778.99, compare: 2206.79, outside 0.2 margin: less 143 | spawn_osu_60_mvap/regression/osu-col-osu_iallreduce-100-1-0.txt 144 | Output: 3367.70, compare: 1699.57, outside 0.2 margin: more 145 | \end{verbatim} 146 | \end{numberedframe} 147 | 148 | -------------------------------------------------------------------------------- /docs/snippetmacs.tex: -------------------------------------------------------------------------------- 1 | % -*- latex -*- 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %%%% 5 | %%%% This text file is part of the source of 6 | %%%% `Introduction to High-Performance Scientific Computing' 7 | %%%% by Victor Eijkhout, copyright 2012-9 8 | %%%% 9 | %%%% This book is distributed under a Creative Commons Attribution 3.0 10 | %%%% Unported (CC BY 3.0) license and made possible by funding from 11 | %%%% The Saylor Foundation \url{http://www.saylor.org}. 12 | %%%% 13 | %%%% 14 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 15 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 16 | 17 | \def\verbatimsnippet#1{\verbatiminput{#1}} % snippets/ 18 | \newcommand{\cverbatimsnippet}[2][XX]{ 19 | \languageVerbatimSnippet{#1}{#2}{C} 20 | } 21 | \newcommand{\cxxverbatimsnippet}[2][XX]{ 22 | \languageVerbatimSnippet{#1}{#2}{C++} 23 | } 24 | \newcommand{\fverbatimsnippet}[2][XX]{ 25 | \languageVerbatimSnippet{#1}{#2}{Fortran} 26 | } 27 | \newcommand{\pverbatimsnippet}[2][XX]{ 28 | \languageVerbatimSnippet{#1}{#2}{Python} 29 | } 30 | % #1 file name (or XX if not given) 31 | % #2 snippet name 32 | % #3 language 33 | \newcommand{\languageVerbatimSnippet}[3]{ 34 | % record this file as bracketed name 35 | \def\xx{XX}\def\yy{#1} 36 | % typeset as nice Python code 37 | \lstset{style=reviewcode,language=#3} 38 | \lstinputlisting{#2} 39 | \lstset{style=reviewcode,language=C} 40 | \nobreak 41 | \ifx\xx\yy\else 42 | {\globaldefs=1 \addchaptersource{#1} } 43 | \moveright .5\unitindent \hbox{% 44 | \textsl{For the full source of this example, see section~\ref{lst:#1}} 45 | }\par 46 | \fi 47 | } 48 | 49 | \newenvironment{clisting} 50 | {\lstset{style=reviewcode,language=C}\begin{lstlisting}} 51 | {\end{lstlisting}} 52 | \newenvironment{cxxlisting} 53 | {\lstset{style=reviewcode,language=C++}\begin{lstlisting}} 54 | {\end{lstlisting}} 55 | \newenvironment{flisting} 56 | {\lstset{style=reviewcode,language=Fortran}\begin{lstlisting}} 57 | {\end{lstlisting}} 58 | \newenvironment{plisting} 59 | {\lstset{style=reviewcode,language=Python}\begin{lstlisting}} 60 | {\end{lstlisting}} 61 | 62 | \makeatletter 63 | \newcommand{\csnippetwithoutput}[2]{ 64 | \message{Code snippet <#1> with output file <#2>} 65 | % go into vertical mode 66 | \par 67 | % make nice two-column layout 68 | \hbox{% 69 | \begin{minipage}[t]{.6\hsize} 70 | \footnotesize\textbf{Code:} 71 | \lstset{language=C,xleftmargin=0pt} 72 | \lstinputlisting{#1} 73 | \lstset{xleftmargin=\unitindent} 74 | \end{minipage} 75 | \begin{minipage}[t]{.3\hsize} 76 | { \footnotesize \raggedright \textbf{Output:}\par 77 | } 78 | \footnotesize 79 | \def\verbatim@startline{\verbatim@line{\leavevmode\relax}} 80 | \verbatiminput{#2.out} 81 | \end{minipage} 82 | } 83 | } 84 | \endinput 85 | 86 | \def\verbatimsnippet#1{\verbatiminput{snippets/#1}} 87 | \def\cverbatimsnippet#1{\verbatiminput{snippets/#1}} 88 | \usepackage{listings,xcolor} 89 | \lstset{language=C} 90 | \lstdefinestyle{reviewcode}{ 91 | belowcaptionskip=1\baselineskip, breaklines=true, frame=L, 92 | xleftmargin=.5\unitindent, showstringspaces=false, 93 | basicstyle=\footnotesize\ttfamily, 94 | keywordstyle=\bfseries\color{blue}, 95 | commentstyle=\color{red!60!black}, 96 | identifierstyle=\slshape\color{black}, 97 | stringstyle=\color{green!60!black}, columns=fullflexible, 98 | keepspaces=true, } 99 | \lstset{style=reviewcode} 100 | \newcommand\pyinline[1]{\lstset{language=Python}\lstinline{#1}\lstset{language=C}} 101 | 102 | \def\codedir{./code} 103 | 104 | \newcommand\snippetwithoutput[4][]{ 105 | %\message{snippet <<#1>> <<#2>> <<#3>> <<#4>>} 106 | %\tracingmacros=2 \tracingcommands=2 107 | \answerwithoutput{#2}{#3}{#4} 108 | } 109 | \newcommand{\snippetoutput}[2]{ 110 | \message{In directory <#1> running program <#2>} 111 | % go into vertical mode 112 | \par 113 | % make nice two-column layout 114 | \begin{minipage}[t]{.3\hsize}{% 115 | \footnotesize \raggedright \textbf{Output\\\relax [#1] #2:}\par } 116 | \immediate\write18{ 117 | cd \codedir /#1 118 | && make run_#2 > #2.out 2>&1 119 | } 120 | \footnotesize 121 | \def\verbatim@startline{\verbatim@line{\leavevmode\relax}} 122 | \verbatiminput{\codedir /#1/#2.out} 123 | \end{minipage} 124 | } 125 | \makeatother 126 | -------------------------------------------------------------------------------- /spawn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Demonspawn 4 | # a utility for quickly generating a slew of batch jobs 5 | # good for benchmarking, regression testing, and such 6 | # 7 | # Victor Eijkhout 8 | # copyright 2020-2022 9 | # 10 | # version 0.4, see the Readme for details 11 | # 12 | # spawn.py : main diver file 13 | # 14 | 15 | import copy 16 | import datetime 17 | import os 18 | import re 19 | import sys 20 | import time 21 | 22 | 23 | #-------------------------------------------------------------------------------- 24 | # Local 25 | from jobsuite import * 26 | from pathlib import Path 27 | 28 | keyword_command = [ "nodes", "ppn", "suite", ] 29 | keyword_reserved = [ "system", "modules", 30 | # slurm variables 31 | "account", "queue", "date", "time", "user", 32 | ] 33 | 34 | def read_batch_template(filename): 35 | """ 36 | Read in Slurm batch submit template and return as a string. 37 | """ 38 | return open(filename, "r").read() 39 | 40 | 41 | def DefaultModules(): 42 | return "intel/18.0.2" 43 | 44 | def wait_for_jobs( jobs ): 45 | while True: 46 | running = []; pending = [] 47 | for j in jobs: 48 | id = j.jobid 49 | status = j.get_status() 50 | if status=="R": 51 | running.append(id) 52 | elif status=="PD": 53 | pending.append(id) 54 | print(f"Running: {running} Pending: {pending}") 55 | if len(running)+len(pending)==0: 56 | break 57 | time.sleep(1) 58 | 59 | def get_suite_name(options,values): 60 | if "name" in options.keys(): 61 | return options["name"] 62 | else: 63 | for kv in values: 64 | if re.match("name:",kv): 65 | k,v = kv.split(":") 66 | return v 67 | return "testsuite" 68 | 69 | def print_configuration(confdict): 70 | print(""" 71 | ################ Configuration ################ 72 | Running as: 73 | ################################ 74 | """.format(str(confdict))) 75 | 76 | def test_job(): 77 | print("""================ 78 | 79 | Test job in main 80 | 81 | ================""") 82 | job = Job(script="/bin/true") 83 | id = job.submit() 84 | print("""Job script 85 | ================ 86 | {} 87 | ================ 88 | submitted as {}""".format(str(job),id)) 89 | 90 | class Configuration(): 91 | def __init__(self,**kwargs): 92 | self.configuration = {} 93 | for key,val in kwargs.items(): 94 | self.configuration[key] = val 95 | jobname = self.configuration["jobname"] 96 | self.configuration["modules"] = "default" 97 | self.configuration["time"] = "0:37:0" 98 | try : 99 | self.configuration["system"] = os.environ["TACC_SYSTEM"] 100 | except: 101 | self.configuration["system"] = None 102 | try : 103 | self.configuration["mpi"] = os.environ["LMOD_FAMILY_MPI"] 104 | except: 105 | self.configuration["mpi"] = "mpich" 106 | self.configuration["pwd"] = os.getcwd() 107 | def parse(self,filename,**kwargs): 108 | for k in [ "suites","sbatch","env" ]: 109 | self.configuration[k] = [] 110 | queue = None 111 | with open(filename,"r") as configuration: 112 | for specline in configuration: 113 | specline = specline.strip() 114 | # 115 | # skip comments 116 | if re.match("#",specline) or re.match(r'^[ \t]*$',specline): 117 | continue 118 | # 119 | # otherwise interpret as key/value 120 | # 121 | key,value = specline.split(" ",1) 122 | # special case: system 123 | if key=="system": 124 | if value!=self.configuration["system"]: 125 | print(f"This configuration can only be run on <<{value}>>") 126 | sys.exit(1) 127 | # substitute any macros 128 | value = macros_substitute( value,self.configuration ) 129 | 130 | # special case: jobname can be set only once 131 | if key=="jobname" and jobname != "spawn": 132 | raise Exception(f"Job name can be set only once, current: {jobname}") 133 | # special case: queue 134 | elif key=="queue": 135 | queue = value; nam_lim = value.split(); qname = nam_lim[0]; qlimit = 1 136 | if len(nam_lim)>1: 137 | qlimit = nam_lim[1] 138 | if re.match("limit",qlimit): 139 | qlimit = qlimit.split(":")[1] 140 | Queues().add_queue( qname,qlimit ) 141 | self.configuration[key] = qname 142 | # special case: output dir needs to be set immediately 143 | elif key=="outputdir": 144 | raise Exception("outputdir key deprecated") 145 | # special case: `sbatch' and `env' lines are appended 146 | elif key in ["sbatch","env"]: 147 | self.configuration[key].append(value) 148 | # 149 | # suite or macro 150 | # 151 | elif key=="suite": 152 | # now parse 153 | fields = value.split(" ") 154 | suitespec = [ macros_substitute(f,self.configuration) for f in fields ] 155 | n = get_suite_name(self.configuration,suitespec) 156 | s = TestSuite( suitespec, copy.copy(self.configuration) ) 157 | self.configuration["suites"].append(s) 158 | else: 159 | self.configuration[key] = value 160 | def run(self): 161 | for s in self.configuration["suites"]: 162 | s.run(debug=self.configuration["debug"], 163 | submit=self.configuration["submit"], 164 | testing=self.configuration["testing"]) 165 | 166 | if __name__ == "__main__": 167 | if sys.version_info[0]<3: 168 | print("Please move to python3"); sys.exit(1) 169 | if sys.version_info[1]<8: 170 | print("This requires at least python 3.8"); sys.exit(1) 171 | args = sys.argv[1:] 172 | testing = False; debug = False; submit = True 173 | jobname = "spawn"; outputdir = None; comparedir = None 174 | rootdir = os.getcwd() 175 | while re.match("^-",args[0]): 176 | if args[0]=="-h": 177 | print("Usage: python3 batch.py [ -h ] [ -d --debug ] [ -f --filesonly ] [ -t --test ] [ -n name ] [ -r --regression dir ] [ -o --output dir ] [ -c --compare dir ]") 178 | sys.exit(0) 179 | elif args[0] == "-n": 180 | args = args[1:]; jobname = args[0] 181 | elif args[0] in [ "-f", "--filesonly" ] : 182 | submit = False; testing = False 183 | elif args[0] in [ "-o", "--outputdir" ] : 184 | args = args[1:]; outputdir = args[0] 185 | elif args[0] in [ "-r", "--regression" ] : 186 | args = args[1:]; outputdir = args[0] 187 | testing = True; submit = False 188 | elif args[0] in [ "-c", "--compare" ] : 189 | args = args[1:]; comparedir = args[0] 190 | if not os.path.exists(comparedir): 191 | raise Exception(f"Compare directory <<{comparedir}>> does not exist") 192 | elif args[0] in [ "-t", "--test" ]: 193 | testing = True; submit = False 194 | elif args[0] in [ "-d", "--debug" ]: 195 | debug = True 196 | SpawnFiles().debug = True 197 | args = args[1:] 198 | now = datetime.datetime.now() 199 | starttime = f"{now.year}{now.month}{now.day}-{now.hour}.{now.minute}" 200 | 201 | print(f"Output dir: {outputdir}") 202 | if not outputdir: 203 | outputdir = f"spawn_output_{starttime}" 204 | SpawnFiles().setoutputdir(outputdir) 205 | 206 | SpawnFiles().open_new(f"logfile-{jobname}-{starttime}",key="logfile") 207 | configuration = Configuration\ 208 | (jobname=jobname,date=starttime,debug=debug,submit=submit,testing=testing, 209 | outputdir=outputdir,comparedir=comparedir) 210 | queues = Queues() 211 | queues.testing = testing 212 | if os.path.exists(".spawnrc"): 213 | configuration.parse(".spawnrc") 214 | else: 215 | globalrc = f"{Path.home()}/.spawnrc" 216 | if os.path.exists( globalrc ): 217 | configuration.parse(globalrc) 218 | configuration.parse(args[0]) 219 | 220 | # now activate all the suites 221 | configuration.run() 222 | # close all files 223 | SpawnFiles().__del__() 224 | 225 | -------------------------------------------------------------------------------- /docs/coursemacs.tex: -------------------------------------------------------------------------------- 1 | % -*- latex -*- 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %%%% 5 | %%%% This text file is part of the lecture slides for 6 | %%%% `Parallel Programming in MPI and OpenMP' 7 | %%%% by Victor Eijkhout, copyright 2012-2021 8 | %%%% 9 | %%%% coursemacs.tex : macros for the lecture slides 10 | %%%% THIS NEEDS TO BE MERGED INTO slidemacs AS MUCH AS POSSIBLE 11 | %%%% 12 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 14 | 15 | \usepackage{pslatex} 16 | \usepackage{verbatim,wrapfig} 17 | 18 | \usepackage{amsmath} 19 | \usepackage{array,booktabs,multirow,multicol} 20 | \newcolumntype{R}{>{\hbox to 1.2em\bgroup\hss}{r}<{\egroup}} 21 | \newcolumntype{T}{>{\hbox to 8em\bgroup}{c}<{\hss\egroup}} 22 | 23 | \def\verbatimsnippet#1{\begingroup\small \verbatiminput{snippets/#1}\endgroup} 24 | \usepackage{hyperref} 25 | 26 | \newdimen\unitindent \unitindent=20pt 27 | \usepackage[algo2e,noline,noend]{algorithm2e} 28 | \newenvironment{displayalgorithm} 29 | {\par 30 | \begin{algorithm2e}[H]\leftskip=\unitindent \parskip=0pt\relax 31 | \DontPrintSemicolon 32 | \SetKwInOut{Input}{Input}\SetKwInOut{Output}{Output} 33 | } 34 | {\end{algorithm2e}\par} 35 | \newenvironment{displayprocedure}[2] 36 | {\everymath{\strut} 37 | \begin{procedure}[H]\leftskip=\unitindent\caption{#1(#2)}} 38 | {\end{procedure}} 39 | \def\sublocal{_{\mathrm\scriptstyle local}} 40 | 41 | %%%% 42 | %%%% Comment environment 43 | %%%% 44 | \usepackage{comment} 45 | %\input longshort 46 | \specialcomment{exercise}{ 47 | \def\CommentCutFile{exercise.cut} 48 | \def\PrepareCutFile{% 49 | \immediate\write\CommentStream{\noexpand\begin{frame}{Exercise}}} 50 | \def\FinalizeCutFile{% 51 | \immediate\write\CommentStream{\noexpand\end{frame}}} 52 | }{} 53 | \excludecomment{pcse} 54 | \excludecomment{book} 55 | 56 | %%%% 57 | %%%% Outlining 58 | %%%% 59 | \usepackage{outliner} 60 | \OutlineLevelStart 0{\part{#1}\frame{\partpage}} %{\frame{\part{#1}\Huge\bf #1}} 61 | %\OutlineLevelStart 1{\section{#1}} 62 | \OutlineLevelStart 1{\frame{\section{#1}\Large\bf#1}} 63 | \def\sectionframe#{\Level 1 } 64 | \setcounter{tocdepth}{2} 65 | \usepackage{framed} 66 | \colorlet{shadecolor}{blue!15} 67 | \OutlineLevelStart 2{\subsection{#1} 68 | \frame{\begin{shaded}\large #1\end{shaded}}} 69 | %% \OutlineLevelCont 2{\end{frame}\begin{frame}{#1}} 70 | %% \OutlineLevelEnd 2{\end{frame}} 71 | %\OutlineTracetrue 72 | 73 | \newcommand\coursepart[1]{ 74 | %\addcontentsline{toc}{part}{Section: #1} 75 | \begin{frame}{} 76 | \setbox0=\hbox to \hsize{\hfil\Huge{#1}\hfil} 77 | \dimen0=\vsize 78 | \advance\dimen0 by -\ht0 \advance\dimen0 by -\dp0 79 | \divide\dimen0 by 2 80 | \hbox{} 81 | \vskip \dimen0 82 | \box0 83 | \vskip \dimen0 84 | \hbox{} 85 | \end{frame} 86 | } 87 | 88 | \def\underscore{_} 89 | \def\mpiRoutineRef#1{\RoutineRefStyle\verbatiminput{#1}} 90 | \def\RoutineRefStyle{\scriptsize} 91 | \def\protoslide#{\bgroup \catcode`\_=12 92 | \afterassignment\protoslideinclude \def\protoname} 93 | \def\protoslideinclude{% 94 | \begin{frame}[containsverbatim]\frametitle{\texttt\protoname} 95 | \scriptsize 96 | \edef\tmp{\lowercase{\def\noexpand\standardroutine{\protoname}}}\tmp 97 | \IfFileExists 98 | {standard/\standardroutine.tex} 99 | {\input{standard/\standardroutine}} 100 | {% if no standard file, then maybe handwritten file 101 | \IfFileExists 102 | {mpireference/\protoname.tex} 103 | {\verbatiminput{mpireference/\protoname}}{}} 104 | %\expandafter\mpiRoutineRef\expandafter{\protoname} 105 | \end{frame}\egroup 106 | } 107 | 108 | \def\innocentcharacters{% move this to commen and unify 109 | \catcode`\_=12 \catcode`\#=12 110 | \catcode`\>=12 \catcode`\<=12 111 | \catcode`\&=12 \catcode`\^=12 \catcode`\~=12 112 | } 113 | \newcommand\slackpoll[1]{\begingroup \par \tiny\texttt{/poll #1} \par \endgroup} 114 | \def\slackpollTF#1{\begingroup \def\terminator{#1} 115 | \innocentcharacters \catcode`\{=12 \catcode`\}=12 116 | \edef\tmp{\def\noexpand\slackpollTFp####1\terminator 117 | {\noexpand\par 118 | \noexpand\tiny \noexpand\texttt{/poll "####1" "T" "F"} 119 | \noexpand\par \endgroup}}\tmp 120 | \slackpollTFp} 121 | 122 | \newcounter{excounter} 123 | \newcounter{revcounter} 124 | \newenvironment 125 | {reviewframe} 126 | {\begin{frame}[containsverbatim] 127 | \frametitle{Review \arabic{revcounter}} 128 | \refstepcounter{revcounter}} 129 | {\end{frame}} 130 | \newenvironment 131 | {exerciseframe}[1][] 132 | {\def\optfile{#1} 133 | \ifx\optfile\empty\else\message{VLE EXERCISE #1}%\index[programming]{#1} 134 | \fi 135 | \begin{frame}[containsverbatim] 136 | \ifx\optfile\empty 137 | \frametitle{Exercise \arabic{excounter}} 138 | \else 139 | \frametitle{Exercise \arabic{excounter} (\tt{#1})} 140 | \fi 141 | \refstepcounter{excounter}} 142 | {\end{frame}} 143 | \newenvironment 144 | {optexerciseframe}[1][] 145 | {\begin{frame}[containsverbatim] 146 | \def\optfile{#1} 147 | \ifx\optfile\empty 148 | \frametitle{Exercise (optional) \arabic{excounter}} 149 | \else 150 | \frametitle{Exercise (optional) \arabic{excounter} (\tt{#1})} 151 | \fi 152 | \refstepcounter{excounter}} 153 | {\end{frame}} 154 | 155 | %\advance\textwidth by 1in 156 | %\advance\oddsidemargin by -.5in 157 | 158 | %%%% 159 | %%%% Beamer customization 160 | %%%% 161 | 162 | \setbeamertemplate{title page}{ 163 | \begin{picture}(0,0) 164 | \put(-10,10){\includegraphics[scale=.45]{tacctitle}} 165 | \put(50,-50){ 166 | \begin{minipage}{10cm} 167 | \usebeamerfont{title}{\inserttitle\par\insertauthor\par\insertdate\par} 168 | \end{minipage} 169 | } 170 | \end{picture} 171 | } 172 | 173 | %%%% 174 | %%%% Indexing, mostly disabled 175 | %%%% 176 | 177 | % the 178 | % \usepackage{imakeidx} 179 | % line is in the master file 180 | % 181 | %% \makeindex 182 | %% \makeindex[name=programming,title=Programming exercises] 183 | 184 | \let\indexterm\emph 185 | \let\indextermtt\n 186 | \let\indextermfunction\indextermtt 187 | \newcommand{\indextermsubh}[2]{\emph{#1-#2}} 188 | 189 | \iffalse 190 | \newcommand{\indextermp}[1]{\emph{#1s}} 191 | \newcommand{\indextermsub}[2]{\emph{#1 #2}} 192 | \newcommand{\indextermsubp}[2]{\emph{#1 #2s}} 193 | \newcommand{\indextermbus}[2]{\emph{#1 #2}} 194 | \newcommand{\indextermstart}[1]{\emph{#1}} 195 | \newcommand{\indextermend}[1]{} 196 | \newcommand{\indexstart}[1]{} 197 | \newcommand{\indexend}[1]{} 198 | \makeatletter 199 | \newcommand\indexac[1]{\emph{\ac{#1}}} 200 | \newcommand\indexacp[1]{\emph{\ac{#1}}} 201 | \newcommand\indexacf[1]{\emph{\acf{#1}}} 202 | \newcommand\indexacstart[1]{} 203 | \newcommand\indexacend[1]{} 204 | \makeatother 205 | 206 | \let\indexmpishow\n 207 | \let\indexmpidef\n 208 | \let\indexmpiex\n 209 | \let\indexmpi\n 210 | \let\indexompshow\n 211 | \let\indexompdef\n 212 | \global\let\indexompshowdef\indexompdef 213 | \let\indexompex\n 214 | \let\indexomp\n 215 | \let\mpitoindex\n 216 | \let\mpitoindexbf\n 217 | \let\mpitoindexit\n 218 | \let\omptoindex\n 219 | \let\omptoindexbf\n 220 | \let\omptoindexit\n 221 | %\let\indextermlet#1{\emph{#1}\index{#1|textbf}} 222 | \let\indextermtt\n 223 | %\let\indextermttlet\indexmpilet 224 | \let\indexcommand\n 225 | \let\indexclause\n 226 | \let\indexclauselet\n 227 | \let\indexclauseoption\n 228 | 229 | \let\indexmpiref\indexmpishow 230 | \let\indexmpidef\indexmpishow 231 | \let\indexmpldef\indexmpishow 232 | \let\indexmplref\indexmpishow 233 | \let\indexmplshow\indexmpishow 234 | \let\indexompshow\indexmpishow 235 | \let\indexpetscshow\indexmpishow 236 | \fi 237 | 238 | 239 | %%%% 240 | %%%% Environments 241 | %%%% 242 | \newcounter{slidecount} 243 | \setcounter{slidecount}{1} 244 | \newenvironment 245 | {numberedframe}[1] 246 | {\begin{frame}[containsverbatim]{\arabic{slidecount}.\ #1} 247 | \refstepcounter{slidecount} 248 | } 249 | {\end{frame}} 250 | \newenvironment{question}{\begin{quotation}\textbf{Question.\ }}{\end{quotation}} 251 | \newenvironment{fortrannote} 252 | {\begin{quotation}\noindent\textsl{Fortran note.\kern1em}\ignorespaces} 253 | {\end{quotation}} 254 | \newenvironment{pythonnote} 255 | {\begin{quotation}\noindent\textsl{Python note.\kern1em}\ignorespaces} 256 | {\end{quotation}} 257 | \newenvironment{taccnote} 258 | {\begin{quotation}\noindent\textsl{TACC note.\kern1em}\ignorespaces} 259 | {\end{quotation}} 260 | \newenvironment{mpifour} 261 | {\begin{quotation}\textbf{MPI-4:\ }\ignorespaces}{\end{quotation}} 262 | 263 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # DemonSpawn 2 | DemonSpawn is a tool for quickly firing off a large number of SLURM jobs. 3 | 4 | This can be used 5 | 6 | * as a benchmarking tool for kernels / micro-apps 7 | * as a stress test doing parametrized runs of a single test 8 | * sanity test: after a system upgrade, see if everything still works. 9 | 10 | Copyright Victor Eijkhout 2020-2022 11 | 12 | For questions and problems, please submit a github issue. 13 | 14 | ## Quick example 15 | 16 | Example configuration file: 17 | 18 | # slurm script parameters 19 | system frontera 20 | user eijkhout 21 | account A-ccsc 22 | modules intel/18.0.2 impi/18.0.2 23 | 24 | # point-to-point benchmark setup 25 | queue development 26 | nodes 2 27 | ppn 1 28 | # program to run 29 | suite name:paw-ptp type:mpi dir:/home/me/paw p2p_* 30 | 31 | # collective benchmark setup 32 | queue normal 33 | nodes 4,8,12,16 34 | ppn 1,5,10 35 | # program to run 36 | suite name:paw-col type:mpi dir:/home/me/paw coll_* 37 | 38 | ## Introduction 39 | 40 | Demonspawn is a script that schedules SLURM job. Invocation: 41 | 42 | python3 spawn.py [ options ] configurationfile 43 | 44 | Option flags: 45 | 46 | * `-h --help` : print help and quit. 47 | * `-d --debug` : lots of debug output. 48 | * `-f --filesonly` : generate all script files, but do not submit. 49 | * `-o --outputdir` + `dir` : specify output directory; omitting this gives a standard output name that includes the current date. 50 | * `-r --regression` + `dir` : only run the regression tests on output generated in a previous run. 51 | * `-c --compare` + `dir` : compare regression results in current output directory, and one generated in a previous run. 52 | 53 | The python script stays active until all submitted SLURM jobs have finished. This is strictly necessary only for handling regression tests after the jobs have finished, but the python script also handles proper closing of files. Thus it is a good idea to 54 | 55 | nohup python3 spawn.py myconf.txt & 56 | 57 | The configuration is specified split over the file on the commandline, and a `.spawnrc` file, which can be used for common options, such as your username, and the slurm account to bill your runs to. The current directory is search first for the `.spawnrc` file, and then the home directory. This makes it possible to have system dependent settings. Since configuration files and `rc` files have the exact same syntax, we will not distinguish between them, and mostly discuss the configuration file. 58 | 59 | ### Macros 60 | 61 | The configuration, specified in the file given on the commandline, is completely macro based. A macro is defined in a key/value line 62 | 63 | key value 64 | otherkey value1 value2 value3 65 | 66 | Macros keys are evaluated with a `%[macro]` syntax, because why not have yet another sytax? Example: 67 | 68 | key value 69 | otherkey another-%[key] 70 | 71 | This syntax can also be used to substitute environment variables, if the key is not explicitly defined. 72 | 73 | Some keys have special meanings; see below. 74 | 75 | The keyword `suite` is special in that it only defines a benchmark suite, but also triggers its execution. 76 | Thus, you can have multiple suites in one configuration file. Each is invoked with the current value of all macros, so you can redefine macros between suite invocations. See the example above, which uses different node and process counts for the point-to-point and collective tests. 77 | 78 | ### File structure 79 | 80 | Demonspawn generates output: 81 | 82 | * A single log file for the full configuration will be created in the current directory. It is identifiable by having the current date in the name. 83 | * An output directory is generated based on the required `outputdir` key. This will contain subdirectories `scripts` and `output` with the SLURM scripts and their standard out/err respectively. 84 | * If you do regression, the output directory will also contain a single regression file for each `suite` line. 85 | 86 | ## SLURM macros 87 | 88 | Some macros have special meaning for your SLURM script: 89 | 90 | * `system` is set to the current hostname. If you specify this macro, it is enforced that this run can only happen on that particular system. 91 | * `account` is used in your slurm script as the value of the `-A` or `--account` flag. 92 | * `modules` is the list of modules that is loaded at the beginning of your slurm script. The value of `%[modules]` has all spaces stripped. Special case: `modules restore foo` will cause the saveset `foo` to be restored. 93 | * `queue` is the value of the slurm `-p` flag: the partition, or queue, name where the jobs of the next suite will be submitted. The demonspawn manager will make sure that queue limits are not violated. The queue name has optional limits on the number of simultaneous jobs: 94 | 95 | `queue somequeue limit:2` 96 | 97 | Suggestion: specify queue limits in the `.spawnrc` file. The last specified queue will be used as the default, or you can explicitly choose a queue in the configuration file. 98 | 99 | * `time` is a `hh:mm:ss` specification for the slurm `-t` flag. 100 | 101 | It is possible to add custom `#SBATCH foo=bar` lines to a script. For this, put one or more lines 102 | 103 | sbatch foo=bar 104 | 105 | in the configuration. This the only option that is cumulative: these options are gathered up and used for each script. Thus they can not be reset between `suite` lines. 106 | 107 | ## Scaling setup 108 | 109 | For an MPI run you want to specify: 110 | 111 | * `nodes` : node count for the testsuite, unless the suite itself overrides this. This is either a single number or a comma-separated list, for scalability studies. 112 | * `ppn` : number of processes-per-node. A single number or a comma-separated list. 113 | * `threads` : OpenMP thread count. Single number or comma-separated list. A negative value indicates a thread count such that the product of MPI processes and OpenMP threads equals `SLURM_CPUS_ON_NODE`. (A zero value means that no threading is used; this value is ignored.) 114 | 115 | ## Suite setup 116 | 117 | Some macros related to running the benchmark programs. 118 | 119 | * `jobname` : this is by default "`spawn`". It is used for the name of the logfile. You can only once define this in your configuration. The logfile, by the way, has as time stamp, in case you re-use the output directory. 120 | * `env` : this is used to specify environment variables. At the moment this is strictly additive: each suite is started with the sum total of specified options at that point. Example: 121 | 122 | `env PETSC_OPTIONS -ksp_max_it 100 -ksp_monitor` 123 | 124 | You can have multiple test suites. A test suite is specified by the keyword: 125 | 126 | * `suite` : this is followed by a list of key:value pairs, followed by a list of programs, which can use wildcards 127 | 128 | Example: 129 | 130 | suite name:paw-mpi-p2p type:mpi dir:%[pawdir] p2p_* 131 | 132 | This line defines the suite with the current value of all macros. 133 | 134 | If there is more than one suite in a configuration file, each suite is fully finished before the next one is started. This is convenient if the suite runs a shell script that does a custom recompilation. You can redefine macros for the next suite in the same configuration file. 135 | 136 | The available keys are: 137 | 138 | * `name` : for identification purposes 139 | * `type` : choice of `seq` or `mpi`; MPI jobs are started with ibrun 140 | * `dir` : location of the programs 141 | 142 | After these pairs, the programs are specified with wildcards but no path. 143 | 144 | ## Directory macros 145 | 146 | It is required that you define this macro: 147 | 148 | * `outputdir` is the directory in which subdirectories `scripts`, `output`, `regression` are created. 149 | 150 | In order to create a unique output directory, the following macros are useful: 151 | 152 | * `pwd` is set to the current working directory where you are running python on the configuration file 153 | * `date` is set to current date-time. 154 | * `mpi` is set to `LMOD_FAMILY_COMPILER`. 155 | 156 | For example: 157 | 158 | outputdir %[pwd]/spawn-mycode-%[mpi]-%[date] 159 | 160 | ## Regression 161 | 162 | It is easy to run a regression on all output files of a suite by specifying the `regression` key. You can specify what to regress on: 163 | 164 | regression grep:Result 165 | 166 | This will grep through each result file in the suite. You can also 167 | 168 | regression line:last 169 | 170 | with possible options `first`, `last`. Since a regression definition stays in effect for subsequent jobsuites, you can disable a previously specified regression with 171 | 172 | regression none 173 | 174 | Regression results are written to a single file 175 | 176 | %[outputdir]/regression-%[suitename].txt 177 | 178 | Additionally, each job regression goes into a separate file 179 | 180 | %[outputdir]/regression/.out 181 | 182 | Note: the regression specification is part of the suite definition, so it needs to come *before* the `suite` line. 183 | 184 | Further options: 185 | 186 | * `field:5` extract only the 5-th whitespace-separated field; this numbering is 1-based 187 | * `label:abcd` put a label in front of the regression line. This can be a literal string, or a macro. If multiple `label` options are given, they are all used, in the sequence specified, separated by a space character. 188 | 189 | If you want to run a regression on already generated output, run the configuration again, but with the `-r` or `--regression` flag. 190 | 191 | You can compare the regressions of two runs by using the `-c old_output_dir` option. This will compare the files in the `regression` subdirectory, leaving the results in a file `regression_compare`. 192 | 193 | Normally, regression comparison results in both values being written to the `regression_compare` file. However, numerical comparison is enabled by having an option `margin:10percent` in the `regression` line. 194 | 195 | ## Limitations 196 | 197 | * Currently the software requires python version 3.8 or higher. 198 | * The `system` keyword only works at TACC 199 | * The `mpi` keyword depends on Lmod. 200 | 201 | ## Changelog 202 | 203 | 0.1 somewhere around `2021/12/01`: posted on reddit 204 | 205 | 0.2 `2022/02/15`: adding module restore, `regressiononly` option 206 | 207 | 0.3 logfile now goes into output dir, regression flag is now `regression` 208 | 209 | 0.4 regressions go into separate files, new flag for regression comparing -------------------------------------------------------------------------------- /docs/listingmacs.tex: -------------------------------------------------------------------------------- 1 | % -*- latex -*- 2 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 3 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 4 | %%%% 5 | %%%% This LaTeX file is part of the source of 6 | %%%% `Parallel Computing' 7 | %%%% by Victor Eijkhout, copyright 2018-2020 8 | %%%% 9 | %%%% listingmacs.tex : listing-related macros. 10 | %%%% 11 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 12 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 13 | 14 | \usepackage{listings,xcolor} 15 | \newcommand\lstineline{LST TYPO\n} 16 | 17 | \lstdefinestyle{reviewcode}{ 18 | belowcaptionskip=1\baselineskip, frame=L, 19 | breaklines=true, postbreak=\hbox{{$\hookrightarrow$}}, % \textcolor{green} 20 | xleftmargin=\unitindent, showstringspaces=false, 21 | basicstyle=\footnotesize\ttfamily, 22 | keywordstyle=\bfseries\color{blue}, 23 | commentstyle=\color{red!60!black}, 24 | identifierstyle=\slshape\color{black}, 25 | stringstyle=\color{green!60!black}, columns=fullflexible, 26 | keepspaces=true,tabsize=8, 27 | } 28 | 29 | \lstset{style=reviewcode} 30 | \newcommand\clstinline{\lstset{language=C}\lstinline} 31 | \newcommand\flstinline{\lstset{language=Fortran}\lstinline} 32 | \newcommand\plstinline{\lstset{language=Python}\lstinline} 33 | 34 | \lstset{emph={ %% MPI commands 35 | MPI_Something,MPI_Something_c,PMPI_Something,MPI_Ixxx,MPI_Xxx_init, 36 | MPI_Init,MPI_Initialized,MPI_Finalize,MPI_Finalized,MPI_Abort, 37 | MPI_Init_thread,MPI_Query_thread,MPI_Is_thread_main, 38 | MPI_Get_processor_name,MPI_Get_library_version, 39 | % comm stuff 40 | MPI_Comm_size,MPI_Comm_rank, 41 | MPI_Comm_dup,MPI_Comm_dup_with_info,MPI_Comm_idup,MPI_Comm_idup_with_info, 42 | MPI_Comm_split,MPI_Comm_split_type, 43 | MPI_Comm_remote_size,MPI_Comm_get_parent, 44 | MPI_Comm_connect,MPI_Comm_disconnect,MPI_Comm_accept, 45 | MPI_Comm_group,MPI_Comm_create,MPI_Comm_create_group,MPI_Comm_create_from_group, 46 | MPI_Comm_join,MPI_Comm_free,MPI_Comm_set_name, 47 | MPI_Comm_compare,MPI_Comm_get_info,MPI_Comm_set_info,MPI_Comm_get_attr, 48 | MPI_Intercomm_create,MPI_Intercomm_merge,MPI_Comm_test_inter,MPI_Comm_remote_group, 49 | MPI_Group_size,MPI_Group_rank,MPI_Group_free,MPI_Group_incl,MPI_Group_excl, 50 | MPI_Group_union,MPI_Group_intersection,MPI_Group_difference, 51 | % collectives 52 | MPI_Allreduce,MPI_Allreduce_c,MPI_Reduce,MPI_Reduce_c,MPI_Reduce_local,MPI_Bcast, 53 | MPI_Gather,MPI_Scatter,MPI_Scatterv,MPI_Gatherv,MPI_All...,MPI_...v, 54 | MPI_Allgather,MPI_Allgatherv,MPI_Alltoall,MPI_Alltoallv, 55 | MPI_Reduce_scatter,MPI_Reduce_scatter_block, 56 | MPI_Barrier_init,MPI_Bcast_init,MPI_Reduce_init,MPI_Allreduce_init,MPI_Reduce_scatter_init,MPI_Reduce_scatter_block_init,MPI_Gather_init,MPI_Gatherv_init,MPI_Allgather_init,MPI_Allgatherv_init,MPI_Scatter_init,MPI_Scatterv_init,MPI_Alltoall_init,MPI_Alltoallv_init,MPI_Alltoallw_init,MPI_Scan_init,MPI_Exscan_init,MPI_Neighbor_allgather_init,MPI_Neighbor_allgatherv_init,MPI_Neighbor_alltoall_init,MPI_Neighbor_alltoallv_init,MPI_Neighbor_alltoallw_init, 57 | MPI_Barrier,MPI_Scan,MPI_Exscan, 58 | MPI_Igather,MPI_Igatherv,MPI_Iallgather,MPI_Iallgatherv, 59 | MPI_Iscatter,MPI_Iscatterv, 60 | MPI_Ireduce,MPI_Iallreduce,MPI_Ireduce_scatter,MPI_Ireduce_scatter_block, 61 | MPI_Ialltoall,MPI_Ialltoallv,MPI_Ialltoallw, 62 | MPI_Ibcast,MPI_Ibarrier,MPI_Iexscan,MPI_Iscan,MPI_Isomething, 63 | % p2p 64 | MPI_Send,MPI_Send_c,MPI_Isend,MPI_Rsend,MPI_Irsend,MPI_Ssend,MPI_Issend, 65 | MPI_Bsend,MPI_Ibsend,MPI_Bsend_init,MPI_Cancel, 66 | MPI_Recv,MPI_Irecv,MPI_Mrecv,MPI_Sendrecv,MPI_Sendrecv_replace, 67 | MPI_Isendrecv,MPI_Isendrecv_replace,MPI_Sendrecv_init,MPI_Sendrecv_replace_init, 68 | MPI_Send_init,MPI_Ssend_init,MPI_Rsend_init,MPI_Recv_init,MPI_Start,MPI_Startall, 69 | MPI_Psend_init,MPI_Precv_init,MPI_Pready,MPI_Pready_range,MPI_Pready_list,MPI_Parrived, 70 | MPI_Wait,MPI_Wait...,MPI_Waitall,MPI_Waitany,MPI_Waitsome, 71 | MPI_Test,MPI_Test...,MPI_Testall,MPI_Testany,MPI_Testsome, 72 | MPI_Probe,MPI_Iprobe,MPI_Mprobe, 73 | MPI_Request_free,MPI_Request_get_status, 74 | MPI_Get_count,MPI_Get_count_c,MPI_Get_count_x, 75 | MPI_Get_elements,MPI_Get_elements_c,MPI_Get_elements_x,MPI_Get_address, 76 | MPI_Get_hw_resource_types, 77 | MPI_Buffer_attach,MPI_Buffer_detach, 78 | % operator 79 | MPI_Op_create,MPI_Op_free,MPI_Op_commutative, 80 | % type 81 | MPI_Type_commit,MPI_Type_free,MPI_Type_something, 82 | MPI_Type_contiguous,MPI_Type_vector,MPI_Type_xxx, 83 | MPI_Type_indexed,MPI_Type_hindexed,MPI_Type_create_hindexed,MPI_Type_create_hindexed_block, 84 | MPI_Type_create_subarray,MPI_Type_match_size,MPI_Type_create_resized, 85 | MPI_Type_struct,MPI_Type_create_struct, 86 | MPI_Type_create_f90_integer,MPI_Type_create_f90_real,MPI_Type_create_f90_complex, 87 | MPI_Type_size,MPI_Type_size_x,MPI_Type_extent,MPI_Type_lb,MPI_Type_ub, 88 | MPI_Type_get_extent,MPI_Type_get_true_extent, 89 | MPI_Type_get_extent_x,MPI_Type_get_true_extent_x, 90 | MPI_Pack,MPI_Unpack,MPI_Pack_size, 91 | % I/O 92 | MPI_File_open,MPI_File_close,MPI_File_delete,MPI_File_set_info, 93 | MPI_File_read,MPI_File_iread,MPI_File_read_all,MPI_File_iread_all, 94 | MPI_File_read_at,MPI_File_read_at_all,MPI_File_iread_at_all, 95 | MPI_File_read_all_begin,MPI_File_read_all_end, 96 | MPI_File_read_ordered,MPI_File_read_shared,MPI_File_iread_shared, 97 | MPI_File_write,MPI_File_iwrite,MPI_File_write_all,MPI_File_iwrite_all, 98 | MPI_File_write_at,MPI_File_write_at_all,MPI_File_iwrite_at_all, 99 | MPI_File_write_all_begin,MPI_File_write_all_end, 100 | MPI_File_write_shared,MPI_File_iwrite_shared,MPI_File_write_ordered, 101 | MPI_File_set_view,MPI_File_get_view,MPI_File_set_size,MPI_File_get_size, 102 | MPI_File_preallocate,MPI_File_seek,MPI_File_seek_shared, 103 | MPI_File_set_atomicity,MPI_File_sync, 104 | % one-sided 105 | MPI_Win_create,MPI_Win_create_dynamic,MPI_Win_allocate,MPI_Win_allocate_shared, 106 | MPI_Win_attach,MPI_Win_detach, 107 | MPI_Win_set_info,MPI_Win_get_info,MPI_Win_get_attr, 108 | MPI_Free_mem,MPI_Win_shared_query,MPI_Win_get_group, 109 | MPI_Win_free,MPI_Alloc_mem,MPI_Win_fence, 110 | MPI_Win_lock,MPI_Win_unlock,MPI_Win_lock_all,MPI_Win_unlock_all, 111 | MPI_Win_flush,MPI_Win_flush_all,MPI_Win_flush_local,MPI_Win_flush_local_all,MPI_Win_sync, 112 | MPI_Win_post,MPI_Win_start,MPI_Win_wait,MPI_Win_complete, 113 | MPI_Put,MPI_Rput,MPI_Get,MPI_Get_c,MPI_Rget,MPI_Accumulate,MPI_Raccumulate,MPI_Rget_accumulate, 114 | MPI_Get_accumulate,MPI_Fetch_and_op,MPI_Compare_and_swap, 115 | % topology 116 | MPI_Topo_test,MPI_Cart_create,MPI_Cart_coords,MPI_Cart_rank,MPI_Cart_shift, 117 | MPI_Dist_graph_create_adjacent,MPI_Dist_graph_create, 118 | MPI_Dist_graph_neighbors,MPI_Dist_graph_neighbors_count, 119 | MPI_Graph_create,MPI_Graph_get,MPI_Graphdims_get, 120 | MPI_Graph_neighbors,MPI_Graph_neighbors_count, 121 | MPI_Neighbor_...,MPI_Neighbor_allgather,MPI_Neighbor_allgatherv, 122 | MPI_Neighbor_alltoall,MPI_Neighbor_alltoallv,MPI_Neighbor_alltoallw, 123 | MPI_Ineighbor_allgather,MPI_Ineighbor_allgatherv, 124 | MPI_Ineighbor_alltoall,MPI_Ineighbor_alltoallv,MPI_Ineighbor_alltoallw, 125 | % process 126 | MPI_Comm_spawn,MPI_Comm_spawn_multiple, 127 | MPI_Open_port,MPI_Close_port,MPI_Publish_name,MPI_Unpublish_name,MPI_Lookup_name, 128 | % keyval 129 | MPI_Keyval_create,MPI_Keyval_free, 130 | MPI_Comm_create_keyval,MPI_Comm_free_keyval,MPI_Type_create_keyval,MPI_Type_free_keyval,MPI_Win_create_keyval,MPI_Win_free_keyval, 131 | % errors 132 | MPI_Add_error_class,MPI_Add_error_code,MPI_Add_error_string,MPI_Error_class, 133 | MPI_Comm_create_errhandler,MPI_Comm_set_errhandler,MPI_Comm_get_errhandler,MPI_Comm_call_errhandler, 134 | MPI_File_create_errhandler,MPI_File_set_errhandler,MPI_File_get_errhandler,MPI_File_call_errhandler, 135 | MPI_Session_create_errhandler,MPI_Session_set_errhandler,MPI_Session_get_errhandler,MPI_Session_call_errhandler, 136 | MPI_Win_create_errhandler,MPI_Win_set_errhandler,MPI_Win_get_errhandler,MPI_Win_call_errhandler, 137 | MPI_Errhandler_set,MPI_Errhandler_create,MPI_Errhandler_free, 138 | MPI_Errhandler_f2c,MPI_Errhandler_c2f, 139 | % utility and info 140 | MPI_Aint_add,MPI_Aint_diff, 141 | MPI_Get_version,MPI_Wtime,MPI_Wtick, 142 | MPI_Info_create,MPI_Info_free,MPI_Info_dup,MPI_Info_create_env, 143 | MPI_Info_set,MPI_Info_get,MPI_Info_get_string,MPI_Info_delete, 144 | MPI_Info_get_nkeys,MPI_Info_get_nthkey,MPI_Info_get_valuelen, 145 | MPI_Info_f2c,MPI_Info_c2f, 146 | MPI_Attr_get,MPI_Error_string,MPI_Sizeof, 147 | % sessions 148 | MPI_Session_init,MPI_Session_finalize, 149 | MPI_Group_from_session_pset,MPI_Session_get_num_psets,MPI_Session_get_nth_pset, 150 | MPI_Session_get_info,MPI_Session_get_pset_info, 151 | % tools interface 152 | MPI_Txxx,MPI_T_init_thread,MPI_T_finalize, 153 | MPI_T_cvar_get_num,MPI_T_cvar_get_info,MPI_T_cvar_get_index, 154 | MPI_T_cvar_handle_alloc,MPI_T_cvar_handle_free,MPI_T_cvar_read,MPI_T_cvar_write, 155 | MPI_T_pvar_get_num,MPI_T_pvar_get_info,MPI_T_pvar_get_index, 156 | MPI_T_pvar_session_create,MPI_T_pvar_session_free, 157 | MPI_T_pvar_handle_alloc,MPI_T_pvar_handle_free, 158 | MPI_T_pvar_start,MPI_T_pvar_stop,MPI_T_pvar_read,MPI_T_pvar_write, 159 | MPI_T_pvar_reset,MPI_T_pvar_readreset, 160 | MPI_T_category_get_num,MPI_T_category_get_info,MPI_T_category_get_index,MPI_T_category_changed, 161 | MPI_T_category_get_cvars,MPI_T_category_get_pvars,MPI_T_category_get_categories, 162 | %% OMP 163 | omp_set_num_threads,omp_get_num_threads,omp_get_max_threads,omp_get_thread_num, 164 | omp_get_num_procs,omp_in_parallel,omp_set_dynamic,omp_get_dynamic,omp_set_nested, 165 | omp_get_nested,omp_get_wtime,omp_get_wtick,omp_set_schedule,omp_get_schedule, 166 | omp_set_max_active_levels,omp_get_max_active_levels,omp_get_thread_limit, 167 | omp_get_level,omp_get_active_level,omp_get_ancestor_thread_num,omp_get_team_size, 168 | },emphstyle={\color{red!70!black}\bfseries} 169 | } 170 | \lstset{emph={[2] %% constants 171 | MPI_VERSION,MPI_SUBVERSION, 172 | MPI_THREAD_SINGLE,MPI_THREAD_FUNNELED,MPI_THREAD_SERIALIZED,MPI_THREAD_MULTIPLE, 173 | % NULLs and ERRORs 174 | MPI_ARGV_NULL,MPI_ARGVS_NULL,MPI_COMM_NULL,MPI_DATATYPE_NULL,MPI_INFO_NULL,MPI_INFO_ENV, 175 | MPI_OP_NULL,MPI_PROC_NULL,MPI_REQUEST_NULL, 176 | MPI_ERROR,MPI_SUCCESS,MPI_LASTUSEDCODE,MPI_ERR_LASTCODE, 177 | MPI_ERRORS_ARE_FATAL,MPI_ERRORS_RETURN,MPI_ERRORS_ABORT, 178 | MPI_ERRCODES_IGNORE, 179 | MPI_ERR_ARG,MPI_ERR_BUFFER,MPI_ERR_INTERN,MPI_ERR_INFO,MPI_ERR_IN_STATUS, 180 | MPI_ERR_OTHER,MPI_ERR_NO_MEM,MPI_ERR_PORT,MPI_ERR_PROC_ABORTED, 181 | MPI_ERR_SERVICE,MPI_ERR_COMM, 182 | % comm stuff 183 | MPI_COMM_WORLD,MPI_COMM_SELF,MPI_ROOT, 184 | MPI_COMM_TYPE_SHARED,MPI_COMM_TYPE_HW_GUIDED,MPI_COMM_TYPE_HW_UNGUIDED, 185 | MPI_IDENT,MPI_CONGRUENT,MPI_SIMILAR,MPI_UNEQUAL, 186 | % stuff 187 | MPI_MAX_DATAREP_STRING,MPI_MAX_ERROR_STRING, 188 | MPI_MAX_INFO_KEY,MPI_MAX_INFO_VAL, MPI_MAX_LIBRARY_VERSION_STRING, 189 | MPI_MAX_OBJECT_NAME,MPI_MAX_PORT_NAME,MPI_MAX_PROCESSOR_NAME, 190 | MPI_WTIME_IS_GLOBAL, MPI_IN_PLACE,MPI_REPLACE, 191 | MPI_STATUS_IGNORE,MPI_STATUSES_IGNORE,MPI_STATUS_SIZE, 192 | MPI_TAG,MPI_SOURCE,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_TAG_UB, 193 | MPI_UNIVERSE_SIZE,MPI_APPNUM,MPI_HOST,MPI_IO, 194 | % operators 195 | MPI_SUM,MPI_PROD,MPI_MAX,MPI_MIN,MPI_MAXLOC,MPI_MINLOC,MPI_NO_OP, 196 | MPI_LAND,MPI_LOR,MPI_LXOR,MPI_BAND,MPI_BOR,MPI_BXOR, 197 | % types 198 | MPI_SHORT,MPI_UNSIGNED_SHORT,MPI_UNSIGNED,MPI_LONG,MPI_UNSIGNED_LONG, 199 | MPI_INT,MPI_INTEGER,MPI_INTEGER1,MPI_INTEGER2,MPI_INTEGER4,MPI_INTEGER8,MPI_INTEGER16, 200 | MPI_FLOAT,MPI_DOUBLE,MPI_LONG_DOUBLE,MPI_REAL,MPI_DOUBLE_PRECISION, 201 | MPI_REAL2,MPI_REAL4,MPI_REAL8,MPI_REAL16, 202 | MPI_COMPLEX,MPI_DOUBLE_COMPLEX,MPI_LOGICAL, 203 | MPI_FLOAT_INT,MPI_LONG_INT,MPI_LONG_LONG_INT,MPI_AINT,MPI_DOUBLE_INT,MPI_SHORT_INT, 204 | MPI_2INT,MPI_2INTEGER,MPI_LONG_DOUBLE_INT,MPI_2REAL,MPI_2DOUBLE_PRECISION, 205 | % -- kinds 206 | MPI_ADDRESS_KIND,MPI_COUNT_KIND,MPI_INTEGER_KIND,MPI_OFFSET_KIND, 207 | MPI_SUBARRAYS_SUPPORTED,MPI_ASYNC_PROTECTS_NONBLOCKING, 208 | MPI_CHAR,MPI_CHARACTER,MPI_SIGNED_CHAR,MPI_UNSIGNED_CHAR, 209 | MPI_BYTE,MPI_PACKED, MPI_MODE_CREATE, 210 | % file modes 211 | MPI_SEEK_SET, 212 | MPI_MODE_RDONLY,MPI_MODE_WRONLY,MPI_MODE_RDWR,MPI_MODE_EXCL,MPI_MODE_DELETE_ON_CLOSE, 213 | MPI_MODE_UNIQUE_OPEN,MPI_MODE_SEQUENTIAL,MPI_MODE_APPEND, 214 | MPI_MODE_NOPUT,MPI_MODE_NOPRECEDE,MPI_MODE_NOSUCCEED,MPI_MODE_NOSTORE, 215 | MPI_MODE_NOCHECK, MPI_LOCK_EXCLUSIVE,MPI_LOCK_SHARED, 216 | MPI_SEEK_SET,MPI_SEEK_CUR,MPI_SEEK_END, 217 | % other 218 | MPI_TYPECLASS_REAL,MPI_TYPECLASS_INTEGER,MPI_TYPECLASS_COMPLEX, 219 | MPI_UB,MPI_BOTTOM,MPI_BSEND_OVERHEAD,MPI_DISPLACEMENT_CURRENT,MPI_KEYVAL_INVALID, 220 | MPI_UNDEFINED,MPI_CART,MPI_GRAPH,MPI_DIST_GRAPH, 221 | MPI_UNWEIGHTED,MPI_WEIGHTS_EMPTY,MPI_ORDER_C,MPI_ORDER_FORTRAN, 222 | MPI_WIN_BASE,MPI_WIN_SIZE,MPI_WIN_DISP_UNIT, 223 | MPI_WIN_CREATE_FLAVOR,MPI_WIN_FLAVOR_CREATE,MPI_WIN_FLAVOR_ALLOCATE,MPI_WIN_FLAVOR_DYNAMIC,MPI_WIN_FLAVOR_SHARED, 224 | MPI_WIN_MODEL,MPI_WIN_SEPARATE,MPI_WIN_UNIFIED, 225 | % tools interface 226 | MPI_T_ENUM_NULL,MPI_T_BIND_NO_OBJECT,MPI_T_ERR_INVALID_NAME,MPI_T_ERR_INVALID_INDEX, 227 | MPI_T_PVAR_CLASS_STATE,MPI_T_PVAR_CLASS_LEVEL,MPI_T_PVAR_CLASS_SIZE,MPI_T_PVAR_CLASS_PERCENTAGE,MPI_T_PVAR_CLASS_HIGHWATERMARK,MPI_T_PVAR_CLASS_LOWWATERMARK,MPI_T_PVAR_CLASS_COUNTER,MPI_T_PVAR_CLASS_AGGREGATE,MPI_T_PVAR_CLASS_TIMER,MPI_T_PVAR_CLASS_GENERIC, 228 | MPI_T_PVAR_SESSION_NULL,MPI_T_ERR_INVALID_HANDLE,MPI_T_PVAR_HANDLE_NULL, 229 | MPI_T_PVAR_ALL_HANDLES,MPI_T_ERR_PVAR_NO_STARTSTOP,MPI_T_ERR_PVAR_NO_WRITE, 230 | %% OMP environment variables 231 | OMP_CANCELLATION,OMP_DISPLAY_ENV,OMP_DEFAULT_DEVICE,OMP_DYNAMIC,OMP_MAX_ACTIVE_LEVELS, 232 | OMP_MAX_TASK_PRIORITY,OMP_NESTED,OMP_NUM_THREADS,OMP_PROC_BIND,OMP_PLACES,OMP_STACKSIZE, 233 | OMP_SCHEDULE,OMP_THREAD_LIMIT,OMP_WAIT_POLICY, 234 | },emphstyle={[2]\color{green!40!black}} 235 | } 236 | 237 | \lstset{emph={[3] %% types 238 | MPI_Aint,MPI_Comm,MPI_Count,MPI_Datatype,MPI_Errhandler,MPI_File,MPI_Group, 239 | MPI_Info,MPI_Message,MPI_Offset,MPI_Op,MPI_Request,MPI_Session,MPI_Status, 240 | MPI_User_function,MPI_Win, 241 | MPI_T_enum,MPI_T_cvar_handle,MPI_T_pvar_session,MPI_T_pvar_handle, 242 | %% OMP directives 243 | for,parallel,sections,section,single,task,taskgroup,workshare 244 | %% MPL methods 245 | environment,processor_name,communicator,comm_world,comm_self,split,split_shared, 246 | rank,size,tag,any,up,get_count,any_source,source, 247 | wtime,wtick,wtime_is_global, 248 | allreduce,bcast,gather,scatter, 249 | send,recv,send_init,sendrecv, 250 | bsend_buffer,bsend_size,bsend_init,bsend_overhead,buffer_attach,buffer_detach, 251 | irequest,prequest,irequest_pool,prequest_pool,status, 252 | wait,waitany,waitall,waitsome,testany,testsome,testall, 253 | contiguous_layout,vector_layout,strided_vector_layout,iterator_layout,subarray_layout,indexed_layout,indexed_block_layout,heterogeneous_layout,absolute,make_absolute,layouts,resize, 254 | threading_modes,single,funneled,serialized,multiple, 255 | %% SYCL methods 256 | host_selector,submit,get_access,accessor, 257 | },emphstyle={[3]\color{yellow!20!brown}\bfseries}, 258 | emph={[4] %% PETSc functions 259 | AnyPetscRoutine, 260 | PetscInitialize,PetscInitializeFortran,PetscFinalize,PetscSplitOwnership, 261 | PetscSynchronizedPrintf,PetscPrintf,PetscSynchronizedFlush, 262 | PetscSplitOwnership,PetscNew,PetscFree,PetscBLASIntCast, 263 | PetscFunctionBegin,PetscFunctionReturn,PetscFunctionBeginUser, 264 | PetscRealPart,PetscImaginaryPart, 265 | PetscLogView, 266 | PetscLogEventRegister,PetscLogEventBegin,PetscLogEventEnd,PetscLogFlops, 267 | PetscLogStagePush,PetscLogStagePop,PetscLogStageRegister, 268 | PetscMalloc,PetscMalloc1,PetscMallocDump,PetscCalloc1, 269 | PetscTime,PetscGetCPUTime, 270 | %% Viewer 271 | AOViewFromOptions,DMViewFromOptions,ISViewFromOptions,ISLocalToGlobalMappingViewFromOptions,KSPConvergedReasonViewFromOptions,KSPViewFromOptions,MatPartitioningViewFromOptions,MatCoarsenViewFromOptions,MatViewFromOptions,PetscObjectViewFromOptions,PetscPartitionerViewFromOptions,PetscDrawViewFromOptions,PetscRandomViewFromOptions,PetscDualSpaceViewFromOptions,PetscSFViewFromOptions,PetscFEViewFromOptions,PetscFVViewFromOptions,PetscSectionViewFromOptions,PCViewFromOptions,PetscSpaceViewFromOptions,PFViewFromOptions,PetscLimiterViewFromOptions,PetscLogViewFromOptions,PetscDSViewFromOptions,PetscViewerViewFromOptions,SNESConvergedReasonViewFromOptions,SNESViewFromOptions,TSTrajectoryViewFromOptions,TSViewFromOptions,TaoLineSearchViewFromOptions,TaoViewFromOptions,VecViewFromOptions,VecScatterViewFromOptions, 272 | PetscViewerCreate,PetscViewerSetType,PetscViewerDestroy, 273 | PetscViewerRead,PetscViewerPushFormat,PetscViewerPopFormat, 274 | %% DM and such 275 | DMDACreate1d,DMDACreate2d,DMDACreate3d,DMCreateMatrix,DMSetUp, 276 | DMDAGetLocalInfo,DMDAGetCorners,DMSetUp,DMCreateMatrix, 277 | DMDACreate, % generic, for text only 278 | DMCreateGlobalVector,DMCreateLocalVector,DMDAGetCorners, 279 | DMGlobalToLocal,DMGlobalToLocalBegin,DMGlobalToLocalEnd, 280 | DMLocalToGlobal,DMLocalToGlobalBegin,DMLocalToGlobalEnd, 281 | MatCreate,MatDestroy,MatCreateSubMatrix,MatCreateSubMatrices, 282 | MatCreateFFT,MatCreateVecs, 283 | MatCreateSeqAIJWithArrays,MatCreateMPIAIJWithArrays, 284 | MatSetSizes,MatSizes,MatSetType,MatSetPreallocation, 285 | MatSetValue,MatSetValues,MatSetValuesStencil,MatAssemblyBegin,MatAssemblyEnd, 286 | MatCreateShell,MatShellSetOperation,MatShellSetContext,MatShellGetContext, 287 | MatSeqAIJSetPreallocation,MatMPIAIJSetPreallocation, 288 | MatSetOption, 289 | MatGetSize,MatGetLocalSize,MatGetOwnershipRange, 290 | MatGetRow,MatRestoreRow,MatGetArray,MatRestoreArray, 291 | MatMult,MatMultTranspose,MatMultHermitianTranspose, 292 | MatAXPY,MatMultAdd,MatRealPart,MatImaginaryPart, 293 | MatPartitioningCreate,MatPartitioningDestroy,MatPartitioningSetType,MatPartitioningApply, 294 | KSPCreate,KSPDestroy,KSPSetTolerances,KSPSetType,KSPSetFromOptions, 295 | KSPSetOperators,KSPGetOperators,KSPSetTolerances,KSPGMRESSetRestart, 296 | KSPSolve,KSPMatSolve,KSPGetIterationNumber, 297 | KSPGetConvergedReason,KSPConvergenceReasonView,KSPReasonView, 298 | KSPGetPC,PCSetType,PCHYPRESetType, 299 | KSPSetConvergenceTest,KSPConvergedDefault, 300 | KSPMonitorSet,KSPMonitorDefault,KSPMonitorTrueResidualNorm,KSPMonitorTrueResidualNorm, 301 | KSPGetSolution,KSPGetRhs,KSPBuildSolution,KSPBuildResidual, 302 | PCFactorSetMatSolverType,PCFactorSetLevels, 303 | PCCompositeSetType,PCCompositeAddPC, 304 | PCShellSetApply,PCShellSetContext,PCShellGetContext,PCShellSetSetUp, 305 | VecCreate,VecDestroy,VecDestroyVecs,VecDuplicate,VecDuplicateVecs,VecXYZ, 306 | VecCreateSeqWithArray,VecCreateMPIWithArray, 307 | VecSetType,VecSetSizes,VecGetSize,VecGetLocalSize,VecGetOwnershipRange, 308 | VecAXPY,VecAYPX,VecMAXPY,VecCopy,VecDot,VecMDot,VecDotBegin,VecDotEnd, 309 | VecNorm,VecNormBegin,VecNormEnd,VecScale,VecMax,VecMin,VecReciprocal,VecShift, 310 | VecSum,VecSwap,VecPointwiseMult,VecPointwiseDivide, 311 | VecSet,VecSetValue,VecSetValues,VecAssemblyBegin,VecAssemblyEnd, 312 | VecGetArray,VecRestoreArray,VecGetArrayRead,VecRestoreArrayRead, 313 | VecGetArrayF90,VecRestoreArrayF90,VecPlaceArray,VecReplaceArray,VecResetArray, 314 | VecRealPart,VecImaginaryPart, 315 | ISCreate,ISSetType,ISDestroy,ISCreateBlock,ISCreateGeneral,ISCreateStride, 316 | ISGetIndices,ISRestoreIndices, 317 | VecScatterCreate,VecScatterDestroy,VecScatterBegin,VecScatterEnd, 318 | KSPView,MatView,VecView,VecViewFromOptions, 319 | PetscOptionsBegin,PetscOptionsEnd,PetscOptionsInt,PetscOptionsGetInt,PetscOptionsSetValue, 320 | PetscOptionsHasName, 321 | KSPSetOptionsPrefix,MatSetOptionsPrefix,PCSetOptionsPrefix,PetscObjectSetOptionsPrefix,PetscViewerSetOptionsPrefix,SNESSetOptionsPrefix,TSSetOptionsPrefix,VecSetOptionsPrefix, 322 | % GPU / CUDA stuff 323 | PetscCUDAInitialize,PetscCUDAInitializeCheck, 324 | PetscMallocSetCUDAHost,PetscMallocResetCUDAHost, 325 | MatCreateDenseCUDA,MatCreateSeqDenseCUDA,MatDenseCUDAGetArray, 326 | VecCreateSeqCUDA,VecCreateMPICUDAWithArray,VecCUDAGetArray, 327 | },emphstyle={[4]\color{red!70!black}\bfseries}, 328 | emph={[5] %% PETSc keywords and macros 329 | ADD_VALUES,INSERT_VALUES,MPIU_REAL,MPIU_SCALAR,MPIU_COMPLEX, 330 | DM_BOUNDARY_NONE,DM_BOUNDARY_GHOSTED,DM_BOUNDARY_PERIODIC,DM_STENCIL_STAR,DM_STENCIL_BOX, 331 | KSPCG,KSPPIPECG,KSPGMRES,KSPBCGS,KSPPREONLY, 332 | MATMPIDENSE,MATMPIAIJ,MATSEQDENSE,MATSEQAIJ,MATMPIDENSECUDA,MATDENSECUDA,MATAIJCUSPARSE, 333 | MAT_FINAL_ASSEMBLY,MAT_FLUSH_ASSEMBLY, 334 | MAT_NEW_NONZERO_LOCATIONS,MAT_NEW_NONZERO_LOCATION_ERR, 335 | NORM_1,NORM_2,NORM_INF, 336 | PCASM,PCBJACOBI,PCGAMG,PCHYPRE,PCILU,PCLU,PCMG,PCSPAI, 337 | PCCOMPOSITE,PC_COMPOSITE_MULTIPLICATIVE, 338 | PETSC_DIR,PETSC_ARCH,PETSC_CC_INCLUDES,PETSC_FC_INCLUDES, 339 | PETSC_COMM_WORLD,PETSC_COMM_SELF,PETSC_i, 340 | PETSC_FALSE,PETSC_TRUE,PETSC_DECIDE,PETSC_DETERMINE,PETSC_DEFAULT, 341 | PETSC_ERR_ARG_OUTOFRANGE,PETSC_MEMALIGN, 342 | PETSC_NULL,PETSC_NULL_CHARACTER,PETSC_NULL_INTEGER,PETSC_NULL_OBJECT, 343 | PETSC_NULL_IS,PETSC_NULL_VIEWER, 344 | PETSC_STDOUT, 345 | PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO_DETAIL, 346 | PETSCVIEWERASCII,PETSCVIEWERBINARY,PETSCVIEWERSTRING,PETSCVIEWERDRAW, 347 | PETSCVIEWERSOCKET,PETSCVIEWERHDF5,PETSCVIEWERVTK, 348 | SCATTER_FORWARD,SCATTER_REVERSE, 349 | VECSEQ,VECMPI,VECCUDA,VECSEQCUDA,VECMPICUDA, 350 | CHKERRQ,SETERRQ,SETERRQ1,SETERRQ2,CHKERRA,SETERRA,CHKMEMQ,CHKMEMA, 351 | % python 352 | PETSc,DECIDE, 353 | },emphstyle={[5]\color{green!60!black}}, 354 | emph={[6] %% PETSc types 355 | PetscDataType,PetscBool,PetscComplex,PetscReal,PetscScalar,PetscLogDouble,PetscErrorCode, 356 | PetscInt,PetscBLASInt,PetscMPIInt,PetscViewer, 357 | DM,DMDA,DMBoundaryType,DMStencilType,DMDALocalInfo, 358 | KSP,KSPConvergedReason, 359 | Mat,MatStencil,MatPartitioning,PC,PCCompositeType, 360 | Vec,IS,VecScatter, 361 | },emphstyle={[6]\bfseries\color{blue}}, 362 | } 363 | -------------------------------------------------------------------------------- /jobsuite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Demonspawn 4 | # a utility for quickly generating a slew of batch jobs 5 | # good for benchmarking, regression testing, and such 6 | # 7 | # Victor Eijkhout 8 | # copyright 2020-2022 9 | # 10 | # version 0.5, see the Readme for details 11 | # 12 | # jobsuite.py : classes for suites and jobs 13 | # 14 | 15 | import copy 16 | import datetime 17 | from functools import reduce 18 | import io 19 | import os 20 | import re 21 | import sys 22 | import subprocess as sp 23 | import time 24 | 25 | 26 | def DefaultModules(): 27 | return "intel/18.0.2" 28 | 29 | def module_string(txt): 30 | # make module list from string 31 | txt = txt.split(" ") 32 | # slashes -> dashes in each module 33 | txt = [ re.sub('/','-',m) for m in txt ] 34 | # join modules with underscore for configuration name 35 | txt = '_'.join( txt ) 36 | return txt 37 | 38 | def macro_value( m,macros ): 39 | if m in macros.keys(): 40 | return str(macros[m]) 41 | elif m in os.environ.keys(): 42 | return os.environ[m] 43 | else: 44 | return m 45 | 46 | def macros_substitute(line,macros): 47 | print(f"macro substitution in <<{line}>>") 48 | subline = line 49 | while True: 50 | if m_match := re.search(r'\%\[([^[]+)\]',subline): 51 | macro = m_match.groups()[0] 52 | print(f".. found macro <<{macro}>>") 53 | value = macro_value(macro,macros) 54 | subline = re.sub(r'\%\['+macro+r'\]',value,subline) 55 | print(f".. giving <<{subline}>>") 56 | else: break 57 | # for m in macros.keys(): 58 | # m_search = r'\%\[{}\]'.format(m) 59 | # if re.search(m_search,line): 60 | # replacement_text = macro_value( m,macros ) 61 | # if m=="modules": 62 | # replacement_text = module_string(replacement_text) 63 | # subline = re.sub( m_search, replacement_text, subline ) 64 | return subline 65 | 66 | ## 67 | ## files may not be unique per job 68 | ## so we need central bookkeeping 69 | ## this is a singleton class 70 | ## 71 | class SpawnFiles(): 72 | instance = None 73 | class __spawnfiles(): 74 | def __init__(self): 75 | self.file_handles = {}; self.file_names = {} 76 | self.outputdir = None 77 | self.debug = False 78 | def debug_print(self,msg): 79 | if self.debug: 80 | print(msg) 81 | if logfile := self.get_by_key("logfile"): 82 | logfile.write(msg+"\n") 83 | def setoutputdir(self,dir): 84 | if not os.path.exists(dir): 85 | self.debug_print(f"Creating output directory <<{dir}>>") 86 | os.mkdir(dir) 87 | else: 88 | self.debug_print(f"Re-using old output directory <<{dir}>>") 89 | self.outputdir = dir 90 | def ensurefiledir(self,dir=None,subdir=None): 91 | if dir: 92 | filedir = dir 93 | else: 94 | filedir = self.outputdir 95 | if not filedir: 96 | raise Exception("No output dir specified, in call or configuration") 97 | if subdir: filedir=f"{filedir}/{subdir}" 98 | try : 99 | os.mkdir( filedir ) 100 | self.debug_print(f"Making dir <<{filedir}>>") 101 | except FileExistsError : 102 | self.debug_print(f"Directory <<{filedir}>> already exists") 103 | pass 104 | return filedir 105 | def open(self,fil,key=None,dir=None,subdir=None,new=False): 106 | ### return handle, dirname, filename, key 107 | filedir = self.ensurefiledir(dir,subdir) 108 | filename = fil 109 | if not key: key = filename 110 | self.debug_print(f"Opening dir={filedir} file={filename} key={key}") 111 | fullname = f"{filedir}/{filename}" 112 | if key not in self.file_handles.keys(): 113 | h = open(fullname,"w") 114 | self.file_handles[key] = h; self.file_names[key] = fullname 115 | return h,filedir,filename,key 116 | elif new: 117 | raise Exception(f"Key <<{key}>> File <<{fullname}>> already exists") 118 | else: 119 | return self.file_handles[key],filedir,filename,key 120 | def open_new(self,fil,key=None,dir=None,subdir=None): 121 | self.debug_print(f"Open new <{fil}>> at <<{dir}/{subdir}>>") 122 | return self.open(fil,key=key,dir=dir,subdir=subdir,new=True) 123 | def get(self,id): 124 | return self.file_handles[id] 125 | def close_by_path(self,path): 126 | for fk in self.file_names.keys(): 127 | if self.file_names[fk]==path: 128 | self.file_handles[fk].close() 129 | def close_files(self,keys): 130 | for k in keys: 131 | if k is None or k not in self.file_handles.keys(): 132 | self.debug_print(f"Suspicious attempt to close {k}") 133 | else: 134 | self.debug_print(f"closing job: {k} => {self.file_names[k]}") 135 | self.file_handles[k].close() 136 | self.file_handles.pop(k,None) 137 | self.file_names.pop(k,None) 138 | def __del__(self): 139 | for f in self.file_handles.keys(): 140 | self.debug_print(f"closing file: {f}") 141 | self.file_handles[f].close() 142 | def __new__(cls): 143 | if not SpawnFiles.instance: 144 | SpawnFiles.instance = SpawnFiles.__spawnfiles() 145 | return SpawnFiles.instance 146 | def __getattr__(self,attr): 147 | return self.instance.__getattr__(attr) 148 | 149 | def regression_test_dict(regression): 150 | ## split `regression' clause, return dict 151 | rtest = {} 152 | for kv in regression.split(): 153 | if not re.search(":",kv): 154 | print(f"ill-formed regression clause <<{kv}>>") 155 | continue 156 | k,v = kv.split(":") 157 | if k=="label": 158 | if k not in rtest.keys(): 159 | rtest[k] = [] 160 | rtest[k].append(v) 161 | else: 162 | rtest[k] = v 163 | return rtest 164 | 165 | class Job(): 166 | def __init__(self,configuration,**kwargs): 167 | 168 | self.configuration = configuration 169 | for key in [ "account", "queue", "sbatch", "user", "time", ]: 170 | try : 171 | self.__dict__[key] = self.configuration[key] 172 | except KeyError: 173 | print(f"\nConfiguration does not have required key <<{key}>>\n") 174 | sys.exit(1) 175 | 176 | self.suite = "paw" 177 | self.nodes = 1; self.cores = 10; self.ppn = 1; self.threads = 0 178 | self.unique_name = None 179 | 180 | self.time = "01:00:00" 181 | self.runner = "./" 182 | self.trace = False; self.debug = False 183 | self.logfile,_,_,_ = SpawnFiles().open("logfile") 184 | self.macros = copy.copy( kwargs.pop("macros",{}) ) 185 | self.set_has_not_been_submitted() 186 | 187 | tracestring = "" 188 | forbidden = [ "logfile","macros", ] 189 | for key,val in kwargs.items(): 190 | if key in forbidden: 191 | continue 192 | tracestring += " {}={}".format(key,val) 193 | self.__dict__[key] = val 194 | self.macros[key] = val 195 | 196 | self.cores = int( self.macros["nodes"] ) * int( self.macros["ppn"] ) 197 | self.macros["cores"] = self.cores 198 | if not self.unique_name: raise Exception(f"Missing key: unique_name") 199 | tracestring = f"Creating job <<{self.unique_name}>> with <<{tracestring}>>" 200 | 201 | script_file_name = f"{self.unique_name}.script" 202 | script_file_handle,scriptdir,script_file_name,_ \ 203 | = SpawnFiles().open_new( script_file_name,subdir="scripts" ) 204 | self.script_file_name = f"{scriptdir}/{script_file_name}" 205 | self.slurm_output_file_name = f"{self.outputdir}/{self.unique_name}.out" 206 | script_file_handle.write(self.script_contents()+"\n") 207 | script_file_handle.close() 208 | self.logfile.write(f""" 209 | %%%%%%%%%%%%%%%% 210 | {self.count:3}: script={self.script_file_name} 211 | logout={self.slurm_output_file_name} 212 | """) 213 | if self.trace: 214 | print(f"Written job file <<{self.script_file_name}>> for <<{self.unique_name}>>") 215 | if self.regression and not self.global_regression_handle: 216 | raise Exception("Trying to create regression job without global regressionfile") 217 | ## if self.trace: print(tracestring) 218 | self.logwrite(tracestring) 219 | def logwrite(self,msg): 220 | if self.logfile: 221 | self.logfile.write(msg+"\n") 222 | def modules_load_line(self): 223 | if self.modules!="default": 224 | return f"""## custom modules 225 | module reset 226 | module load {self.modules} 227 | """ 228 | else: return "" 229 | def omp_thread_spec(self): 230 | if self.threads==0: 231 | return "" 232 | else: 233 | if self.threads>0: 234 | threadcount = self.threads 235 | else: 236 | threadcount = "$(( SLURM_CPUS_ON_NODE / SLURM_NTASKS * SLURM_NNODES ))" 237 | return f"""## OpenMP thread specification 238 | threadcount={threadcount} 239 | if [ $threadcount -lt 1 ] ; then threadcount=1 ; fi 240 | export OMP_NUM_THREADS=$threadcount 241 | export OMP_PROC_BIND=true 242 | """ 243 | def script_contents(self): 244 | bench_program = self.runner+self.programdir+"/"+self.unique_name 245 | module_spec = self.modules_load_line() 246 | thread_spec = self.omp_thread_spec() 247 | sbatch = "" 248 | for s in self.sbatch: 249 | sbatch += f"""#SBATCH {s} 250 | """ 251 | return \ 252 | f"""#!/bin/bash 253 | #SBATCH -J {self.unique_name} 254 | #SBATCH -o {self.slurm_output_file_name} 255 | #SBATCH -e {self.slurm_output_file_name} 256 | #SBATCH -p {self.queue} 257 | #SBATCH -t {self.time} 258 | #SBATCH -N {self.nodes} 259 | #SBATCH --tasks-per-node {self.ppn} 260 | #SBATCH -A {self.account} 261 | {sbatch} 262 | 263 | {module_spec}{thread_spec} 264 | cd {self.outputdir} 265 | program={self.programdir}/{self.program_name} 266 | if [ ! -f "$program" ] ; then 267 | echo "Program does not exist: $program" 268 | exit 1 269 | fi 270 | {self.runner}$program 271 | """ 272 | def nodespec(self): 273 | if self.threads>0: 274 | thread_spec = f"-t{self.threads}" 275 | elif self.threads<0: 276 | thread_spec = f"-tx" 277 | else: 278 | thread_spec = "" 279 | return f"N{self.nodes}-ppn{self.ppn}{thread_spec}" 280 | def __str__(self): 281 | return f"{self.unique_name} N={self.nodes} cores={self.cores} threads={self.threads} regression={self.regression}" 282 | def submit(self): 283 | if self.trace: 284 | print(f"sbatch: {self.script_file_name}") 285 | p = sp.Popen(["sbatch",self.script_file_name],stdout=sp.PIPE) 286 | submitted = False 287 | for line in io.TextIOWrapper(p.stdout, encoding="utf-8"): 288 | line = line.strip() 289 | if False and ( self.trace or self.debug ): 290 | print( line ) 291 | self.logfile.write(line+"\n") 292 | submitted = re.search("(Submitted.* )([0-9]+)",line) 293 | if submitted: 294 | id = submitted.groups()[1] 295 | self.set_has_been_submitted(id) 296 | return self.jobid 297 | if not submitted: 298 | raise Exception(f"Failure to submit <<{self.script_file_name}>>") 299 | return 0 300 | def set_has_not_been_submitted(self): 301 | self.jobid = "1"; self.status = "PRE" 302 | def get_has_been_submitted(self): 303 | # meaning: submitted or running or finished 304 | return self.status!="PRE" 305 | def set_has_been_submitted(self,id): 306 | self.status = "PD"; self.jobid = id 307 | self.logfile.write(f"Status to pending, id={id}") 308 | if re.search("%j",self.slurm_output_file_name): 309 | self.slurm_output_file_name = re.sub("%j",self.jobid,self.slurm_output_file_name) 310 | self.logfile.write(f", output file name set to {self.slurm_output_file_name}") 311 | self.logfile.write("\n") 312 | def status_update(self,status): 313 | if status!="NS": 314 | # job was found in slurm, status is PD or R or CG 315 | self.status = status 316 | else: 317 | # job not found in slurm: either not scheduled, or already finished 318 | if self.jobid!="1": 319 | # it has an actual id 320 | if not self.done_running(): 321 | self.set_done_running() 322 | def is_running(self): 323 | return self.jobid!="1" and self.status=="R" 324 | def is_pending(self): 325 | return self.jobid!="1" and self.status=="PD" 326 | def done_running(self): 327 | return self.status=="POST" 328 | def set_done_running(self): 329 | self.status = "POST" # done running 330 | ## filter crud from output file 331 | with open(self.slurm_output_file_name,"r") as slurm_out: 332 | lines = slurm_out.readlines() 333 | with open(self.slurm_output_file_name,"w") as slurm_out: 334 | for line in lines: 335 | if not re.match("TACC",line): 336 | slurm_out.write(line) 337 | ## regression 338 | if self.regression: 339 | self.do_regression() 340 | def get_status(self): 341 | id = self.jobid 342 | # squeue -j 6137988 -h -o "%t" 343 | p = sp.Popen(["squeue","-j",id,"-h","-o","%t"],stdout=sp.PIPE) 344 | status = "CD" 345 | for status in io.TextIOWrapper(p.stdout, encoding="utf-8"): 346 | status = status.strip() 347 | return status 348 | def regression_line_pick_field(self,line,rtest): 349 | if "field" in rtest.keys(): 350 | fields = line.split(); ifield = rtest["field"] 351 | try: 352 | return fields[ int(ifield)-1 ] 353 | except: 354 | error = f"ERROR Can not extract field {ifield} from <<{line}>>" 355 | print(error) 356 | return None 357 | else: return line 358 | def regression_label_prepend(self,string,rtest): 359 | if "label" in rtest.keys(): 360 | labels = "" 361 | for l in rtest["label"]: 362 | if labels=="": 363 | labels = macro_value( l, self.macros ) 364 | else: 365 | labels = labels+" "+macro_value( l, self.macros ) 366 | return f"{labels} {string}" 367 | else: return string 368 | def regression_grep(self,output_file,rtest): 369 | unique_name = self.unique_name 370 | greptext = re.sub("_"," ",rtest["grep"]) 371 | found = False; rkey = None 372 | for line in output_file: 373 | line = line.strip() 374 | if re.search(greptext,line): 375 | found = True 376 | string = self.regression_line_pick_field(line,rtest) 377 | string = self.regression_label_prepend(string,rtest) 378 | self.logfile.write(f"File: {self.unique_name}\n{string}\n") 379 | self.global_regression_handle.write(f"File: {self.unique_name} Result: {string}\n") 380 | rfile,_,_,rkey = SpawnFiles().open\ 381 | (f"{self.unique_name}.out",subdir="regression",key=f"r-{unique_name}") 382 | rfile.write(f"{string}\n") 383 | break 384 | if not found: 385 | self.logfile.write\ 386 | (f"{self.unique_name}: regression failed to find <<{greptext}>>\n") 387 | self.global_regression_handle.write\ 388 | (f"{self.unique_name}: regression failed to find <<{greptext}>>\n") 389 | return rkey 390 | def regression_line(self,output_file,rtest): 391 | what_line = rtest["line"] 392 | for line in output_file: 393 | the_line = line.strip() 394 | if what_line=="first": 395 | break; 396 | return_value = self.regression_line_pick_field(the_line,rtest) 397 | return_value = self.regression_label_prepend(return_value,rtest) 398 | return return_value 399 | def apply_regression(self,rtest,filename): 400 | ## get regression result from filename 401 | rreturn = None; regger = None 402 | if "line" in rtest.keys(): 403 | regger = self.regression_line 404 | elif "grep" in rtest.keys(): 405 | regger = self.regression_grep 406 | if regger: 407 | try: 408 | with open(filename,"r") as output_file: 409 | rreturn = regger(output_file,rtest) 410 | except FileNotFoundError as e : 411 | print(f"Could not open file for regression: <<{e}>>") 412 | return rreturn 413 | def do_regression(self,filename=None): 414 | ## regress on `filename', writing private and global file 415 | if not filename: filename = self.slurm_output_file_name 416 | self.logwrite(f"Doing regression <<{self.regression}>> on job {self.unique_name} from <<{filename}>>") 417 | print(f"Doing regression on {filename}") 418 | if self.regression is None or self.regression=="none": return None 419 | rtest = regression_test_dict( self.regression ) 420 | ## rtest = self.get_regression_tests() 421 | rreturn = self.apply_regression(rtest,filename) 422 | if not rreturn: rreturn = "REGRESSION ERROR" 423 | rfilekey = None 424 | self.logwrite(f".. done regression on {self.unique_name}, giving: {rreturn}") 425 | self.global_regression_handle.write(rreturn+"\n") 426 | rfilename = f"{self.unique_name}.txt" 427 | rfilehandle,_,_,rfilekey \ 428 | = SpawnFiles().open(rfilename,subdir="regression",new=True) 429 | self.logwrite(f"writing regression result <<{rreturn}>> to global and <<{rfilename}>>") 430 | rfilehandle.write(rreturn+"\n") 431 | return rfilekey 432 | 433 | def parse_suite(suite_option_list): 434 | suite = { "name" : "unknown", "runner" : "", "dir" : "./", "apps" : [] } 435 | for opt in suite_option_list: 436 | if re.search(":",opt): 437 | key,val = opt.split(":") 438 | #print("benchmark suite option {}={}".format(key,val)) 439 | if key=="type": 440 | if val=="mpi": 441 | suite["runner"] = "ibrun " 442 | else: 443 | suite[key] = val 444 | else: 445 | #print("benchmark suite app {}".format(opt)) 446 | if re.search(r'\*',opt): 447 | dir = suite["dir"] 448 | if not os.path.exists(dir) or not os.path.isdir(dir): 449 | raise Exception("No such directory: <<{}>>".format(dir)) 450 | p = sp.Popen( "cd {} ; ls {}".format( dir,opt ),\ 451 | stdout=sp.PIPE,shell=True ) 452 | out,err = p.communicate() 453 | for a in out.split(): 454 | suite["apps"].append(a.decode("utf-8")) 455 | print("application wildcards gives apps <<{}>>".format(suite["apps"])) 456 | else: 457 | suite["apps"].append(opt) 458 | return suite 459 | 460 | ## 461 | ## return nodes and cores as list of equal length 462 | ## 463 | def nodes_cores_threads_values(configuration): 464 | def str2set(s): 465 | if re.search(",",s): 466 | s = [ int(p) for p in s.split(",") ] 467 | elif re.search(":",s): 468 | print("colon notation not yet supported"); raise Exception() 469 | else: 470 | s = [ int(s) ] 471 | return s 472 | nodes = configuration.get("nodes","1") 473 | cores = configuration.get("cores",None) 474 | if not cores is None: 475 | print("Cores keyword not supported"); raise Exception() 476 | ppn = configuration.get("ppn","1") 477 | threads = configuration.get("threads","0") 478 | 479 | ## nodes 480 | print(f"""Parallel configuration: 481 | nodes: {nodes} 482 | ppn: {ppn} 483 | threads: {threads} 484 | """) 485 | nodes = str2set(nodes); ppn = str2set(ppn); threads = str2set(threads) 486 | 487 | ## 488 | ## node/core combinations 489 | ## 490 | nodes_cores_threads = [ [ [ [n,p,t] 491 | for n in nodes ] 492 | for p in ppn ] 493 | for t in threads ] 494 | nodes_cores_threads = [ n for npt in nodes_cores_threads 495 | for np in npt for n in np ] 496 | print("nodes_cores_threads:",nodes_cores_threads) 497 | return nodes_cores_threads 498 | 499 | def running_jobids(qname,user): 500 | ids = [] 501 | p = sp.Popen(["squeue","-u",user,"-p",qname,"-h","-o","%A %t"],stdout=sp.PIPE) 502 | for status in io.TextIOWrapper(p.stdout, encoding="utf-8"): 503 | status = status.strip() # print("job status line <<{}>>".format(status)) 504 | id,stat = status.split() 505 | ids.append(id) 506 | print("Running jobs for user={} on queue={}: {}".format(user,qname,ids)) 507 | return ids 508 | 509 | class Queue(): 510 | def __init__(self,name,limit=1): 511 | self.name = name; self.jobs = []; self.set_limit(limit); self.debug = False 512 | def set_limit(self,limit): 513 | self.limit = int(limit) 514 | def enqueue(self,j): 515 | self.jobs.append(j) 516 | qrunning = running_jobids(self.name,j.user) 517 | if len(qrunning)> to <<{value}>>") 635 | os.environ[name] = value 636 | 637 | self.configuration = configuration 638 | self.testing = self.configuration.get( "testing",False ) 639 | self.modules = self.configuration.get( "modules",None ) 640 | print(f"Test suite with modules {self.modules}") 641 | 642 | self.nodes_cores_threads = nodes_cores_threads_values(self.configuration) 643 | self.suites = [ parse_suite( suite_spec ) ] 644 | print("{}".format(str(self))) 645 | def __str__(self): 646 | description = f""" 647 | ################################################################ 648 | Test suite: {self.name} 649 | modules: {self.modules} 650 | nodes/cores/threads: {self.nodes_cores_threads} 651 | regression: {self.regression} 652 | suites: {self.suites} 653 | ################################################################ 654 | """ 655 | return description 656 | def type_dir(self,typ,dir=None): 657 | if dir: 658 | return f"{typ}-{dir}" 659 | else: 660 | return f"{typ}" 661 | def tracemsg(self,msg): 662 | print(msg) 663 | self.logfile.write(msg+"\n") 664 | def run(self,**kwargs): 665 | testing = kwargs.get("testing",False) 666 | debug = kwargs.get("debug",False) 667 | submit = kwargs.get("submit",True) 668 | 669 | count = 1 670 | jobs = []; jobids = [] 671 | ## for now all output goes in the same directory 672 | outputdir = SpawnFiles().ensurefiledir(subdir="output") 673 | jobnames = []; regressionfiles = [] 674 | ## iterate over suites 675 | ## I think this only does one iteration. 676 | for suite in self.suites: 677 | suitename = suite["name"] 678 | print(f"Suitename: {suitename}") 679 | regressionfilename = f"regression-{suitename}.txt" 680 | if self.regression: 681 | global_regression_handle,_,_,k = SpawnFiles().open_new( f"{regressionfilename}" ) 682 | else: global_regression_handle = None 683 | self.tracemsg(f"Test suite {self.name} run at {self.starttime}") 684 | self.logfile.write(str(self)) 685 | for benchmark in suite["apps"]: 686 | self.tracemsg("="*16+"\n"+f"{count}: submitting suite=<<{suitename}>> benchmark=<<{benchmark}>>") 687 | for nodes,ppn,threads in self.nodes_cores_threads: 688 | self.tracemsg(f" .. N={nodes} ppn={ppn} threads={threads}") 689 | unique_name = f"{suitename}-{benchmark}-{nodes}-{ppn}-{threads}" 690 | if unique_name in jobnames: 691 | raise Exception(f"Job name conflict: {unique_name}") 692 | else: 693 | jobnames.append(unique_name) 694 | job = Job(self.configuration, 695 | program_name=benchmark,unique_name=unique_name, 696 | outputdir=outputdir, 697 | nodes=nodes,ppn=ppn,threads=threads, 698 | programdir=suite["dir"], 699 | modules=self.modules, 700 | regression=self.regression,global_regression_handle=global_regression_handle, 701 | runner=suite["runner"], 702 | macros=self.configuration, 703 | count=count,trace=True, 704 | ) 705 | if submit: 706 | Queues().enqueue(job) 707 | elif job.regression: 708 | regression_key = job.do_regression() 709 | regressionfiles.append( regression_key ) 710 | count += 1 711 | if submit: 712 | Queues().wait_for_jobs() 713 | else: 714 | SpawnFiles().close_files( regressionfiles ) 715 | if cdir := self.configuration["comparedir"]: 716 | print("All jobs finished, only regression comparison left to do") 717 | cdir = cdir+"/regression" 718 | odir = self.configuration["outputdir"]+"/regression" 719 | if self.regression: ## we can have both regression and none in the same job 720 | comparefile = self.regression_compare(suitename,cdir,odir) 721 | print( f" .. comparison output in {comparefile}" ) 722 | def regression_compare(self,suitename,cdir,odir): 723 | rtest = regression_test_dict( self.regression ) 724 | comparison,comp_dir,comp_fil,comp_key \ 725 | = SpawnFiles().open_new(f"regression_compare-{suitename}") 726 | comparison_path = comp_dir+"/"+comp_fil 727 | majorly_off = []; within_margin = 0; 728 | for ofile in [ f for f in os.listdir(odir) 729 | if os.path.isfile( os.path.join( odir,f ) ) ]: 730 | opath = os.path.join( odir,ofile ) 731 | SpawnFiles().close_by_path(opath) 732 | cpath = os.path.join( cdir,ofile ) 733 | if os.path.isfile( cpath ): 734 | comparison.write(f"Comparing: output={opath} compare={cpath}\n") 735 | with open( opath,"r" ) as ohandle: 736 | oline = ohandle.readline().strip() 737 | with open( cpath,"r" ) as chandle: 738 | cline = chandle.readline().strip() 739 | dev = "" 740 | if "margin" in rtest.keys(): 741 | margin = rtest["margin"] 742 | violate = False 743 | if perc := re.match(r'([0-9]+)p.*',margin): 744 | dev = float( perc.groups()[0] )/100 745 | try : 746 | oval = float( oline ); cval = float( cline ) 747 | violate_more = (oval-cval)/cval>dev 748 | violate_less = (cval-oval)/oval>dev 749 | violate = violate_more or violate_less 750 | if violate_less: 751 | dev = f", outside {dev} margin: less" 752 | elif violate_more: 753 | dev = f", outside {dev} margin: more" 754 | else: 755 | dev = f", inside {dev} margin" 756 | within_margin += 1 757 | except: 758 | dev = f", margin comparison failed" 759 | report = f"Output: {oline}, compare: {cline}{dev}" 760 | if violate: majorly_off.append( f"{opath} {report}" ) 761 | else: report = f"Output: {oline}, compare: {cline}" 762 | comparison.write( f"{report}\n" ) 763 | if within_margin>0: 764 | comparison.write( f"================ Tests within margin: {within_margin} ================\n" ) 765 | if len(majorly_off)>0: 766 | comparison.write( f"================ Major violations: {len(majorly_off)} ================\n" ) 767 | for m in majorly_off: 768 | comparison.write( m+"\n" ) 769 | SpawnFiles().close_files( [comp_key] ) ## comparison.close() 770 | #print( f" .. comparison in <<{comparison_path}>>" ) 771 | return comparison_path 772 | --------------------------------------------------------------------------------