├── Makefile ├── Makefile.inc ├── README ├── checks ├── awk_fflush └── sleep_fract ├── doc ├── INSTALL ├── LICENSE ├── Makefile ├── NEWS └── TODO ├── examples ├── Makefile ├── Makefile.inc ├── all_substr │ ├── Makefile │ ├── cmd_all_substr.in │ └── run_all_substr ├── cc_wrapper │ ├── Makefile │ ├── cmd_cc_wrapper │ ├── func1.c │ ├── func2.c │ ├── func3.c │ └── run_cc_wrapper ├── cc_wrapper2 │ ├── Makefile │ ├── func1.c │ ├── func2.c │ ├── func3.c │ └── run_cc_wrapper2 ├── dirtest │ ├── Makefile │ ├── run_dirtest │ └── tasks ├── divide │ ├── Makefile │ ├── cmd_divide.in │ ├── cmd_divide2.in │ ├── run_divide │ └── run_divide2 ├── make_package │ ├── Makefile │ ├── cmd_make_package.in │ ├── cmd_xxx_failed_make_package.in │ ├── run_cycle_make_package │ ├── run_make_package │ ├── tasks │ ├── tasks2 │ └── tasks_cycle ├── toupper │ ├── Makefile │ ├── cmd_toupper.in │ └── run_toupper └── wav2flac │ ├── Makefile │ └── run_wav2flac ├── help.mk ├── paargs ├── Makefile ├── paargs.in └── paargs.pod ├── paexec ├── Makefile ├── common.h ├── decls.h ├── nodes.c ├── nodes.h ├── paexec.c ├── paexec.pod ├── paexec_reorder ├── paexec_reorder.pod ├── shquote.c ├── signals.c ├── signals.h ├── tasks.c ├── tasks.h ├── wrappers.c └── wrappers.h ├── presentation ├── Makefile ├── dep-graph.dot └── paexec.tex ├── tests ├── Makefile ├── Makefile.inc ├── _huge_pkgsrc_test.in ├── delayed_output │ ├── Makefile │ └── delayed_output.c ├── fakeflac │ ├── fake1.wav │ ├── fake2.wav │ ├── fake3.wav │ ├── fake4.wav │ ├── fake5.wav │ └── flac ├── scripts │ ├── Makefile │ ├── big_result_cmd.in │ ├── paexec_notransport │ ├── transport_broken_echo.in │ ├── transport_broken_echo2 │ ├── transport_broken_rnd.in │ └── transport_broken_toupper.in ├── test.sh └── transp_closed_stdin │ ├── Makefile │ └── transp_closed_stdin.c └── use.mk /Makefile: -------------------------------------------------------------------------------- 1 | BIRTHDATE = 2008-01-25 2 | PROJECTNAME = paexec 3 | 4 | ##### 5 | SUBPRJ = paexec:tests paargs presentation doc 6 | SUBPRJ_DFLT?= paexec paargs 7 | 8 | examples = divide all_substr cc_wrapper cc_wrapper2 \ 9 | make_package toupper dirtest 10 | .for d in ${examples} 11 | SUBPRJ += examples/${d}:tests examples/${d}:examples 12 | .endfor 13 | 14 | tests = transp_closed_stdin scripts delayed_output 15 | .for d in ${tests} 16 | SUBPRJ += tests/${d}:tests 17 | .endfor 18 | 19 | ##### 20 | MKC_REQD = 0.33.0 21 | 22 | ##### 23 | test: all-tests 24 | @: 25 | 26 | clean: clean-tests clean-presentation clean-doc clean-examples 27 | cleandir: cleandir-tests cleandir-presentation cleandir-doc cleandir-examples 28 | 29 | # new recursive target for making a distribution tarball 30 | TARGETS += _prepdist 31 | 32 | DIST_TARGETS= prepdist 33 | 34 | .PHONY: prepdist 35 | prepdist: all-presentation _prepdist 36 | rm ${MKC_CACHEDIR}/_mkc* 37 | ${MAKE} -Cpresentation ${MAKEFILES} _clean_garbage 38 | 39 | ##### 40 | .include "help.mk" 41 | .include "use.mk" 42 | .include "Makefile.inc" 43 | .include 44 | 45 | # Copyright (c) 2007-2024 Aleksey Cheusov 46 | # 47 | # Permission is hereby granted, free of charge, to any person obtaining 48 | # a copy of this software and associated documentation files (the 49 | # "Software"), to deal in the Software without restriction, including 50 | # without limitation the rights to use, copy, modify, merge, publish, 51 | # distribute, sublicense, and/or sell copies of the Software, and to 52 | # permit persons to whom the Software is furnished to do so, subject to 53 | # the following conditions: 54 | # 55 | # The above copyright notice and this permission notice shall be 56 | # included in all copies or substantial portions of the Software. 57 | # 58 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 59 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 60 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 61 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 62 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 63 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 64 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 65 | -------------------------------------------------------------------------------- /Makefile.inc: -------------------------------------------------------------------------------- 1 | # initial buffer size for the tasks and results 2 | BUFSIZE ?= 4096 3 | 4 | # 5 | EGDIR ?= ${DATADIR}/doc/paexec/examples 6 | DOCDIR ?= ${DATADIR}/doc/paexec 7 | 8 | # Those poor souls who run Solaris can set AWK to gawk 9 | AWK ?= /usr/bin/awk 10 | 11 | .ifndef PAEXEC_SH 12 | . if exists(/usr/xpg4/bin/sh) 13 | # Solaris-10's /bin/sh is completely broken, 14 | # /usr/xpg4/bin/sh sucks too but sucks less. 15 | PAEXEC_SH = /usr/xpg4/bin/sh 16 | . else 17 | PAEXEC_SH = /bin/sh 18 | . endif 19 | .endif 20 | 21 | MKC_REQUIRE_PROGS += ${AWK} runawk ${PAEXEC_SH} 22 | 23 | .export AWK 24 | MKC_CHECK_CUSTOM += awk_fflush 25 | MKC_CUSTOM_FN.awk_fflush = checks/awk_fflush 26 | 27 | .include "mkc.configure.mk" 28 | 29 | .if !${CUSTOM.awk_fflush:U1} 30 | MKC_ERR_MSG += "ERROR: ${AWK} doesnt not support fflush() function" 31 | .endif 32 | 33 | INTEXTS_REPLS += awk ${PROG.${AWK}:U${AWK}} 34 | INTEXTS_REPLS += sh ${PROG.${PAEXEC_SH}:U${PAEXEC_SH}} 35 | INTEXTS_REPLS += version ${VERSION:U} 36 | 37 | # 38 | VERSION= 1.1.6 39 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | ====================================================================== 2 | 3 | paexec: Small program that processes a list of tasks in parallel 4 | on different CPUs, computers in a network or whatever. 5 | Full documentation is available in paexec.1 and paexec.pod. 6 | paexec.pdf presentation is also available in presentation/ 7 | subdirectory. 8 | 9 | author: Aleksey Cheusov 10 | 11 | source code: https://github.com/cheusov/paexec 12 | https://sourceforge.net/projects/paexec 13 | 14 | licence: MIT license 15 | 16 | ====================================================================== 17 | 18 | INSTALLATION 19 | 20 | See INSTALL file 21 | 22 | ====================================================================== 23 | 24 | PACKAGES 25 | 26 | 1) pkgsrc (NetBSD, DragonFlyBSD, Linux, Solaris...) 27 | 28 | See parallel/paexec (http://pkgsrc.se/parallel/paexec) 29 | 30 | 2) FreeBSD 31 | 32 | See devel/paexec (http://www.freshports.org/devel/paexec) 33 | 34 | 3) Debian Linux 35 | 36 | https://packages.debian.org/ 37 | 38 | 4) AltLinux 39 | 40 | https://packages.altlinux.org/en/sisyphus/srpms/paexec 41 | 42 | 5) ArchLinux 43 | 44 | https://aur.archlinux.org/packages/paexec/ 45 | 46 | 6) OpenSuSE Linux 47 | 48 | https://build.opensuse.org/package/show/openSUSE:Factory/paexec 49 | 50 | 7) Darwin 51 | 52 | brew packaging system 53 | -------------------------------------------------------------------------------- /checks/awk_fflush: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | if echo | "$AWK" '{fflush()}' 2>/dev/null 1>&2; then 4 | echo 1 5 | else 6 | echo 0 7 | fi 8 | -------------------------------------------------------------------------------- /checks/sleep_fract: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | if sleep 0.1 2>/dev/null; then 4 | echo 1 5 | else 6 | echo 0 7 | fi 8 | -------------------------------------------------------------------------------- /doc/INSTALL: -------------------------------------------------------------------------------- 1 | ====================================================================== 2 | 3 | INSTALLATION 4 | 5 | 0) 6 | Maybe paexec has been already packaged for your platform. See 7 | PACKAGES section in README file and your system package management 8 | tools. 9 | 10 | 1) 11 | You need bmake (portable version of NetBSD make) for building paexec. 12 | I'd recommend to use latest stable version. 13 | 14 | http://crufty.net/help/sjg/bmake.html 15 | 16 | NOTE: !!! GNU make IS NOT GOOD !!! 17 | 18 | 2) 19 | We also need mk-configure >= 0.33.0. 20 | 21 | https://github.com/cheusov/mk-configure 22 | 23 | or 24 | 25 | http://sourceforge.net/projects/mk-configure/ 26 | 27 | 3) 28 | "libmaa" library is also needed. It is a part of 29 | "dict" project available here: 30 | 31 | https://github.com/cheusov/libmaa 32 | 33 | or 34 | 35 | http://sourceforge.net/projects/dict/ 36 | 37 | 4) 38 | If you want to change the default build options, 39 | run mkcmake like this 40 | 41 | env [YOUR_ASSIGNMENTS] mkcmake [YOUR_ASSIGNMENTS] 42 | 43 | See example section below 44 | 45 | 5) Uncompress tarball you've downloaded like this 46 | gzip -dc paexec-X-Y-Z.tar.gz | tar -xf- 47 | 48 | 6) cd paexec-X-Y-Z 49 | 50 | 7) mkcmake 51 | 52 | In order to see additional information about settings and 53 | the structure of project you want to run 54 | 55 | mkcmake help 56 | 57 | 8) mkcmake test 58 | If this step fails on your system, please let me now. 59 | 60 | 9) mkcmake install 61 | 62 | 10) If you also want to install examples, run the following command 63 | 64 | mkcmake all-examples install-examples 65 | 66 | By default examples are installed 67 | to ${DATADIR}/doc/paexec/examples/. 68 | For changing it, set EGDIR environment variable. 69 | 70 | 11) If you also want to install README, NEWS, TODO etc., run the 71 | following command 72 | 73 | mkcmake all-doc install-doc 74 | 75 | By default these files are installed to ${PREFIX}/share/doc/. 76 | For changing it, set DOCDIR environment variable. 77 | 78 | 12) paexec project consists of the following subprojects: 79 | - paexec (paexec and paexec_reorder executables and appropriate 80 | manual pages) 81 | - doc (README, NEWS etc.) 82 | - examples (examples of use). 83 | By default only "paexec" is built and installed. You can build and 84 | install all "subprojects" like the following. 85 | 86 | env SUBPRJ_DFLT='paexec examples doc' mkcmake all install 87 | 88 | 13) 89 | paexec_reorder utility is written in runawk, so we need runawk>=1.4.3 90 | at runtime. 91 | 92 | http://sourceforge.net/projects/runawk/ 93 | 94 | 14) 95 | There is a lot of Makefile variables that can be changed during 96 | build and installation. 97 | 98 | PREFIX - where paexec and paexec_reorder are installed to 99 | MANDIR - root directory for manual pages 100 | DESTDIR - fake root for installation 101 | CPPFLAGS 102 | CFLAGS 103 | LDFLAGS 104 | LDADD 105 | ... 106 | 107 | NOTE: Environment variable must be the same at build and installation time! 108 | See mk-configure(7) for details. 109 | 110 | ------------------------------ 111 | Examples of build and installation: 112 | 113 | 1) export PREFIX=/usr SYSCONFDIR=/etc MANDIR=/usr/share/man 114 | export SUBPRJ_DFLT='paexec examples doc' 115 | mkcmake all 116 | env DESTDIR=/tmp/fake-root mkcmake install 117 | 118 | 2) env CC='icc' \ 119 | PREFIX=/usr/pkg \ 120 | CPPFLAGS='-I/usr/pkg/include' \ 121 | LDFLAGS='-L/usr/pkg/lib -Wl,-rpath -Wl,/usr/pkg/lib' \ 122 | mkcmake -s all install 123 | -------------------------------------------------------------------------------- /doc/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2007-2013 Aleksey Cheusov 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | FILES = LICENSE NEWS ../README TODO 2 | FILESDIR = ${DOCDIR} 3 | 4 | .include 5 | -------------------------------------------------------------------------------- /doc/NEWS: -------------------------------------------------------------------------------- 1 | ====================================================================== 2 | Version 1.1.6, by Aleksey Cheusov, Fri, 7 Jun 2024 9:19:17 +0400 3 | 4 | option -n: ignore leading spaces and tabs 5 | 6 | Add more tests 7 | 8 | ====================================================================== 9 | Version 1.1.5, by Aleksey Cheusov, Wed, 29 May 2024 20:06:42 +0400 10 | 11 | Fix: Before running commands reset signal handlers for ALRM, PIPE 12 | and CHLD to SIG_DFL. Also, unblock these signals. This fixes some 13 | misterius problems with commands running alarm(2). 14 | 15 | libmaa>=1.5.1 is required for build 16 | 17 | ====================================================================== 18 | Version 1.1.4, by Aleksey Cheusov, Mon, 1 Jun 2020 17:10:38 +0300 19 | 20 | Remove local function pr_open and use it from libmaa library. 21 | So, now paexec requires libmaa. 22 | 23 | "mkcmake help" shows configuring variables and descriptions for 24 | targets. 25 | 26 | Use mkc's feature "getdelim" instead of local implementation. 27 | So, we need at least mk-configure 0.33.0 28 | 29 | Optimization flags defaults to "-O2 -g" 30 | 31 | ====================================================================== 32 | Version 1.1.3, by Aleksey Cheusov, Thu, 14 May 2020 20:54:21 +0300 33 | 34 | Fix use of uninitialized array. 35 | As a rsult paexec generates incorrect shell command to run 36 | on nodes. Seen with "clang-7.0.1" with -O2 and -D_FORTIFY_SOURCE=2 37 | 38 | Add one line description to projects. 39 | So, one can run "mkcmake help" 40 | 41 | ====================================================================== 42 | Version 1.1.2, by Aleksey Cheusov, Mon, 30 Mar 2020 21:50:03 +0300 43 | 44 | paexec -x: 45 | * fix handling tasks with leading spaces 46 | * fix handling tasks with backslashes 47 | 48 | paargs: 49 | * on SunOS-5.10 use /usr/xpg4/bin/sh 50 | instead of horribly broken /bin/sh 51 | 52 | Fix build failure on SunOS-5.10 (getdelim(3) is absent) 53 | 54 | Introduce PAEXEC_SH environment variable 55 | for fixing /bin/sh on SunOS-5.10 56 | 57 | Introduce PAEXEC_FIND variable for regression tests (required on 58 | SunOS-5.10) 59 | 60 | ====================================================================== 61 | Version 1.1.1, by Aleksey Cheusov, Fri, 15 Feb 2019 21:58:39 +0300 62 | 63 | Fix build using clang 64 | 65 | Verify snprintf didn't truncate the output 66 | 67 | ====================================================================== 68 | Version 1.1.0, by Aleksey Cheusov, Wed, 25 Apr 2018 23:21:24 +0300 69 | 70 | paexec: 71 | - add new option -0. It works just like in "xargs -0". 72 | - add new option -J. 73 | - add new option -mw=. 74 | - fix help message display by -h. 75 | - -md= now allows no delimiter mode in -g mode. 76 | - -c and -C override each other if one is implied after another. 77 | 78 | Add new tool "paargs". It is a wrapper over paexec(1) that 79 | simplifies use of paexec. 80 | 81 | Fix transport_broken_rnd test script. 82 | 83 | This fixes regression test on Solaris. 84 | 85 | Update man page for paexec(1). 86 | 87 | ====================================================================== 88 | Version 1.0.1, by Aleksey Cheusov, Thu, 14 Aug 2014 01:44:39 +0300 89 | 90 | Unflushed (broken) stdout seen on Darwin was fixes. 91 | Who knows, this may happen on other systems too. 92 | 93 | ====================================================================== 94 | Version 1.0.0, by Aleksey Cheusov, Sun, 13 Jul 2014 19:10:48 +0300 95 | 96 | mk-configure>=0.27.0 is required for build. 97 | 98 | Option -X was introduced to paexec(1) for ignoring 99 | calculator's stdout. 100 | 101 | PAEXEC_ENV environment variable sets a list of variables passed to 102 | the calculator. 103 | 104 | PAEXEC_TRANSPORT environment variable sets the transport unless 105 | option -t was applied. 106 | 107 | PAEXEC_NODES environment variable sets the nodes unless 108 | option -n was applied. 109 | 110 | pareorder(1) is a synonym for paexec_reorder(1) 111 | 112 | ====================================================================== 113 | Version 0.19.1, by Aleksey Cheusov, Mon, 26 Aug 2013 19:05:53 +0300 114 | 115 | paexec: Fix for segfault seen on Linux 116 | 117 | Version 0.19.0, by Aleksey Cheusov, Sun, 25 Aug 2013 17:21:03 +0300 118 | 119 | This release of paexec was successfully tested on the following 120 | platforms: NetBSD-6.1/amd64, OpenBSD-5.3/i386, FreeBSD-8.3/i386, 121 | Solaris-10/sparc, Solaris-11/amd64 and diverse Linux/{i386,amd64}. 122 | 123 | paexec is now selfcontained, libmaa is not needed anymore. 124 | 125 | Presentation paexec.pdf was added to presentation/. I hope it will 126 | help easier understand how "paexec" works. It is installed to 127 | ${DOCDIR}. 128 | 129 | paexec: 130 | - POSIX-2008 getline(3) is used for reading lines 131 | instead of home-made function. 132 | - t '' is equivalent to "no transport", spaces are trimmed. 133 | - "-n +NNN" has higher priority than -t, i.e. if they both are 134 | specified, transport is ignored. 135 | - Fix for -W1. 136 | - Environment variable PAEXEC_EOT was introduced. 137 | - Option -y was added to paexec(1) and paexec_reorder(1). 138 | - Option -C was added to paexec(1). 139 | 140 | paexec_reorder: 141 | - Option -x was added to paexec_reorder(1). 142 | 143 | A number of fixes and improvements in regression tests. 144 | 145 | ====================================================================== 146 | Version 0.18.0, by Aleksey Cheusov, Thu, 7 Mar 2013 15:17:33 +0300 147 | 148 | paexec: 149 | - fixed: NULL dereference when the first line given on input is 150 | empty. Thanks to Sergey Revyako for bug report! 151 | - fixed: entire command passed tp ssh-like transport should be 152 | shquoted. In particular this fixes -x that didn't work with -t. 153 | Thanks to Sergey Revyako for bug report! 154 | - 'paexec -g' accepts empty strings as tasks. 155 | - More regression tests were added 156 | 157 | paexec_reorder.1: 158 | - Mistype fix 159 | 160 | ====================================================================== 161 | Version 0.17.0, by Aleksey Cheusov, Sun, 9 Sep 2012 02:07:34 +0300 162 | 163 | paexec: 164 | - Option -x was added. With its help paexec can run one command 165 | per task. If -g is also specified, command's exit status is 166 | analysed. Appropriate task and dependants are marked as "failed" 167 | if it is non-zero. 168 | - First character of -n argument must be alphanumeric, `+', `_', 169 | `:' or `/'. Other symbols are reserved for future extentions. 170 | - With '-n :filename' paexec reads a list of nodes from the 171 | specified file. 172 | - With a help of new option '-m t=' end of task string 173 | may be specified, which is an empty line by default. 174 | - Option -md= was added that overrides the default 175 | delimiter (space character) between tasks in graph mode (-g). 176 | - Output line that contains failed dependants no longer ends with 177 | unnecessary space. 178 | - Long options were completely removed. 179 | 180 | paexec_reorder: 181 | - Fix. "paexec_reorder -g" now handles correctly failed tasks' 182 | output. One extra line after "fatal" is expected. 183 | - Options -m was added. It does the same things as paexec's -m. 184 | 185 | More examples of use and regression tests. 186 | 187 | Documentation update, clean-ups and improvements. 188 | 189 | Regression tests: 190 | - Signals handling was fixed in. 191 | - LC_ALL is always set to C in regression tests, this fixes some 192 | problems in internationalized environment. 193 | 194 | mk-configure>=0.23.0 is required at build time 195 | 196 | ====================================================================== 197 | Version 0.16.1, by Aleksey Cheusov, Wed, 23 Mar 2011 00:14:15 +0200 198 | 199 | paexec.1 and paexec_reorder.1 are included to paexec-0.16.1.tgz. 200 | Just like in paexec<=0.15.1 pod2man is not needed 201 | for building paexec. 202 | 203 | ====================================================================== 204 | Version 0.16, by Aleksey Cheusov, Fri, 11 Mar 2011 11:58:31 +0200 205 | 206 | Project's structure has been reorganized. Now the top-level Makefile 207 | uses mkc.subprj.mk. This adds a lot of flexibility in building the 208 | project and development. 209 | See doc/INSTALL for updated installation instructions. 210 | 211 | New modes for reordering tasks were added: -W0 and -W2 212 | See the manual page for details about option -W. 213 | 214 | Long options are considered deprecated. They are still supported but 215 | will be removed in the future. Please use POSIX short options. At 216 | the moment use of them produces warning message on stderr. 217 | 218 | Fix the compilation bug on old versions of OpenBSD (at least <=3.8) 219 | and probably other systems where intptr_t is declared in inttypes.h 220 | but not in stdint.h 221 | 222 | Fix for 'mkcmake test' failure on Solaris and HP-UX because their 223 | /usr/bin/awk sucks. On these platforms now it is possible to run the 224 | following command. 225 | env AWK=/full/path/to/gawk mkcmake all test 226 | 227 | ====================================================================== 228 | Version 0.15.0, by Aleksey Cheusov, Sun, 10 Oct 2010 18:06:31 +0300 229 | 230 | After some thoughts I decided to switch from plain mk-files to 231 | mk-configure build system. It provides very cool features and makes 232 | development drammatically easier. It also makes Makefiles much 233 | cleaner and easier. Installation instructions has been updated in 234 | README file. 235 | 236 | PAEXEC reads the tasks from stdin and distributes them to all 237 | available hosts or CPU. If some tasks are easy and can be made 238 | quickly while others require much more CPU time it makes sense to 239 | reorder tasks calcultion in order to reduce total calculation time 240 | and reduce computers/CPUs idle time. For this purpose -W option 241 | is added to paexec(1) using weights assigned to each tasks. At the 242 | moment only two values are allowed for : 0 -- do not use 243 | weights at all, 1 -- run heavier tasks first as soon as possible. 244 | 245 | New tool paexec_reorder(1) for reordering sliced output of 246 | paexec(1). It is written in runawk. So, you'll need it at run time. 247 | 248 | FIXED: 1 byte buffer overflow if -d option is applied. 249 | 250 | Documentation update. Tons of new regression tests. Regression 251 | tests framework has been significantly reworked. Clean-ups. 252 | 253 | README: notes about my Debian/Lenny/x86 repository. 254 | 255 | Minor fixes (warning messages) for different compilers. 256 | 257 | ====================================================================== 258 | Version 0.14.0, by Aleksey Cheusov, Sun, 3 Jan 2010 12:38:54 +0200 259 | 260 | fixed: "paexec -z" without -s never worked. Now it is fine. 261 | 262 | New options -g|--graph are synonyms for -s|--pos. 263 | 264 | New option -w for waiting for failed nodes if they ALL failed. 265 | By default (with -Z) if ALL nodes fail paexec exits with error. 266 | 267 | New option -m for setting alternative strings for 268 | success/failure/fatal. 269 | 270 | Target for installing directories has been renamed from 271 | install-dirs to installdirs. 272 | 273 | Other minor fixes and code clean-ups. 274 | 275 | ====================================================================== 276 | Version 0.13.0, by Aleksey Cheusov, Sat, 7 Mar 2009 19:17:03 +0200 277 | 278 | FIXED: When 'paexec -s' retreives 10000 tasks, it allocates 279 | 10000*10000*sizeof(int) bytes for detecting cycles, i.e. ~400Mb on 280 | 32-bit system or 800Mb on 64-bit system. This is absolutely 281 | inacceptable. This bad algorithm is replaced with new one which 282 | doesn't need quadratic matrix and works much faster. 283 | 284 | ADDED: -Z option 285 | When I<-z> applied, if a I fails, appropriate node is 286 | marked as broken and is excluded from the following task 287 | distribution. But if B<-Z> applied, every I seconds an 288 | attempt to rerun a comand on a failed node is made. I<-Z> implies 289 | I<-z>. This option makes possible to organize clusters over 290 | unreliable networks/hardware. 291 | 292 | No EINTR wrappers anymore (iread, xread etc.). SIGCHLD and SIGALRM 293 | are blocked most of the time. They are unblocked before select(2) 294 | and blocked just after it. SA_RESTART is not used anymore. 295 | 296 | Minor clean-ups in Makefile. 297 | 298 | ====================================================================== 299 | Version 0.12.1, by Aleksey Cheusov, Tue, 23 Dec 2008 20:33:20 +0200 300 | 301 | FIX: support for -z appeared in paexec-0.12.0 was actually 302 | incomplete :-/ Now everything should be fixed. More regression 303 | tests. 304 | 305 | Makefile: adapted for FreeBSD make which doesn't support .PARSEDIR 306 | 307 | ====================================================================== 308 | Version 0.12.0, by Aleksey Cheusov, Mon, 22 Dec 2008 21:48:01 +0200 309 | 310 | ADDED: -z option. If applied, read/write(2) operations from/to nodes 311 | becomes not critical. In case paexec has lost connection to the 312 | node, it will reassign failed task to another node and, if -s option 313 | applied, will output string "fatal" to stdout. This makes paexec 314 | resistant to the I/O errors, as a result you can create paexec 315 | clusters even over network consisting of unreliable hosts 316 | (Internet?). Failed hosts are marked as such and will not be used 317 | during the current run of paexec. NOTE: "success", "failure" and 318 | "fatal" strings should be used to process the output of 'paexec -s'. 319 | 320 | select(2) do not listen stdin (fd=0) anymore. Blocking read(2) is 321 | used to read tasks from stdin. 322 | 323 | Makefile: CPPFLAGS -> CFLAGS, FreeBSD make Mk scripts 324 | don't use CPPFLAGS. 325 | 326 | ====================================================================== 327 | Version 0.11.0, by Aleksey Cheusov, Sat, 25 Oct 2008 17:49:47 +0300 328 | 329 | paexec -s: before beginning actual work an input tasks graph is 330 | checked for cycles. If they are detected, paexec exits with error. 331 | 332 | minor fix in man page 333 | 334 | ====================================================================== 335 | Version 0.10.1, by Aleksey Cheusov, Sat, 25 Oct 2008 02:00:55 +0300 336 | 337 | This version of paexec was sucessfully tested on the following 338 | platforms: 339 | NetBSD/x86 340 | NetBSD/Alpha(64bit) 341 | Linux/x86 342 | Linux/x86-64 343 | Interix-3.5/x86 344 | FreeBSD/x86 345 | Solaris-10/x86 346 | 347 | Minor fix for rhomb-like dependencies (paexec -s). 348 | Suppose A depends on (B1 and B2), (B1 and B2) depends on C. 349 | If C fails, earlier paexec versions produced 350 | the following output 351 | failure 352 | C B1 A B2 A 353 | That is, A was output twice. Now it is fixed and A 354 | is printed only once. New regression test for this. 355 | 356 | By default(!) getopt_long(3) is enabled on the following platforms 357 | (macro): __NetBSD__, __FreeBSD__, __OpenBSD__, __DragonFly__, 358 | __linux__ and __APPLE__. On others only short options provided by 359 | getopt(3) are used. This means that long option may to not work 360 | on your platform. 361 | 362 | fix in manual page: accept(2) -> select(2) 363 | 364 | code clean-ups 365 | 366 | .sinclude removed from Makefile 367 | 368 | tests/test.sh: 369 | diff -U ---> standard diff -C10 370 | gawk appeared by mistake removed 371 | 372 | ====================================================================== 373 | Version 0.10.0, by Aleksey Cheusov, Sun, 31 Aug 2008 21:37:59 +0300 374 | 375 | Lots of new regression tests 376 | 377 | README file: 'make test' is documented 378 | 379 | ADDED: -s option 380 | Partially ordered set of tasks are read from stdin. 381 | 382 | Instead of autonomous tasks, graph of the tasks is read from 383 | stdin. In this mode every task can either FAIL or SUCCEED. As 384 | always an empty line output by command means end of task. The 385 | line before it shows an EXIT STATUS of the task. The word 386 | "failure" means failure, "success" - success. 387 | See examples/1_div_x/1_div_X_cmd for the sample. An input line 388 | (paexec's stdin) should contain either single task without 389 | spaces inside or two tasks separated by single space character, 390 | e.g. task1task2. task1task2 line means that task1 391 | must be done before task2 and it is mandatory, that is if task1 392 | fail all dependent tasks (including task2) are also failed 393 | recursively. Tasks having dependencies are started only after 394 | all dependencies are succeeded. When a task succeeds paexec 395 | outputs "success" word just before end_of_task marker (see -e 396 | or -E), otherwise "failure" word is output followed by a list 397 | of tasks failed because of it. 398 | 399 | Samples: 400 | 401 | tasks (examples/make_package/make_package_tasks file) 402 | 403 | textproc/dictem 404 | devel/autoconf wip/libmaa 405 | devel/gmake wip/libmaa 406 | wip/libmaa wip/dict-server 407 | wip/libmaa wip/dict-client 408 | devel/m4 wip/dict-server 409 | devel/byacc wip/dict-server 410 | devel/byacc wip/dict-client 411 | devel/flex wip/dict-server 412 | devel/flex wip/dict-client 413 | devel/glib2 414 | devel/libjudy 415 | 416 | command (examples/make_package/make_package_cmd__flex) 417 | 418 | #!/usr/bin/awk -f 419 | { 420 | print $0 # print a package name 421 | 422 | if ($0 == "devel/flex") 423 | print "failure" # cannot build flex ;-) 424 | else 425 | print "success" # all other packages are ok 426 | 427 | print "" # end of task marker 428 | fflush() 429 | } 430 | 431 | output of "paexec -s -l -c make_package_cmd__flex -n +10 \ 432 | < make_package_tasks" 433 | 434 | 3 devel/autoconf 435 | 3 success 436 | 4 devel/gmake 437 | 4 success 438 | 7 devel/m4 439 | 7 success 440 | 8 devel/byacc 441 | 8 success 442 | 9 devel/flex 443 | 9 failure 444 | 9 devel/flex wip/dict-server wip/dict-client 445 | 10 devel/glib2 446 | 10 success 447 | 11 devel/libjudy 448 | 11 success 449 | 1 textproc/dictem 450 | 1 success 451 | 2 wip/libmaa 452 | 2 success 453 | 454 | ====================================================================== 455 | Version 0.9.0, by Aleksey Cheusov, Sun, 15 Jun 2008 10:23:52 +0300 456 | 457 | -t '' means "no transport". This significantly simplifies writing 458 | shell scripts with paexec. Added: tests for this case. 459 | 460 | paexec has no limited internal buffers anymore. All they are 461 | resized automatically as it is needed. PAEXEC_BUFSIZE environment 462 | variable sets an *initial* buffer size, not *maximum* one. 463 | 464 | README: note about non-standard function getopt_long, and advice 465 | how to build paexec on platforms with no getopt_long support 466 | (HP-UX, Solaris etc.). 467 | 468 | More regressions tests 469 | 470 | paexec.1: minor corrections. 471 | 472 | 'make test' fix: In case regression test fails, 'make test' exits 473 | with non-zero exit status. 474 | 475 | paexec can be built with ancient version pmake-1.45 (found in som 476 | Linux distributions). 477 | 478 | paexec -h|--help outputs messages to stderr - my new religion :-) 479 | 480 | ====================================================================== 481 | Version 0.8.0, by Aleksey Cheusov, Tue, 4 Mar 2008 00:32:56 +0200 482 | 483 | New options implemented: -E, -i and -I 484 | See manual page for details. 485 | 486 | Fixes and minor improvements in the documentation 487 | 488 | ====================================================================== 489 | Version 0.7.0, by Aleksey Cheusov, Tue, 26 Feb 2008 23:57:15 +0200 490 | 491 | new -e|--eot option implemented. 492 | It prints the end-of-task marker (an empty line) to stdout. 493 | See manual page. 494 | 495 | WARNS=4 to see compilation warnings. 496 | several gcc warnings fixed. 497 | 498 | fixed: '-n +0' and '-n ""' now fails with error message. 499 | 500 | minor fix in man page 501 | 502 | minor clean-ups 503 | 504 | ====================================================================== 505 | Version 0.6.0, by Aleksey Cheusov, Thu, 24 Jan 2008 21:31:26 +0200 506 | 507 | First publicly available release 508 | 509 | ====================================================================== 510 | -------------------------------------------------------------------------------- /doc/TODO: -------------------------------------------------------------------------------- 1 | paexec: 2 | - paexec: print exit status of task 3 | - EXIT STATUS section in man page 4 | - SIGUSR2 for graceful exit, i.e. complete currently running tasks 5 | and then exit (Thanks to Mikhailian for idea) 6 | - New option -gg (double -g) for reading graph of tasks in 7 | lower-to-upper order and running new tasks as soon as they appear 8 | on input. If tasks appears on input slowly (distbb/pkgsrc) this 9 | may significantly improve parallelism and reduce total running time 10 | - pamake -- 1) shell/whatever-based script with special section that 11 | describes hosts, dependency graph of targets and other metainfo, 12 | where functions are "targets" 2) preprocessing (m4,cpp,whatever). 13 | Maybe it make sense to see dsh/pdsh. 14 | - mimic dsh and pdsh 15 | - PAEXEC_COMMAND as a default for -c 16 | - pass PAEXEC_NODEID to appropriate nodes 17 | - -n 'server1 server2 +5' two remote servers and five local 18 | - Option -n%'node_[1-20,22-30]' 19 | - non-blocking write(2) for sending task to the slave hosts/CPUs. 20 | - SIGINFO for displaying current status of tasks 21 | - SIGUSR1 for trying restore connection to dead client hosts 22 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | .include 2 | -------------------------------------------------------------------------------- /examples/Makefile.inc: -------------------------------------------------------------------------------- 1 | .include "../Makefile.inc" 2 | -------------------------------------------------------------------------------- /examples/all_substr/Makefile: -------------------------------------------------------------------------------- 1 | INSCRIPTS = cmd_all_substr 2 | 3 | SCRIPTS = ${INSCRIPTS} run_all_substr 4 | SCRIPTSDIR = ${EGDIR}/${.CURDIR:T} 5 | 6 | .include 7 | -------------------------------------------------------------------------------- /examples/all_substr/cmd_all_substr.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | { 4 | # Read tasks line-by-line 5 | 6 | # Our task is to print all not empty 7 | # substrings of the input string 8 | len=length($0) 9 | 10 | for (i=1; i <= len; ++i){ 11 | for (j=i; j <= len; ++j){ 12 | sz = j-i+1 13 | printf "substr[%d,%d]=%s\n", i, sz, substr($0, i, sz) 14 | } 15 | } 16 | 17 | # End of task marker, empty line by default 18 | print "" 19 | 20 | # In the end, stdout must be flushed 21 | fflush() 22 | } 23 | -------------------------------------------------------------------------------- /examples/all_substr/run_all_substr: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Obtains all substrings in parallel using 3 subprocesses. 4 | # Subprocesses are run on the same host where paexec is run. 5 | # Output lines are prepanded with task number (-l) 6 | # and process pid (-p). 7 | # In the end of task, an empty line is added (-e) 8 | # Output is sliced. 9 | paexec -lpe -c "`pwd`/cmd" -n +3 < 8 | -------------------------------------------------------------------------------- /examples/cc_wrapper/cmd_cc_wrapper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | while read fn; do # we obtain .c files from stdin 6 | log1=${fn%%.c}.log1 7 | log2=${fn%%.c}.log2 8 | obj=${fn%%.c}.o 9 | 10 | if ${CC} ${CPPFLAGS} ${CFLAGS} -c -o $obj ${fn} > $log1 2> $log2; then 11 | echo "done: $fn" 12 | else 13 | echo "failed: $fn" 14 | fi 15 | 16 | echo '' # end of job 17 | done 18 | -------------------------------------------------------------------------------- /examples/cc_wrapper/func1.c: -------------------------------------------------------------------------------- 1 | int a; 2 | -------------------------------------------------------------------------------- /examples/cc_wrapper/func2.c: -------------------------------------------------------------------------------- 1 | int b; 2 | -------------------------------------------------------------------------------- /examples/cc_wrapper/func3.c: -------------------------------------------------------------------------------- 1 | int c; 2 | -------------------------------------------------------------------------------- /examples/cc_wrapper/run_cc_wrapper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # here we compile .c sources in parallel on two hosts: localhost and chen 4 | # a la distcc 5 | paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cmd" \ 6 | -n 'host1 host2' \ 7 | -t '/usr/bin/ssh -x' < 8 | -------------------------------------------------------------------------------- /examples/cc_wrapper2/func1.c: -------------------------------------------------------------------------------- 1 | int a; 2 | -------------------------------------------------------------------------------- /examples/cc_wrapper2/func2.c: -------------------------------------------------------------------------------- 1 | int b; 2 | -------------------------------------------------------------------------------- /examples/cc_wrapper2/func3.c: -------------------------------------------------------------------------------- 1 | int c; 2 | -------------------------------------------------------------------------------- /examples/cc_wrapper2/run_cc_wrapper2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # here we compile .c sources in parallel using 4 parallel processes 4 | 5 | : ${CC:=cc} 6 | : ${CFLAGS:=-O0} 7 | 8 | ls -1 $PWD/*.c | 9 | paexec -c "env $CC $CFLAGS -c " -n +4 -x 10 | -------------------------------------------------------------------------------- /examples/dirtest/Makefile: -------------------------------------------------------------------------------- 1 | SCRIPTS = run_dirtest 2 | SCRIPTSDIR = ${EGDIR}/${.CURDIR:T} 3 | 4 | FILES = tasks 5 | FILESDIR = ${EGDIR}/${.CURDIR:T} 6 | 7 | .include 8 | -------------------------------------------------------------------------------- /examples/dirtest/run_dirtest: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # paexec -gx: exit status of command passed 5 | # to -c is treated as a status of task 6 | # 7 | 8 | paexec -gx -l -c 'test -d' -md=';' -n +3 9 | -------------------------------------------------------------------------------- /examples/dirtest/tasks: -------------------------------------------------------------------------------- 1 | /nonexistant;/nonexistant/subdir 2 | /nonexistant/subdir;/nonexistant/subdir/subsubdir 3 | examples 4 | examples/divide 5 | examples/all_substr 6 | examples/cc_wrapper 7 | examples/make_package 8 | examples/toupper 9 | examples/cc_wrapper2 10 | examples/wav2flac 11 | paexec 12 | tests;tests/transp_closed_stdin 13 | tests;tests/fakeflac 14 | tests;tests/fakeflac 15 | tests;tests/scripts 16 | /etc;/etc/dir with spaces 17 | /etc/dir with spaces;/etc/dir with spaces/subdir 18 | -------------------------------------------------------------------------------- /examples/divide/Makefile: -------------------------------------------------------------------------------- 1 | INSCRIPTS = cmd_divide cmd_divide2 2 | 3 | SCRIPTS = ${INSCRIPTS} run_divide run_divide2 4 | SCRIPTSDIR = ${EGDIR}/${.CURDIR:T} 5 | 6 | .include 7 | -------------------------------------------------------------------------------- /examples/divide/cmd_divide.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | # This a sample command passed to paexec via option -c 4 | 5 | 6 | { 7 | # Read tasks line-by-line 8 | 9 | if ($1 != 0){ 10 | # Result which should not contain empty lines 11 | print "1/" $1 "=" 1/$1 12 | 13 | # "paexec -g" requires either "success" or "failure" in the end. 14 | # For non-zero input number x we are able to calculate 1/x 15 | print "success" 16 | }else{ 17 | # Oops, dependent tasks will fail as well 18 | print "Cannot calculate 1/0" 19 | 20 | # "paexec -g" requires either "success" or "failure" in the end. 21 | print "failure" 22 | } 23 | 24 | # End of task marker, empty line by default 25 | print "" 26 | 27 | # In the end, stdout must be flushed 28 | fflush() 29 | } 30 | -------------------------------------------------------------------------------- /examples/divide/cmd_divide2.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | # The same as "cmd" but with non-standard "success", 4 | # "failure" and "eot" strings. 5 | 6 | BEGIN { 7 | EOT = ENVIRON ["PAEXEC_EOT"] 8 | } 9 | 10 | { 11 | # Read tasks line-by-line 12 | 13 | if ($1 != 0){ 14 | # Result which should not contain empty lines 15 | print "1/" $1 "=" 1/$1 16 | 17 | # "paexec -g" requires either "success" or "failure" in the end. 18 | # For non-zero input number x we are able to calculate 1/x 19 | print "Ura!" 20 | }else{ 21 | # Oops, dependent tasks will fail as well 22 | print "Cannot calculate 1/0" 23 | 24 | # "paexec -g" requires either "success" or "failure" in the end. 25 | print "Zhopa!" 26 | } 27 | 28 | # End of task marker, empty line by default 29 | print EOT 30 | 31 | # In the end, stdout must be flushed 32 | fflush() 33 | } 34 | -------------------------------------------------------------------------------- /examples/divide/run_divide: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Calculate 1/$1 values in 3 parallel subprocesses. 4 | # Dependency graph of tasks comes from stdin. 5 | # Tasks 7, 8, 9 and 10 will be marked as failed 6 | # due to task 0 failure. See cmd for details. 7 | 8 | paexec -s -l -c cmd_divide -n +3 < 10 | -------------------------------------------------------------------------------- /examples/make_package/cmd_make_package.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | { 4 | # Read package names one-by-one 5 | 6 | # Suppose we built the package... 7 | print $0 8 | 9 | # ...succesfully 10 | print "success" 11 | 12 | # End of task marker, empty line by default 13 | print "" 14 | 15 | # In the end, stdout must be flushed 16 | fflush() 17 | } 18 | -------------------------------------------------------------------------------- /examples/make_package/cmd_xxx_failed_make_package.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | BEGIN { 4 | regexp = ARGV [1] 5 | ARGV [1] = "-" 6 | } 7 | 8 | { 9 | print $0 10 | if ($1 ~ regexp) 11 | print "failure" 12 | else 13 | print "success" 14 | 15 | print "" # end of task marker 16 | fflush() 17 | } 18 | -------------------------------------------------------------------------------- /examples/make_package/run_cycle_make_package: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Attempt to build packages with cyclic dependencies 4 | paexec -g -le -c "`pwd`/cmd" -n +3 < tasks_cycle 5 | -------------------------------------------------------------------------------- /examples/make_package/run_make_package: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Build packages taking a dependency graph on input 4 | # and sort the sliced output using sort(1) -- first task 5 | # before the second one and so on. 6 | paexec -g -le -c "`pwd`/cmd" -n +3 < tasks | paexec_reorder -g -Ms 7 | -------------------------------------------------------------------------------- /examples/make_package/tasks: -------------------------------------------------------------------------------- 1 | textproc/dictem 2 | devel/autoconf wip/libmaa 3 | devel/gmake wip/libmaa 4 | wip/libmaa wip/dict-server 5 | wip/libmaa wip/dict-client 6 | devel/m4 wip/dict-server 7 | devel/byacc wip/dict-server 8 | devel/byacc wip/dict-client 9 | devel/flex wip/dict-server 10 | devel/flex wip/dict-client 11 | devel/glib2 12 | devel/libjudy 13 | -------------------------------------------------------------------------------- /examples/make_package/tasks2: -------------------------------------------------------------------------------- 1 | devel/flex wip/dict-server 2 | devel/flex wip/dict-client 3 | wip/dict-server wip/pkg_online 4 | wip/dict-client wip/pkg_online 5 | -------------------------------------------------------------------------------- /examples/make_package/tasks_cycle: -------------------------------------------------------------------------------- 1 | devel/gettext-lib devel/gmake 2 | devel/gmake lang/gcc 3 | pkgtools/pkg_install-info lang/gcc 4 | devel/libtool-base devel/gettext-lib 5 | sysutils/checkperms devel/gettext-lib 6 | lang/gcc converters/libiconv 7 | lang/gcc devel/gettext-lib 8 | -------------------------------------------------------------------------------- /examples/toupper/Makefile: -------------------------------------------------------------------------------- 1 | INSCRIPTS = cmd_toupper 2 | 3 | SCRIPTS = ${INSCRIPTS} run_toupper 4 | SCRIPTSDIR = ${EGDIR}/${.CURDIR:T} 5 | 6 | .include 7 | -------------------------------------------------------------------------------- /examples/toupper/cmd_toupper.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | BEGIN { 4 | EOT = ENVIRON ["PAEXEC_EOT"] 5 | } 6 | 7 | { 8 | # Prepand every line with a space in order 9 | # to avoid empty lines on output 10 | printf " " 11 | 12 | # Single-line output 13 | print toupper($0) 14 | 15 | # End of task marker, empty line by default 16 | print EOT 17 | 18 | # In the end, stdout must be flushed 19 | fflush() 20 | } 21 | -------------------------------------------------------------------------------- /examples/toupper/run_toupper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Converts all strings to upper case in parallel. 4 | # Subprocesses are run on the same host where paexec is run. 5 | # Input may contains empty lines. 6 | 7 | input (){ 8 | cat <<'EOF' 9 | My English is awesome. 10 | Who is absent? 11 | My name is Aleksey. 12 | I was born in USSR. 13 | London is a capital of Great Britain. 14 | Who is on duty today? :-) 15 | EOF 16 | } 17 | 18 | input | paexec -c "`pwd`/cmd" -n +2 | cut -b 2- 19 | -------------------------------------------------------------------------------- /examples/wav2flac/Makefile: -------------------------------------------------------------------------------- 1 | SCRIPTS = run 2 | SCRIPTSDIR ?= ${FILESDIR} 3 | 4 | .include 5 | -------------------------------------------------------------------------------- /examples/wav2flac/run_wav2flac: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Here we convert .wav files to .flac 4 | # Usage: ./run /wav/directory a_number_of_parallel_processes 5 | # Ex: ./run $HOME/wavs 7 6 | 7 | set -e 8 | 9 | test $# -eq 2 10 | 11 | dir="$1" # directory with .wav files 12 | num="$2" # a number of flac(1) processes 13 | 14 | ls -1 "$dir"/*.wav | paexec -x -c 'flac --silent' -n +"$num" 15 | -------------------------------------------------------------------------------- /help.mk: -------------------------------------------------------------------------------- 1 | HELP_MSG.paexec = "paexec, paexec_reorder and pareorder executables" 2 | HELP_MSG.paargs = "paargs executable" 3 | HELP_MSG.tests = 'tests for target "test"' 4 | HELP_MSG.examples = "samples of use" 5 | HELP_MSG.presentation = "PDF presentation" 6 | HELP_MSG.doc = "LICENSE, README, TODO and NEWS" 7 | -------------------------------------------------------------------------------- /paargs/Makefile: -------------------------------------------------------------------------------- 1 | INSCRIPTS = paargs 2 | SCRIPTS = ${INSCRIPTS} 3 | 4 | MAN = paargs.1 5 | 6 | CLEANFILES = paargs.1 7 | 8 | .PHONY: _prepdist 9 | _prepdist: ${MAN} 10 | 11 | .include 12 | -------------------------------------------------------------------------------- /paargs/paargs.in: -------------------------------------------------------------------------------- 1 | #!@sh@ 2 | 3 | usage (){ 4 | cat 1>&2 <<'EOF' 5 | paargs -- wrapper for paexec 6 | usage: paargs [OPTIONS] 7 | OPTIONS: 8 | -h display this help 9 | -V display version 10 | 11 | -P <+num> number of subprocesses to run 12 | list of nodes separated by space character 13 | <:filename> filename containing a list of nodes, one node per line 14 | -t set a transport program 15 | -c command with its arguments. By default, free arguments 16 | are used for setting command and its arguments 17 | 18 | -I execute command for each task, replacing one 19 | or more occurrences of replstr with the entire task 20 | -X ignore calculator's stdout 21 | -f flushes stdout after recieving an end-of-task signal 22 | -0 change paexec to expect NUL characters as separators 23 | instead of newline 24 | 25 | -d debug mode, for debugging only 26 | 27 | -Z passed directly to paexec 28 | -z passed directly to paexec 29 | 30 | -m s= set an alternative for 'success' message 31 | f= set an alternative for 'failure' message 32 | F= set an alternative for 'fatal' message 33 | t= set an alternative for EOT marker 34 | d= set the delimiter for -g mode, 35 | no delimiter by default 36 | w= set an alternative for 'weight:' marker 37 | 38 | -P is a mandatory option 39 | EOF 40 | } 41 | 42 | command='paexec -Cxleg -md=' 43 | version='@version@' 44 | 45 | shquote (){ 46 | __cmd=`printf '%s\n' "$1" | sed "s|'|'\\\\\''|g"` 47 | printf "%s\n" "'$__cmd'" 48 | } 49 | 50 | while getopts 0c:dfI:hm:P:t:VXzZ: f; do 51 | case "$f" in 52 | '?') 53 | exit 1;; 54 | h) 55 | usage 56 | exit 0;; 57 | V) 58 | echo "paargs $version written by Aleksey Cheusov" 59 | exit 0;; 60 | P) 61 | addon=$(shquote "$OPTARG") 62 | command="$command -n$addon";; 63 | t) 64 | addon=$(shquote "$OPTARG") 65 | command="$command -t$addon";; 66 | X) 67 | command="$command -X";; 68 | f) 69 | command="$command -E";; 70 | m) 71 | addon=$(shquote "$OPTARG") 72 | command="$command -m$addon";; 73 | I) 74 | addon=$(shquote "$OPTARG") 75 | command="$command -J$addon";; 76 | 0) 77 | command="$command -0";; 78 | d) 79 | command="$command -d";; 80 | Z) 81 | addon=$(shquote "$OPTARG") 82 | command="$command -w -Z$addon";; 83 | z) 84 | command="$command -wz";; 85 | c) 86 | command_specified=1 87 | addon=$(shquote "$OPTARG") 88 | command="$command -c$addon";; 89 | esac 90 | done 91 | shift `expr $OPTIND - 1` 92 | 93 | if test -n "$command_specified"; then 94 | if test $# -ne 0; then 95 | echo 'paargs: extra arguments. Run paargs -h for details' 1>&2 96 | exit 1 97 | fi 98 | else 99 | if test $# -eq 0; then 100 | echo 'paargs: missing arguments. Run paargs -h for details' 1>&2 101 | exit 1 102 | fi 103 | fi 104 | 105 | for fa in "$@"; do 106 | addon=$(shquote "$fa") 107 | command="$command $addon" 108 | done 109 | 110 | #echo $command 1>&2 111 | eval "$command" 112 | -------------------------------------------------------------------------------- /paargs/paargs.pod: -------------------------------------------------------------------------------- 1 | =head1 NAME 2 | 3 | paargs - wrapper for paexec similar to xargs 4 | 5 | =head1 SYNOPSIS 6 | 7 | B I<[OPTIONS]> 8 | 9 | =head1 DESCRIPTION 10 | 11 | B is a simple wrapper over B that simplifies 12 | use of B's capalities. The use of B is actually very 13 | similar to B. B runs B 14 | with the following options enabled by default: 15 | 16 | =over 4 17 | 18 | =item * 19 | 20 | I<-C> -- for specifying I as free arguments, 21 | 22 | =item * 23 | 24 | I<-x> -- for being B-like, 25 | 26 | =item * 27 | 28 | I<-l> -- 0-based task number is included to the output of B, this 29 | allows to reorder the sliced output, 30 | 31 | =item * 32 | 33 | I<-e> -- for the same purposes as I<-l>, 34 | 35 | =item * 36 | 37 | I<-g> -- this flag allows analysis of I's exit status, 38 | 39 | =item * 40 | 41 | I<-md=> -- disables task delimiter for I<-g> mode enabled by default, 42 | as a result the whole line given on input is considered as a task. 43 | 44 | =back 45 | 46 | =head1 OPTIONS 47 | 48 | =over 4 49 | 50 | =item B<-h> 51 | 52 | Display this help. 53 | 54 | =item B<-V> 55 | 56 | Display version. 57 | 58 | =item B<-P> I 59 | 60 | Same as I<-n> in B. 61 | 62 | =item B<-t> I 63 | 64 | Passed directly to B. 65 | 66 | =item B<-c> I 67 | 68 | Command with its arguments. By default, free arguments 69 | are used for setting command and its arguments. 70 | 71 | =item B<-I> I 72 | 73 | Execute command for each task, replacing one 74 | or more occurrences of I with the entire task. 75 | 76 | =item B<-X> 77 | 78 | Passed directly to B. 79 | 80 | =item B<-f> 81 | 82 | Flushes stdout after recieving an end-of-task line. 83 | 84 | =item B<-0> 85 | 86 | Passed directly to B. 87 | 88 | =item B<-Z> I 89 | 90 | Passed directly to B. 91 | 92 | =item B<-z> 93 | 94 | Passed directly to B. 95 | 96 | =item B<-m> I 97 | 98 | Passed directly to B. 99 | 100 | =item B<-d> 101 | 102 | Turn on a debugging mode (for debugging purposes only) 103 | 104 | =back 105 | 106 | =head1 BUGS/FEEDBACK 107 | 108 | Please send any comments, questions, bug reports etc. to me by e-mail 109 | or (even better) register them at sourceforge project home. Feature 110 | requests are also welcomed. 111 | 112 | =head1 HOME 113 | 114 | L 115 | 116 | =head1 SEE ALSO 117 | L 118 | -------------------------------------------------------------------------------- /paexec/Makefile: -------------------------------------------------------------------------------- 1 | # initial buffer size for the tasks and results 2 | BUFSIZE ?= ${USE_BUFSIZE:U4096} 3 | 4 | # optimization flags 5 | COPTS ?= ${USE_COPTS:U-O2 -g} 6 | 7 | # 8 | PROG = paexec 9 | SRCS = paexec.c wrappers.c tasks.c nodes.c signals.c 10 | 11 | SCRIPTS = paexec_reorder 12 | 13 | MAN = paexec.1 paexec_reorder.1 14 | 15 | LINKS = ${BINDIR}/paexec_reorder ${BINDIR}/pareorder 16 | MLINKS = paexec_reorder.1 pareorder.1 17 | 18 | WARNS ?= 4 19 | WARNERR ?= yes 20 | 21 | CFLAGS += -DPAEXEC_VERSION='"${VERSION}"' 22 | CFLAGS += -DBUFSIZE=${BUFSIZE} 23 | 24 | MKC_FEATURES = strlcat strlcpy SLIST RB getdelim 25 | MKC_SOURCE_FUNCLIBS = shquote 26 | 27 | MKC_REQUIRE_HEADERS = maa.h 28 | MKC_REQUIRE_FUNCLIBS = pr_open2:maa 29 | MKC_REQUIRE_FUNCS6 = pr_open2:maa.h 30 | 31 | MKC_COMMON_DEFINES = -D_GNU_SOURCE 32 | MKC_COMMON_HEADERS = unistd.h stdlib.h 33 | MKC_CHECK_TYPES = intptr_t:stdint.h intptr_t:inttypes.h 34 | MKC_CHECK_HEADERS = sys/select.h 35 | MKC_CHECK_FUNCS3 = shquote 36 | 37 | CLEANFILES = *~ core* *.1 *.html ktrace* ChangeLog *.tmp 38 | 39 | .PHONY: _prepdist 40 | _prepdist: ${MAN} 41 | 42 | ############################################################ 43 | .include 44 | -------------------------------------------------------------------------------- /paexec/common.h: -------------------------------------------------------------------------------- 1 | #ifndef _COMMON_H_ 2 | #define _COMMON_H_ 3 | 4 | #if defined(__GNUC__) 5 | #define attr_unused __attribute__ ((unused)) 6 | #else 7 | #define attr_unused 8 | #endif 9 | 10 | #endif /* _COMMON_H_ */ 11 | -------------------------------------------------------------------------------- /paexec/decls.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #ifndef _DECLS_H_ 25 | #define _DECLS_H_ 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #ifndef HAVE_FUNC3_SHQUOTE 33 | size_t shquote(const char *arg, char *buf, size_t bufsize); 34 | #endif 35 | 36 | extern char eol_char; 37 | 38 | #endif // _DECLS_H_ 39 | -------------------------------------------------------------------------------- /paexec/nodes.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2013 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #include "nodes.h" 25 | #include "wrappers.h" 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | char **nodes = NULL; 34 | int nodes_count = 0; 35 | 36 | static void nodes_create__count(const char *nodes_str) 37 | { 38 | int i; 39 | 40 | nodes_count = (int) strtol(nodes_str, NULL, 10); 41 | if (nodes_count == (int) LONG_MAX) 42 | err__fatal_errno(NULL, "paexec: invalid option -n:"); 43 | 44 | nodes = xmalloc(nodes_count * sizeof(nodes [0])); 45 | 46 | for (i=0; i < nodes_count; ++i){ 47 | char num [50]; 48 | snprintf(num, sizeof(num), "%d", i); 49 | nodes [i] = xstrdup(num); 50 | } 51 | } 52 | 53 | static void nodes_create__list(char *nodes_str) 54 | { 55 | char *last = NULL; 56 | char *p = nodes_str; 57 | char c; 58 | 59 | /* "node1 nodes2 ..." format */ 60 | for (;;){ 61 | c = *p; 62 | 63 | switch (c){ 64 | case ' ': 65 | case '\t': 66 | case '\r': 67 | case '\n': 68 | case 0: 69 | if (last){ 70 | *p = 0; 71 | 72 | ++nodes_count; 73 | 74 | nodes = xrealloc( 75 | nodes, 76 | nodes_count * sizeof(*nodes)); 77 | 78 | nodes [nodes_count - 1] = xstrdup(last); 79 | 80 | last = NULL; 81 | } 82 | break; 83 | default: 84 | if (!last){ 85 | last = p; 86 | } 87 | break; 88 | } 89 | 90 | if (!c) 91 | break; 92 | 93 | ++p; 94 | } 95 | } 96 | 97 | static void nodes_create__file(const char *nodes_str) 98 | { 99 | char node [4096]; 100 | FILE *fd = fopen(nodes_str, "r"); 101 | size_t len = 0; 102 | 103 | if (!fd) 104 | err__fatal_errno(NULL, "paexec: Cannot obtain a list of nodes"); 105 | 106 | while (fgets(node, sizeof(node), fd)){ 107 | len = strlen(node); 108 | if (len > 0 && node [len-1] == '\n') 109 | node [len-1] = 0; 110 | 111 | ++nodes_count; 112 | 113 | nodes = xrealloc( 114 | nodes, 115 | nodes_count * sizeof(*nodes)); 116 | 117 | nodes [nodes_count - 1] = xstrdup(node); 118 | } 119 | 120 | fclose(fd); 121 | } 122 | 123 | void nodes_create(const char *nodes_str) 124 | { 125 | unsigned char c0 = nodes_str [0]; 126 | char *nodes_copy; 127 | 128 | if (c0 == '+'){ 129 | /* "+NUM" format */ 130 | nodes_create__count(nodes_str + 1); 131 | }else if (c0 == ':'){ 132 | /* "+NUM" format */ 133 | nodes_create__file(nodes_str + 1); 134 | }else if (isalnum(c0) || c0 == '/' || c0 == '_'){ 135 | /* list of nodes */ 136 | nodes_copy = xstrdup(nodes_str); 137 | nodes_create__list(nodes_copy); 138 | xfree(nodes_copy); 139 | }else{ 140 | err__fatal(NULL, "paexec: invalid argument for option -n"); 141 | } 142 | 143 | /* final check */ 144 | if (nodes_count == 0) 145 | err__fatal(NULL, "paexec: invalid argument for option -n"); 146 | } 147 | 148 | void nodes_destroy(void) 149 | { 150 | int i; 151 | 152 | if (nodes){ 153 | for (i=0; i < nodes_count; ++i){ 154 | if (nodes [i]) 155 | xfree(nodes [i]); 156 | } 157 | xfree(nodes); 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /paexec/nodes.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2011 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | extern char **nodes; 25 | extern int nodes_count; 26 | void nodes_create(const char *nodes_str); 27 | void nodes_destroy(void); 28 | -------------------------------------------------------------------------------- /paexec/paexec.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2013 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #ifdef HAVE_CONFIG_H 25 | /* if you need, add extra includes to config.h */ 26 | #include "config.h" 27 | #endif 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include 41 | 42 | /***********************************************************/ 43 | 44 | #include "decls.h" 45 | 46 | #include "wrappers.h" 47 | #include "common.h" 48 | #include "tasks.h" 49 | #include "nodes.h" 50 | #include "signals.h" 51 | 52 | #include "mkc_strlcpy.h" 53 | 54 | #ifndef BUFSIZE 55 | #define BUFSIZE 2048 56 | #endif 57 | 58 | #ifndef PAEXEC_VERSION 59 | #define PAEXEC_VERSION "x.y.z" 60 | #endif 61 | 62 | static void usage(void) 63 | { 64 | fprintf(stderr, "\ 65 | paexec -- parallel executor\n\ 66 | that distributes tasks over CPUs or machines in a network.\n\ 67 | usage: paexec [OPTIONS]\n\ 68 | paexec -C [OPTIONS] cmd [args...]\n\ 69 | OPTIONS:\n\ 70 | -h give this help\n\ 71 | -V show version\n\ 72 | \n\ 73 | -n <+num> number of subprocesses to run\n\ 74 | -n list of nodes separated by space character\n\ 75 | -n <:filename> filename containing a list of nodes, one node per line\n\ 76 | -c set a command\n\ 77 | -C use free arguments as a command and its arguments\n\ 78 | -t set a transport program\n\ 79 | \n\ 80 | -x run command once per task\n\ 81 | -X implies -x and ignore calculator's stdout.\n\ 82 | -y magic line is used as an end-of-task marker\n\ 83 | -0 change paexec to expect NUL character as\n\ 84 | a line separator instead of newline\n\ 85 | \n\ 86 | -r include a node (or a number) to the output\n\ 87 | -l include 0-based task number to the output\n\ 88 | -p include a pid of subprocess to the output\n\ 89 | \n\ 90 | -e print an empty line when end-of-task is reached\n\ 91 | -E implies -e and flushes stdout\n\ 92 | \n\ 93 | -i copy input lines (i.e. tasks) to stdout\n\ 94 | -I implies -i and flushes stdout\n\ 95 | \n\ 96 | -s|-g graph of tasks is given on stdin, by default a list of\n\ 97 | independent tasks is read from stdin\n\ 98 | \n\ 99 | -d debug mode, for debugging only\n\ 100 | \n\ 101 | -z failed nodes are marked as dead\n\ 102 | -Z timeout to restart faild command, imply -z\n\ 103 | -w wait for restoring nodes (needs -Z)\n\ 104 | \n\ 105 | -W heavier tasks are processed first, a weight\n\ 106 | of task is a sum of its own weight and weights\n\ 107 | of all tasks that depend on it,\n\ 108 | directly or indirectly\n\ 109 | \n\ 110 | -m s= set an alternative for 'success' message\n\ 111 | f= set an alternative for 'failure' message\n\ 112 | F= set an alternative for 'fatal' message\n\ 113 | t= set an alternative for EOT marker\n\ 114 | d= set the delimiter for -g mode.\n\ 115 | The default is space character.\n\ 116 | w= set an alternative for 'weight:' marker\n\ 117 | \n\ 118 | -J replstr\n\ 119 | execute command for each task, replacing\n\ 120 | one or more occurrences of replstr with the entire task.\n\ 121 | \n\ 122 | -n and -c are mandatory options\n\ 123 | \n\ 124 | "); 125 | } 126 | 127 | static const char magic_eot [] = "HG>&OSO@#;L8N;!&.U4ZC_9X:0AF,2Y>SRXAD_7U&QZ5S>N^?Y,I=W?@5"; 128 | 129 | /* arguments */ 130 | static char *arg_nodes = NULL; 131 | static char *arg_cmd = NULL; 132 | static char *arg_transport = NULL; 133 | 134 | /**/ 135 | static int *fd_in = NULL; 136 | static int *fd_out = NULL; 137 | 138 | static char **buf_out = NULL; 139 | static size_t *bufsize_out = NULL; 140 | static size_t *size_out = NULL; 141 | 142 | static int *busy = NULL; 143 | static int busy_count = 0; 144 | 145 | static pid_t *pids = NULL; 146 | 147 | static int *node2taskid = NULL; 148 | 149 | typedef enum { 150 | rt_undef = -1, 151 | rt_success = 0, 152 | rt_failure = 1, 153 | } ret_code_t; 154 | static ret_code_t *ret_codes = NULL; 155 | 156 | static int max_fd = 0; 157 | 158 | static char *buf_stdin = NULL; 159 | 160 | static int alive_nodes_count = 0; 161 | 162 | static int show_pid = 0; 163 | static int show_taskid = 0; 164 | static int show_node = 0; 165 | 166 | static int print_eot = 0; 167 | static int flush_eot = 0; 168 | 169 | static int print_i2o = 0; 170 | static int flush_i2o = 0; 171 | 172 | static int initial_bufsize = BUFSIZE; 173 | 174 | static int debug = 0; 175 | 176 | static char **node2task = NULL; 177 | static size_t *node2task_buf_sz = NULL; 178 | 179 | static int end_of_stdin = 0; 180 | 181 | static const char *msg_success = "success"; 182 | static const char *msg_failure = "failure"; 183 | static const char *msg_fatal = "fatal"; 184 | static const char *msg_weight = "weight:"; 185 | static const char *msg_eot = NULL; 186 | char msg_delim = ' '; /* also used in tasks.c */ 187 | 188 | static const char *shell = NULL; 189 | 190 | static int resistant = 0; 191 | static int resistance_timeout = 0; 192 | static int resistance_last_restart = 0; 193 | 194 | static int wait_mode = 0; 195 | 196 | static int exec_mode = 0; 197 | 198 | static int use_weights = 0; 199 | 200 | char eol_char = '\n'; 201 | 202 | static char replstr[2] = ""; 203 | 204 | struct envvar_entry { 205 | SLIST_ENTRY(envvar_entry) entries; /* List. */ 206 | char* name; 207 | char* value; 208 | }; 209 | static SLIST_HEAD(envvar_head, envvar_entry) envvars = SLIST_HEAD_INITIALIZER(envvars); 210 | 211 | static void add_envvar(const char *name, const char *value) 212 | { 213 | struct envvar_entry *val = calloc(1, sizeof(struct envvar_entry)); 214 | val->name = strdup(name); 215 | val->value = (value ? strdup(value) : NULL); 216 | SLIST_INSERT_HEAD(&envvars, val, entries); 217 | } 218 | 219 | static void assign_str(char **ptr, const char *str) 220 | { 221 | size_t len = strlen(str); 222 | 223 | *ptr = xrealloc(*ptr, len+1); 224 | memcpy(*ptr, str, len+1); 225 | } 226 | 227 | static void close_all_ins(void) 228 | { 229 | int i; 230 | for (i=0; i < nodes_count; ++i){ 231 | if (!busy [i] && fd_in [i] != -1){ 232 | close(fd_in [i]); 233 | fd_in [i] = -1; 234 | } 235 | } 236 | } 237 | 238 | static void bad_input_line(const char *line) 239 | { 240 | char buf [4000]; 241 | snprintf(buf, sizeof(buf), "Bad input line: %s", line); 242 | 243 | err__fatal(NULL, buf); 244 | } 245 | 246 | static void init__read_graph_tasks(void) 247 | { 248 | char *buf = NULL; 249 | size_t buf_sz = 0; 250 | 251 | ssize_t len = 0; 252 | 253 | int id1, id2; 254 | 255 | char *tok1, *tok2, *tok3, *tok; 256 | int tok_cnt; 257 | char *p; 258 | 259 | char buf_copy [2000]; 260 | 261 | /* */ 262 | if (debug){ 263 | fprintf(stderr, "start: init__read_graph_tasks\n"); 264 | } 265 | 266 | /* */ 267 | if (!graph_mode){ 268 | /* completely independent tasks */ 269 | return; 270 | } 271 | 272 | /* reading all tasks with their dependancies */ 273 | while (len = xgetdelim(&buf, &buf_sz, eol_char, stdin), len != -1){ 274 | if (len > 0 && buf [len-1] == '\n'){ 275 | buf [len-1] = 0; 276 | --len; 277 | } 278 | 279 | strlcpy(buf_copy, buf, sizeof(buf_copy)); 280 | 281 | tok1 = tok2 = tok3 = tok = NULL; 282 | tok_cnt = 0; 283 | 284 | for (p=buf; 1; ++p){ 285 | char ch = *p; 286 | if (ch == msg_delim){ 287 | *p = 0; 288 | } 289 | 290 | if (!tok) 291 | tok = p; 292 | 293 | if (*p == 0){ 294 | if (!tok1){ 295 | tok1 = tok; 296 | tok_cnt = 1; 297 | }else if (!tok2){ 298 | tok2 = tok; 299 | tok_cnt = 2; 300 | }else if (!tok3){ 301 | tok3 = tok; 302 | tok_cnt = 3; 303 | }else{ 304 | bad_input_line(buf_copy); 305 | } 306 | tok = NULL; 307 | } 308 | 309 | if (!ch) 310 | break; 311 | } 312 | 313 | if (tok_cnt == 3 && !strcmp(tok1, msg_weight)){ 314 | /* weight: */ 315 | id2 = tasks__add_task(xstrdup(tok2), atoi(tok3)); 316 | continue; 317 | } 318 | 319 | if (tok_cnt == 2){ 320 | /* */ 321 | id1 = tasks__add_task(xstrdup(tok1), 1); 322 | id2 = tasks__add_task(xstrdup(tok2), 1); 323 | 324 | tasks__add_task_arc(id1, id2); 325 | continue; 326 | } 327 | 328 | if (tok_cnt == 1){ 329 | /* */ 330 | id1 = tasks__add_task(xstrdup(tok1), 1); 331 | continue; 332 | } 333 | 334 | bad_input_line(buf_copy); 335 | } 336 | 337 | if (buf) 338 | free(buf); 339 | 340 | /* */ 341 | if (debug){ 342 | fprintf(stderr, "end: init__read_graph_tasks\n"); 343 | } 344 | } 345 | 346 | static char const runner_function1[] = "run() { %s \"$1\"; }"; 347 | static char const runner_function2[] = "run() { %s; }"; 348 | 349 | static char* generate_run_command(void) 350 | { 351 | static char run_command [4096]; 352 | char *repl_ptr = NULL; 353 | 354 | if (replstr[0]){ 355 | repl_ptr = arg_cmd; 356 | while (repl_ptr = strstr(repl_ptr, replstr), repl_ptr != NULL){ 357 | repl_ptr[0] = '$'; 358 | repl_ptr[1] = '1'; 359 | repl_ptr += 2; 360 | } 361 | 362 | snprintf(run_command, sizeof(run_command), runner_function2, arg_cmd); 363 | }else{ 364 | snprintf(run_command, sizeof(run_command), runner_function1, arg_cmd); 365 | } 366 | 367 | if (strlen(run_command) + 1 == sizeof(run_command)){ 368 | err__fatal(__func__, "paexec: Internal error6! (buffer size)"); 369 | } 370 | 371 | // fprintf(stderr, "run_command: %s\n", run_command); 372 | 373 | return run_command; 374 | } 375 | 376 | static void init__postproc_arg_cmd(void) 377 | { 378 | char shq_cmd [4096]; 379 | char cmd [4096]; 380 | char tmp [4096]=""; 381 | char env_str [4096]=""; 382 | char tmp2 [4096]=""; 383 | struct envvar_entry *p; 384 | 385 | if (exec_mode){ 386 | char cond_cmd [4096] = ""; 387 | 388 | if (exec_mode == 'x'){ 389 | strlcpy(tmp, " printf '%s\\n' \"$res\";", sizeof(tmp)); 390 | } 391 | 392 | if (graph_mode){ 393 | snprintf( 394 | cond_cmd, sizeof(cond_cmd), 395 | "if test $ex = 0; then echo '%s'; else echo '%s'; fi;", 396 | msg_success, msg_failure); 397 | } 398 | 399 | if (snprintf(cmd, sizeof(cmd), 400 | "%s; IFS=; " 401 | "while read -r f; do" 402 | " res=`run \"$f\"`;" 403 | " ex=$?;" 404 | " %s" /* printing result */ 405 | " %s" /* condition. success/failure */ 406 | " echo '%s';" /* EOT */ 407 | "done", generate_run_command(), tmp, cond_cmd, magic_eot) >= sizeof(cmd)){ 408 | err__fatal(__func__, "paexec: Internal error7! (buffer size)"); 409 | } 410 | 411 | xfree(arg_cmd); 412 | arg_cmd = xstrdup(cmd); 413 | } 414 | 415 | xshquote(arg_cmd, shq_cmd, sizeof(shq_cmd)); 416 | 417 | /* env(1) arg for environment variables */ 418 | add_envvar("PAEXEC_EOT", msg_eot); 419 | 420 | SLIST_FOREACH(p, &envvars, entries){ 421 | xshquote((p->value ? p->value : ""), tmp, sizeof(tmp)); 422 | if (snprintf(tmp2, sizeof(tmp2), "%s=%s ", p->name, tmp) >= sizeof(tmp2)){ 423 | err__fatal(__func__, "paexec: Internal error! (buffer size)"); 424 | } 425 | strlcat(env_str, tmp2, sizeof(env_str)); 426 | } 427 | 428 | /**/ 429 | if (snprintf(cmd, sizeof(cmd), "env %s %s -c %s", env_str, shell, shq_cmd) >= sizeof(cmd)){ 430 | err__fatal(__func__, "paexec: Internal error! (buffer size)"); 431 | } 432 | xfree(arg_cmd); 433 | arg_cmd = xstrdup(cmd); 434 | 435 | /**/ 436 | if (arg_transport){ 437 | /* one more shquote(3) for ssh-like transport */ 438 | xshquote(arg_cmd, shq_cmd, sizeof(shq_cmd)); 439 | xfree(arg_cmd); 440 | arg_cmd = xstrdup(shq_cmd); 441 | } 442 | 443 | if (strlen(cmd) + 20 >= sizeof(shq_cmd)){ 444 | err__fatal(__func__, "paexec: internal error, buffer size limit"); 445 | } 446 | } 447 | 448 | static void restore_signals(void) 449 | { 450 | unblock_signals(); 451 | set_sig_handler(SIGALRM, SIG_DFL); 452 | set_sig_handler(SIGCHLD, SIG_DFL); 453 | set_sig_handler(SIGPIPE, SIG_DFL); 454 | } 455 | 456 | static void init__child_processes(void) 457 | { 458 | char full_cmd [2000]; 459 | int i; 460 | 461 | /* */ 462 | if (debug){ 463 | fprintf(stderr, "start: init__child_processes\n"); 464 | } 465 | 466 | /* */ 467 | for (i=0; i < nodes_count; ++i){ 468 | if (pids [i] != (pid_t) -1) 469 | continue; 470 | 471 | if (!buf_out [i]){ 472 | /* +1 for \0 and -d option */ 473 | buf_out [i] = xmalloc(initial_bufsize+1); 474 | } 475 | if (!bufsize_out [i]) 476 | bufsize_out [i] = initial_bufsize; 477 | 478 | size_out [i] = 0; 479 | 480 | busy [i] = 0; 481 | 482 | ret_codes [i] = rt_undef; 483 | 484 | if (arg_transport) 485 | snprintf(full_cmd, sizeof(full_cmd), "%s %s %s", 486 | arg_transport, nodes [i], arg_cmd); 487 | else 488 | snprintf(full_cmd, sizeof(full_cmd), "%s", arg_cmd); 489 | 490 | if (debug){ 491 | fprintf(stderr, "running cmd: %s\n", full_cmd); 492 | } 493 | 494 | pids [i] = pr_open2( 495 | full_cmd, 496 | restore_signals, 497 | PR_CREATE_STDIN | PR_CREATE_STDOUT, 498 | &fd_in [i], &fd_out [i], NULL); 499 | 500 | ++alive_nodes_count; 501 | 502 | nonblock(fd_out [i]); 503 | 504 | if (fd_in [i] > max_fd){ 505 | max_fd = fd_in [i]; 506 | } 507 | if (fd_out [i] > max_fd){ 508 | max_fd = fd_out [i]; 509 | } 510 | } 511 | 512 | /* */ 513 | if (debug){ 514 | fprintf(stderr, "end: init__child_processes\n"); 515 | } 516 | } 517 | 518 | static void mark_node_as_dead(int node) 519 | { 520 | if (debug){ 521 | fprintf(stderr, "mark_node_as_dead (%d)\n", node); 522 | } 523 | 524 | if (busy [node]){ 525 | busy [node] = 0; 526 | --busy_count; 527 | tasks__mark_task_as_failed(node2taskid [node]); 528 | --alive_nodes_count; 529 | } 530 | 531 | if (fd_in [node] >= 0) 532 | close(fd_in [node]); 533 | if (fd_out [node] >= 0) 534 | close(fd_out [node]); 535 | 536 | fd_in [node] = -1; 537 | fd_out [node] = -1; 538 | 539 | unblock_signals(); 540 | waitpid(pids [node], NULL, WNOHANG); 541 | block_signals(); 542 | 543 | pids [node] = (pid_t) -1; 544 | } 545 | 546 | static void init(void) 547 | { 548 | /* arrays */ 549 | pids = xmalloc(nodes_count * sizeof(*pids)); 550 | memset(pids,-1, nodes_count * sizeof(*pids)); 551 | 552 | fd_in = xmalloc(nodes_count * sizeof(*fd_in)); 553 | memset(fd_in, -1, nodes_count * sizeof(*fd_in)); 554 | 555 | fd_out = xmalloc(nodes_count * sizeof(*fd_out)); 556 | memset(fd_out, -1, nodes_count * sizeof(*fd_out)); 557 | 558 | node2task = xmalloc(nodes_count * sizeof(*node2task)); 559 | node2task_buf_sz = xmalloc(nodes_count * sizeof(*node2task_buf_sz)); 560 | memset(node2task, 0, nodes_count * sizeof(*node2task)); 561 | memset(node2task_buf_sz, 0, nodes_count * sizeof(*node2task_buf_sz)); 562 | 563 | buf_out = xmalloc(nodes_count * sizeof(*buf_out)); 564 | memset(buf_out, 0, nodes_count * sizeof(*buf_out)); 565 | 566 | bufsize_out = xmalloc(nodes_count * sizeof(*bufsize_out)); 567 | memset(bufsize_out, 0, nodes_count * sizeof(*bufsize_out)); 568 | 569 | size_out = xmalloc(nodes_count * sizeof(*size_out)); 570 | memset(size_out, 0, nodes_count * sizeof(*size_out)); 571 | 572 | busy = xmalloc(nodes_count * sizeof(*busy)); 573 | memset(busy, 0, nodes_count * sizeof(*busy)); 574 | 575 | node2taskid = xmalloc(nodes_count * sizeof(*node2taskid)); 576 | 577 | ret_codes = xmalloc(nodes_count * sizeof(*ret_codes)); 578 | 579 | /* stdin */ 580 | buf_stdin = xmalloc(initial_bufsize); 581 | buf_stdin [0] = 0; 582 | 583 | /* tasks */ 584 | tasks__init(); 585 | 586 | /**/ 587 | init__read_graph_tasks(); 588 | 589 | /**/ 590 | tasks__check_for_cycles(); 591 | 592 | /**/ 593 | switch (use_weights){ 594 | case 1: 595 | tasks__make_sum_weights(); 596 | break; 597 | case 2: 598 | tasks__make_max_weights(); 599 | break; 600 | } 601 | 602 | if (debug) 603 | tasks__print_sum_weights(); 604 | 605 | /* */ 606 | init__postproc_arg_cmd(); 607 | 608 | /* in/out */ 609 | init__child_processes(); 610 | 611 | /* signal handlers */ 612 | set_sigchld_handler(); 613 | set_sigalrm_handler(); 614 | 615 | /* alarm(2) */ 616 | if (resistance_timeout) 617 | alarm(1); 618 | 619 | /* ignore SIGPIPE signal */ 620 | ignore_sigpipe(); 621 | } 622 | 623 | void kill_childs(void) 624 | { 625 | int i; 626 | 627 | if (!pids) 628 | return; 629 | 630 | for (i=0; i < nodes_count; ++i){ 631 | if (pids [i] != (pid_t) -1){ 632 | kill(pids [i], SIGTERM); 633 | } 634 | } 635 | } 636 | 637 | void wait_for_childs(void) 638 | { 639 | int i; 640 | 641 | if (!pids) 642 | return; 643 | 644 | if (debug) 645 | printf("wait for childs\n"); 646 | 647 | for (i=0; i < nodes_count; ++i){ 648 | if (pids [i] != (pid_t) -1){ 649 | mark_node_as_dead(i); 650 | } 651 | } 652 | } 653 | 654 | static int find_free_node(void) 655 | { 656 | int i; 657 | for (i=0; i < nodes_count; ++i){ 658 | if (pids [i] != (pid_t) -1 && !busy [i]) 659 | return i; 660 | } 661 | 662 | err__internal(__func__, "there is no free node"); 663 | return -1; 664 | } 665 | 666 | static void print_header(int num) 667 | { 668 | if (show_node){ 669 | if (nodes && nodes [num]) 670 | printf("%s ", nodes [num]); 671 | else 672 | printf("%d ", num); 673 | } 674 | if (show_taskid){ 675 | printf("%d ", node2taskid [num]); 676 | } 677 | if (show_pid){ 678 | printf("%ld ", (long) pids [num]); 679 | } 680 | } 681 | 682 | static void print_line(int num, const char *line) 683 | { 684 | print_header(num); 685 | printf("%s\n", line); 686 | } 687 | 688 | static void print_EOT(int num) 689 | { 690 | if (print_eot){ 691 | print_line(num, msg_eot); 692 | if (flush_eot) 693 | fflush(stdout); 694 | } 695 | } 696 | 697 | static void send_to_node(void) 698 | { 699 | int n = find_free_node(); 700 | size_t task_len = strlen(current_task); 701 | 702 | assert(n >= 0); 703 | 704 | if (debug){ 705 | printf("send to %d (pid: %ld)\n", n, (long) pids [n]); 706 | } 707 | 708 | busy [n] = 1; 709 | size_out [n] = 0; 710 | node2taskid [n] = current_taskid; 711 | 712 | ++busy_count; 713 | 714 | if (task_len >= node2task_buf_sz [n]){ 715 | node2task_buf_sz [n] = task_len + 1; 716 | node2task [n] = xrealloc(node2task [n], node2task_buf_sz [n]); 717 | } 718 | memcpy(node2task [n], current_task, task_len + 1); 719 | 720 | if (print_i2o){ 721 | print_line(n, current_task); 722 | if (flush_i2o){ 723 | fflush(stdout); 724 | } 725 | } 726 | 727 | if (-1 == write(fd_in [n], current_task, task_len) || 728 | -1 == write(fd_in [n], "\n", 1)) 729 | { 730 | if (resistant){ 731 | mark_node_as_dead(n); 732 | if (msg_fatal [0]) 733 | print_line(n, msg_fatal); 734 | print_EOT(n); 735 | 736 | if (alive_nodes_count == 0 && !wait_mode){ 737 | err__fatal(NULL, "all nodes failed"); 738 | } 739 | return; 740 | }else{ 741 | err__fatal_errno(NULL, "paexec: Sending task to the node failed:"); 742 | } 743 | } 744 | } 745 | 746 | static int unblock_select_block( 747 | int nfds, fd_set * readfds, fd_set * writefds, 748 | fd_set * exceptfds, struct timeval * timeout) 749 | { 750 | int ret; 751 | char msg [200]; 752 | 753 | unblock_signals(); 754 | 755 | do { 756 | errno = 0; 757 | ret = select(nfds, readfds, writefds, exceptfds, timeout); 758 | }while(ret == -1 && errno == EINTR); 759 | 760 | if (ret == -1){ 761 | snprintf(msg, sizeof(msg), "select(2) failed: %s", strerror(errno)); 762 | err__fatal(__func__, msg); 763 | } 764 | 765 | block_signals(); 766 | 767 | return ret; 768 | } 769 | 770 | static int try_to_reinit_failed_nodes(void) 771 | { 772 | if (resistance_timeout && 773 | sigalrm_tics - resistance_last_restart >= resistance_timeout) 774 | { 775 | resistance_last_restart = sigalrm_tics; 776 | init__child_processes(); 777 | return 1; 778 | } 779 | 780 | return 0; 781 | } 782 | 783 | static int condition( 784 | fd_set *rset, int max_descr, 785 | int *ret, const char **task) 786 | { 787 | *ret = -777; 788 | *task = NULL; 789 | 790 | if (busy_count < alive_nodes_count && !end_of_stdin && 791 | (*task = tasks__get_new_task()) != NULL) 792 | { 793 | return 1; 794 | } 795 | 796 | if (!task && !graph_mode && feof(stdin)){ 797 | end_of_stdin = 1; 798 | close_all_ins(); 799 | } 800 | 801 | if (busy_count > 0 802 | && (*ret = unblock_select_block( 803 | max_descr+1, rset, NULL, NULL, NULL)) != 0) 804 | { 805 | return 1; 806 | } 807 | 808 | if (failed_taskids_count > 0 && wait_mode){ 809 | wait_for_sigalrm(); 810 | try_to_reinit_failed_nodes(); 811 | return 1; 812 | } 813 | 814 | return 0; 815 | } 816 | 817 | static void loop(void) 818 | { 819 | char msg [2000]; 820 | fd_set rset; 821 | 822 | int printed = 0; 823 | 824 | int ret = 0; 825 | int cnt = 0; 826 | int i, j; 827 | char *buf_out_i = 0; 828 | const char *task = NULL; 829 | const char *curr_line = NULL; 830 | 831 | if (graph_mode && tasks_count == 1){ 832 | /* no tasks */ 833 | close_all_ins(); 834 | wait_for_childs(); 835 | return; 836 | } 837 | 838 | FD_ZERO(&rset); 839 | 840 | FD_CLR(0, &rset); 841 | 842 | while (condition(&rset, max_fd, &ret, &task)){ 843 | /* ret == -777 means select(2) was not called */ 844 | 845 | if (try_to_reinit_failed_nodes()) 846 | continue; 847 | 848 | if (ret == -1 && errno == EINTR){ 849 | continue; 850 | } 851 | 852 | if (debug){ 853 | printf("select ret=%d\n", ret); 854 | } 855 | 856 | if (ret == -777 && task) 857 | send_to_node(); 858 | 859 | /* fd_out */ 860 | for (i=0; ret != -777 && i < nodes_count; ++i){ 861 | if (fd_out [i] >= 0 && FD_ISSET(fd_out [i], &rset)){ 862 | buf_out_i = buf_out [i]; 863 | 864 | assert(bufsize_out [i] > size_out [i]); 865 | 866 | cnt = read(fd_out [i], 867 | buf_out_i + size_out [i], 868 | bufsize_out [i] - size_out [i]); 869 | 870 | if (debug && cnt >= 0){ 871 | buf_out_i [size_out [i] + cnt] = 0; 872 | printf("cnt = %d\n", cnt); 873 | printf("fd_out [%d] = %d\n", i, fd_out [i]); 874 | printf("buf_out [%d] = %s\n", i, buf_out_i); 875 | printf("size_out [%d] = %d\n", i, (int) size_out [i]); 876 | } 877 | 878 | if (cnt == -1 || cnt == 0){ 879 | /* read error or unexpected end of file */ 880 | if (resistant){ 881 | FD_CLR(fd_out [i], &rset); 882 | mark_node_as_dead(i); 883 | if (msg_fatal [0]) 884 | print_line(i, msg_fatal); 885 | print_EOT(i); 886 | 887 | if (alive_nodes_count == 0 && !wait_mode){ 888 | err__fatal(NULL, "all nodes failed"); 889 | } 890 | continue; 891 | }else{ 892 | if (cnt == 0){ 893 | snprintf( 894 | msg, sizeof(msg), 895 | "Node %s exited unexpectedly", 896 | nodes [i]); 897 | }else{ 898 | snprintf( 899 | msg, sizeof(msg), 900 | "reading from node %s failed: %s", 901 | nodes [i], strerror(errno)); 902 | } 903 | 904 | err__fatal(NULL, msg); 905 | } 906 | } 907 | 908 | printed = 0; 909 | cnt += size_out [i]; 910 | for (j=size_out [i]; j < cnt; ++j){ 911 | if (buf_out_i [j] == '\n'){ 912 | buf_out_i [j] = 0; 913 | 914 | curr_line = buf_out_i + printed; 915 | 916 | if (!strcmp(curr_line, msg_eot)){ 917 | /* end of task marker */ 918 | assert(busy [i] == 1); 919 | 920 | busy [i] = 0; 921 | --busy_count; 922 | 923 | if (end_of_stdin){ 924 | close(fd_in [i]); 925 | fd_in [i] = -1; 926 | } 927 | 928 | /* EOT line means end-of-task */ 929 | if (graph_mode){ 930 | switch (ret_codes [i]){ 931 | case rt_failure: 932 | print_header(i); 933 | tasks__delete_task_rec(node2taskid [i]); 934 | printf("\n"); 935 | break; 936 | case rt_success: 937 | tasks__delete_task(node2taskid [i], 0, 0); 938 | break; 939 | case rt_undef: 940 | print_line(i, "?"); 941 | break; 942 | default: 943 | abort(); 944 | } 945 | 946 | end_of_stdin = (remained_tasks_count == 0); 947 | if (end_of_stdin) 948 | close_all_ins(); 949 | }else{ 950 | tasks__delete_task(node2taskid [i], 0, 0); 951 | } 952 | 953 | print_EOT(i); 954 | break; 955 | } 956 | 957 | if (graph_mode){ 958 | if (!strcmp(curr_line, msg_success)){ 959 | ret_codes [i] = rt_success; 960 | }else if (!strcmp(curr_line, msg_failure)){ 961 | ret_codes [i] = rt_failure; 962 | }else{ 963 | ret_codes [i] = rt_undef; 964 | } 965 | } 966 | 967 | print_line(i, curr_line); 968 | 969 | printed = j + 1; 970 | } 971 | } 972 | 973 | if (printed){ 974 | cnt -= printed; 975 | 976 | memmove(buf_out_i, 977 | buf_out_i + printed, 978 | cnt); 979 | } 980 | 981 | size_out [i] = cnt; 982 | 983 | if (size_out [i] == bufsize_out [i]){ 984 | bufsize_out [i] *= 2; 985 | 986 | /* +1 for \0 and -d option */ 987 | buf_out [i] = xrealloc(buf_out [i], bufsize_out [i]+1); 988 | } 989 | } 990 | } 991 | 992 | /* fd_out */ 993 | for (i=0; i < nodes_count; ++i){ 994 | if (fd_out [i] < 0) 995 | continue; 996 | 997 | if (busy [i]){ 998 | FD_SET(fd_out [i], &rset); 999 | }else{ 1000 | FD_CLR(fd_out [i], &rset); 1001 | } 1002 | } 1003 | 1004 | if (debug){ 1005 | printf("alive_nodes_count = %d\n", alive_nodes_count); 1006 | printf("busy_count = %d\n", busy_count); 1007 | printf("end_of_stdin = %d\n", end_of_stdin); 1008 | for (i=0; i < nodes_count; ++i){ 1009 | printf("busy [%d]=%d\n", i, busy [i]); 1010 | printf("pid [%d]=%d\n", i, (int) pids [i]); 1011 | } 1012 | } 1013 | 1014 | /* exit ? */ 1015 | if (graph_mode) 1016 | end_of_stdin = (remained_tasks_count == 0); 1017 | 1018 | if (!busy_count && end_of_stdin) 1019 | break; 1020 | } 1021 | 1022 | close_all_ins(); 1023 | } 1024 | 1025 | static void check_msg(const char *msg) 1026 | { 1027 | if (strpbrk(msg, "'\"")){ 1028 | err__fatal(NULL, "paexec: symbols ' and \" are not allowed in -m argument"); 1029 | } 1030 | } 1031 | 1032 | static char *gen_cmd(int *argc, char ***argv) 1033 | { 1034 | char cmd [4096]; 1035 | char *curr_token; 1036 | size_t len; 1037 | size_t curr_len; 1038 | int i; 1039 | 1040 | len = 0; 1041 | for (i=0; i < *argc; ++i){ 1042 | curr_token =(*argv) [i]; 1043 | if (replstr[0] && !strcmp(curr_token, replstr)){ 1044 | cmd[len+0] = '"'; 1045 | cmd[len+1] = '$'; 1046 | cmd[len+2] = '1'; 1047 | cmd[len+3] = '"'; 1048 | curr_len = 4; 1049 | }else{ 1050 | curr_len = shquote(curr_token, cmd+len, sizeof(cmd)-len-1); 1051 | } 1052 | if (curr_len == (size_t)-1){ 1053 | err__fatal(__func__, "paexec: Internal error4! (buffer size)"); 1054 | } 1055 | len += curr_len; 1056 | cmd [len++] = ' '; 1057 | 1058 | if (len >= sizeof(cmd)-20){ /* 20 chars is enough for "$1" :-) */ 1059 | err__fatal(__func__, "paexec: Internal error5! (buffer size)"); 1060 | } 1061 | } 1062 | 1063 | cmd [len++] = 0; 1064 | 1065 | assert(!arg_cmd); 1066 | return xstrdup(cmd); 1067 | } 1068 | 1069 | static void process_args(int *argc, char ***argv) 1070 | { 1071 | int c; 1072 | int mode_C = 0; 1073 | 1074 | /* leading + is for shitty GNU libc */ 1075 | static const char optstring [] = "+0c:CdeEghiIJ:lm:n:prst:VwW:xXyzZ:"; 1076 | 1077 | while (c = getopt(*argc, *argv, optstring), c != EOF){ 1078 | switch (c) { 1079 | case 'V': 1080 | printf("paexec %s written by Aleksey Cheusov\n", PAEXEC_VERSION); 1081 | exit(0); 1082 | break; 1083 | case 'h': 1084 | usage(); 1085 | exit(0); 1086 | break; 1087 | case 'd': 1088 | debug = 1; 1089 | break; 1090 | case 'n': 1091 | optarg += strspn(optarg, " \t"); 1092 | assign_str(&arg_nodes, optarg); 1093 | break; 1094 | case 'c': 1095 | mode_C = 0; 1096 | arg_cmd = xstrdup(optarg); 1097 | break; 1098 | case 'C': 1099 | mode_C = 1; 1100 | break; 1101 | case 't': 1102 | optarg += strspn(optarg, " \t"); 1103 | if (optarg [0]) 1104 | assign_str(&arg_transport, optarg); 1105 | break; 1106 | case 'p': 1107 | show_pid = 1; 1108 | break; 1109 | case 'l': 1110 | show_taskid = 1; 1111 | break; 1112 | case 'r': 1113 | show_node = 1; 1114 | break; 1115 | case 'e': 1116 | print_eot = 1; 1117 | break; 1118 | case 'E': 1119 | print_eot = 1; 1120 | flush_eot = 1; 1121 | break; 1122 | case 'i': 1123 | print_i2o = 1; 1124 | break; 1125 | case 'I': 1126 | print_i2o = 1; 1127 | flush_i2o = 1; 1128 | break; 1129 | case 'J': 1130 | if (strlen(optarg) != 2){ 1131 | fprintf(stderr, "-Jreplstr argument must have 2-characters length"); 1132 | exit(2); 1133 | } 1134 | 1135 | replstr[0] = optarg[0]; 1136 | replstr[1] = optarg[1]; 1137 | 1138 | if (!msg_eot) 1139 | msg_eot = magic_eot; 1140 | if (!exec_mode) 1141 | exec_mode = 'x'; 1142 | break; 1143 | case 's': 1144 | case 'g': 1145 | graph_mode = 1; 1146 | break; 1147 | case 'w': 1148 | wait_mode = 1; 1149 | break; 1150 | case 'z': 1151 | resistant = 1; 1152 | break; 1153 | case 'Z': 1154 | resistant = 1; 1155 | resistance_timeout = atoi(optarg); 1156 | break; 1157 | case 'm': 1158 | if (optarg [0] == 's' && optarg [1] == '='){ 1159 | msg_success = xstrdup(optarg+2); 1160 | check_msg(msg_success); 1161 | }else if (optarg [0] == 'f' && optarg [1] == '='){ 1162 | msg_failure = xstrdup(optarg+2); 1163 | check_msg(msg_failure); 1164 | }else if (optarg [0] == 'F' && optarg [1] == '='){ 1165 | msg_fatal = xstrdup(optarg+2); 1166 | check_msg(msg_fatal); 1167 | }else if (optarg [0] == 't' && optarg [1] == '='){ 1168 | msg_eot = xstrdup(optarg+2); 1169 | }else if (optarg [0] == 'd' && optarg [1] == '='){ 1170 | if (optarg [2] != 0 && optarg [3] != 0){ 1171 | err__fatal(NULL, "paexec: bad argument for -md=. At most one character is allowed"); 1172 | } 1173 | msg_delim = optarg [2]; 1174 | }else if (optarg [0] == 'w' && optarg [1] == '='){ 1175 | msg_weight = xstrdup(optarg+2); 1176 | }else{ 1177 | err__fatal(NULL, "paexec: bad argument for -m"); 1178 | } 1179 | 1180 | break; 1181 | case 'W': 1182 | use_weights = atoi(optarg); 1183 | graph_mode = 1; 1184 | break; 1185 | case 'x': 1186 | exec_mode = 'x'; 1187 | msg_eot = magic_eot; 1188 | break; 1189 | case 'X': 1190 | exec_mode = 'X'; 1191 | msg_eot = magic_eot; 1192 | break; 1193 | case 'y': 1194 | msg_eot = magic_eot; 1195 | break; 1196 | case '0': 1197 | eol_char = '\0'; 1198 | break; 1199 | default: 1200 | usage(); 1201 | exit(1); 1202 | } 1203 | } 1204 | 1205 | *argv += optind; 1206 | *argc -= optind; 1207 | 1208 | if (!msg_eot){ 1209 | msg_eot = ""; 1210 | } 1211 | 1212 | if (mode_C){ 1213 | if (!*argc){ 1214 | err__fatal(NULL, "paexec: missing arguments. Run paexec -h for details"); 1215 | } 1216 | 1217 | arg_cmd = gen_cmd(argc, argv); 1218 | }else{ 1219 | if (*argc){ 1220 | err__fatal(NULL, "paexec: extra arguments. Run paexec -h for details"); 1221 | } 1222 | } 1223 | 1224 | if (!resistance_timeout && wait_mode){ 1225 | err__fatal(NULL, "paexec: -w is useless without -Z"); 1226 | } 1227 | 1228 | if (arg_nodes){ 1229 | if (arg_nodes [0] == '+'){ 1230 | free(arg_transport); 1231 | arg_transport = NULL; 1232 | } 1233 | 1234 | nodes_create(arg_nodes); 1235 | }else{ 1236 | err__fatal(NULL, "paexec: -n option is mandatory!"); 1237 | } 1238 | 1239 | if (!arg_cmd){ 1240 | err__fatal(NULL, "paexec: -c option is mandatory!"); 1241 | } 1242 | 1243 | if (use_weights < 0 || use_weights > 2){ 1244 | err__fatal(NULL, "paexec: Only -W1 and -W2 are supported!"); 1245 | } 1246 | 1247 | if (arg_transport && !arg_transport [0]){ 1248 | free(arg_transport); 1249 | arg_transport = NULL; 1250 | } 1251 | } 1252 | 1253 | static void free_memory(void) 1254 | { 1255 | int i; 1256 | 1257 | if (arg_nodes) 1258 | xfree(arg_nodes); 1259 | 1260 | if (arg_transport) 1261 | xfree(arg_transport); 1262 | 1263 | if (arg_cmd) 1264 | xfree(arg_cmd); 1265 | 1266 | nodes_destroy(); 1267 | 1268 | if (fd_in) 1269 | xfree(fd_in); 1270 | if (fd_out) 1271 | xfree(fd_out); 1272 | 1273 | if (buf_stdin) 1274 | xfree(buf_stdin); 1275 | 1276 | if (buf_out){ 1277 | for (i=0; i < nodes_count; ++i){ 1278 | if (buf_out [i]) 1279 | xfree(buf_out [i]); 1280 | } 1281 | xfree(buf_out); 1282 | } 1283 | 1284 | if (node2task){ 1285 | for (i=0; i < nodes_count; ++i){ 1286 | if (node2task [i]) 1287 | xfree(node2task [i]); 1288 | } 1289 | xfree(node2task); 1290 | } 1291 | if (node2task_buf_sz) 1292 | xfree(node2task_buf_sz); 1293 | 1294 | if (size_out) 1295 | xfree(size_out); 1296 | 1297 | if (busy) 1298 | xfree(busy); 1299 | 1300 | if (pids) 1301 | xfree(pids); 1302 | 1303 | if (node2taskid) 1304 | xfree(node2taskid); 1305 | 1306 | if (ret_codes) 1307 | xfree(ret_codes); 1308 | 1309 | tasks__destroy(); 1310 | } 1311 | 1312 | static void init_env(void) 1313 | { 1314 | char *tok; 1315 | char *env_msg_eot = getenv("PAEXEC_EOT"); 1316 | if (env_msg_eot) 1317 | msg_eot = env_msg_eot; 1318 | 1319 | char *env_bufsize = getenv("PAEXEC_BUFSIZE"); 1320 | if (env_bufsize) 1321 | initial_bufsize = atoi(env_bufsize); 1322 | 1323 | char *env_transport = getenv("PAEXEC_TRANSPORT"); 1324 | if (env_transport) 1325 | assign_str(&arg_transport, env_transport); 1326 | 1327 | char *env_nodes = getenv("PAEXEC_NODES"); 1328 | if (env_nodes) 1329 | assign_str(&arg_nodes, env_nodes); 1330 | 1331 | char *env_shell = getenv("PAEXEC_SH"); 1332 | if (env_shell) 1333 | shell = env_shell; 1334 | else 1335 | shell = "/bin/sh"; 1336 | 1337 | char *paexec_env = getenv("PAEXEC_ENV"); 1338 | if (paexec_env){ 1339 | for (tok = strtok(paexec_env, " ,"); 1340 | tok; 1341 | tok = strtok(NULL, " ,")) 1342 | { 1343 | add_envvar(tok, getenv(tok)); 1344 | } 1345 | } 1346 | } 1347 | 1348 | int main(int argc, char **argv) 1349 | { 1350 | int i; 1351 | pid_t pid; 1352 | int status; 1353 | 1354 | block_signals(); 1355 | 1356 | init_env(); 1357 | 1358 | process_args(&argc, &argv); 1359 | 1360 | if (debug){ 1361 | printf("nodes_count = %d\n", nodes_count); 1362 | for (i=0; i < nodes_count; ++i){ 1363 | printf("nodes [%d]=%s\n", i, nodes [i]); 1364 | } 1365 | printf("cmd = %s\n", arg_cmd); 1366 | } 1367 | 1368 | init(); 1369 | 1370 | loop(); 1371 | 1372 | free_memory(); 1373 | 1374 | set_sig_handler(SIGCHLD, SIG_DFL); 1375 | set_sig_handler(SIGALRM, SIG_IGN); 1376 | 1377 | unblock_signals(); 1378 | 1379 | while (pid = waitpid(-1, &status, WNOHANG), pid > 0){ 1380 | } 1381 | 1382 | return 0; 1383 | } 1384 | -------------------------------------------------------------------------------- /paexec/paexec.pod: -------------------------------------------------------------------------------- 1 | =head1 NAME 2 | 3 | paexec - parallel executor, distribute tasks over network or CPUs 4 | 5 | =head1 SYNOPSIS 6 | 7 | B I<[options]> 8 | 9 | B -C I<[options]> I I<[args...]> 10 | 11 | =head1 DESCRIPTION 12 | 13 | Suppose you have a long list of tasks that need to be 14 | done, for example, you want to convert thousands of .wav audio files 15 | to .ogg format. Also suppose that multiple CPUs are 16 | available, e.g. multi-CPU system or a 17 | cluster consisting of individual computers connected to the network or 18 | internet. B can efficiently do this work, that 19 | is, B efficiently distributes different tasks to different 20 | processors (or computers), receives the results of processing from them 21 | and sends these results to stdout. 22 | 23 | There are several notions that should be defined: I, I, 24 | I, I. 25 | 26 | I are read by B from stdin and are represented as one 27 | line of text, i.e. one input line - one task. 28 | 29 | I identifier - remote computer or CPU identifier, for 30 | example CPU ordinal number or computer's DNS name like 31 | node12.cluster.company.com. 32 | 33 | I - user's program that reads one-line task from stdin and sends 34 | multiline result to stdout ending with an end of task (EOT) marker. 35 | EOT marker means 36 | "Task is done. I am ready for the next one". After sending EOT 37 | to stdout, stdout MUST be flushed. Remember that EOT marker 38 | MUST NOT appears in general result. Otherwise, B may hang 39 | due to deadlock. The default EOT is an empty line. Command may use 40 | environment variable PAEXEC_EOT for the end-of-task marker. 41 | 42 | I - special program that helps to run I on 43 | I. It takes the I identifier as its first 44 | argument and I with its arguments as the rest. 45 | Good examples for transport are I and I. 46 | Both I and I may 47 | be specified with their arguments, e.g, '/usr/bin/ssh -x' is allowed 48 | as a I program. 49 | 50 | How B works. 51 | I are run on each I with a help of 52 | I program. Then, I are read from stdin one-by-one 53 | and sent to free I (exactly one task per node at 54 | a time). At the same time result lines are read from I 55 | stdout and are output to B stdout. When EOT marker is 56 | obtained from the I, it is marked as free and becomes ready for the next 57 | task. These steps repeat until the end of stdin is reached and all 58 | I finish their job. 59 | 60 | More formally (to better understand how paexec works): 61 | 62 | run_command_on_each_node 63 | mark_all_nodes_as_free 64 | while not(end_of_stdin) or not(all_nodes_are_free) 65 | while there_is_free_node/i and not(end_of_stdin) 66 | task = read_task_from_stdin 67 | send_task_to_node(task, i) 68 | mark_node_as_busy(i) 69 | end 70 | while result_line_from_node_is_available/i 71 | result = read_result_line_from_node(i) 72 | send_line_to_stdout(result) 73 | if is_EOT(result) 74 | mark_node_as_free(i) 75 | end 76 | end 77 | end 78 | close_command_on_each_node 79 | 80 | Note that Is are run once per node, 81 | it is not restarted for every task. 82 | 83 | Also note that output contains result lines (obtained from different 84 | I) in the mixed order. That is, the first line of the 85 | output may contain a result line obtain from the first I, 86 | the second line of output - from the second I, but the 87 | third output line may contain result line from the first I 88 | again. It is also not guaranteed that the first line of output will be 89 | from the first I or from the first I. All result 90 | lines are output as soon as they are read by B, i.e as soon as 91 | they are ready. B works this way for the 92 | efficiency reasons. You can play with I<-l>, I<-r> and I<-p> options to see 93 | what happens. For reordering output line you can use B utility. 94 | 95 | =head1 OPTIONS 96 | 97 | =over 6 98 | 99 | =item B<-h> 100 | 101 | Display help information. 102 | 103 | =item B<-V> 104 | 105 | Display version information. 106 | 107 | =item B<-c> I 108 | 109 | Command with its arguments. 110 | 111 | =item B<-C> 112 | 113 | Command and its arguments are specified by free arguments. 114 | 115 | =item B<-t> I 116 | 117 | Transport program. 118 | 119 | =item B<-n> I<+number> 120 | 121 | A number of commands to run in parallel. 122 | 123 | =item B<-n> I 124 | 125 | List of nodes separated by space or tab characters. The first character must be 126 | alphanumeric, `_' or `/'. All other characters are reserved 127 | for future extensions. Leading and trailing spaces and tabs are ignored. 128 | 129 | =item B<-n> I<:filename> 130 | 131 | Filename containing list of nodes, one per line. 132 | 133 | =item B<-x> 134 | 135 | Run command specificed by I<-c> for each task. Its stdout is sent to B. 136 | If both C<-x> and C<-g> are specified, task is considered failed 137 | if command's exit status is non-zero. 138 | 139 | =item B<-X> 140 | 141 | Implies I<-x> and ignore command's stdout. 142 | 143 | =item B<-r> 144 | 145 | Include node identifier or node number (0-based) to the 146 | output, i.e. id/number of node that produces this particular 147 | output line. This identifier or number appears before line number if 148 | I<-l> is also applied. Space character is used as a separator. 149 | 150 | =item B<-l> 151 | 152 | Include a 0-based task number (input line number) to the output, 153 | i.e. line number from 154 | which this particular output line was produced. It appears before pid 155 | if I<-p> is also applied. Space character is used as a separator. 156 | 157 | =item B<-p> 158 | 159 | Include pid of paexec's subprocess that communicates with 160 | I to the output. Pid prepends the actual result 161 | line. Space character is used as a separator. 162 | 163 | =item B<-e> 164 | 165 | When end-of-task marker is obtained from node, EOT marker is 166 | output to stdout. This option may be useful together with I<-l> 167 | and/or I<-r>. 168 | 169 | =item B<-E> 170 | 171 | Imply B<-e> and flushes stdout. 172 | 173 | =item B<-d> 174 | 175 | Turn on a debugging mode (for debugging purposes only). 176 | 177 | =item B<-i> 178 | 179 | Copy input lines (i.e. tasks) to stdout. 180 | 181 | =item B<-I> 182 | 183 | Imply B<-i> and flushes stdout. 184 | 185 | =item B<-s>|B<-g> 186 | 187 | Orgraph of tasks (partially ordered set) is read from stdin. 188 | 189 | Instead of autonomous tasks, graph of the tasks is read from stdin. 190 | In this mode every task can either FAIL or SUCCEED. 191 | As always an empty line output by I means I. 192 | The line before it shows an EXIT STATUS of the task. 193 | The word "failure" means failure, "success" - success and 194 | "fatal" means that the current task is reassigned to another node 195 | (and restarted, of course) (see option -z). 196 | See examples/1_div_x/cmd for the sample. 197 | An input line (paexec's stdin) should contain 198 | either single task without spaces inside 199 | or two tasks separated by single space character, 200 | e.g. task1task2. task1task2 line means that task1 must be done 201 | before task2 and it is mandatory, that is if task1 I all dependent 202 | tasks (including task2) are also failed recursively. 203 | Tasks having dependencies are started only after all dependencies 204 | are succeeded. When a task succeeds paexec outputs "success" word 205 | just before end_of_task marker (see -e or -E), otherwise "failure" 206 | word is output followed by a list of tasks failed because of it. 207 | 208 | Samples: 209 | 210 | tasks (examples/make_package/tasks file) 211 | 212 | textproc/dictem 213 | devel/autoconf wip/libmaa 214 | devel/gmake wip/libmaa 215 | wip/libmaa wip/dict-server 216 | wip/libmaa wip/dict-client 217 | devel/m4 wip/dict-server 218 | devel/byacc wip/dict-server 219 | devel/byacc wip/dict-client 220 | devel/flex wip/dict-server 221 | devel/flex wip/dict-client 222 | devel/glib2 223 | devel/libjudy 224 | 225 | command (examples/make_package/cmd__flex) 226 | 227 | #!/usr/bin/awk -f 228 | { 229 | print $0 230 | if ($0 == "devel/flex") 231 | print "failure" 232 | else 233 | print "success" 234 | 235 | print "" # end of task marker 236 | fflush() 237 | } 238 | 239 | output of "paexec -s -l -c cmd__flex -n +10 \ 240 | < tasks" 241 | 242 | 3 devel/autoconf 243 | 3 success 244 | 4 devel/gmake 245 | 4 success 246 | 7 devel/m4 247 | 7 success 248 | 8 devel/byacc 249 | 8 success 250 | 9 devel/flex 251 | 9 failure 252 | 9 devel/flex wip/dict-server wip/dict-client 253 | 10 devel/glib2 254 | 10 success 255 | 11 devel/libjudy 256 | 11 success 257 | 1 textproc/dictem 258 | 1 success 259 | 2 wip/libmaa 260 | 2 success 261 | 262 | =item B<-z> 263 | 264 | If applied, read/write(2) operations from/to nodes becomes not 265 | critical. In case paexec has lost connection to the node, it will 266 | reassign failed task to another node and, if -s applied, will output 267 | "fatal" string to stdout ("success" + "failure" + "fatal"). This 268 | makes paexec resistant to the I/O errors, as a result you can create 269 | paexec clusters even over network consisting of unreliable hosts 270 | (Internet?). Failed hosts are marked as such and will not be used 271 | during the current run of paexec. 272 | 273 | =item B<-Z> I 274 | 275 | When I<-z> applied, if a I fails, appropriate node is marked 276 | as broken and is excluded from the following task distribution. But if 277 | B<-Z> applied, every I seconds an attempt to rerun a comand 278 | on a failed node is made. I<-Z> implies I<-z>. This option makes 279 | possible to organize clusters over unreliable networks/hardware. 280 | 281 | =item B<-w> 282 | 283 | If I<-Z> option were applied, B exits with error 284 | if B nodes failed. With B<-w> it will not exit 285 | and will wait for restoring nodes. 286 | 287 | =item B<-m> s=I 288 | 289 | Set an alternative string for 'success' message. 290 | 291 | =item B<-m> f=I 292 | 293 | Set an alternative string for 'failure' message. 294 | 295 | =item B<-m> F=I 296 | 297 | Set an alternative string for 'fatal' message. 298 | An empty string for 'fatal' 299 | means it will not be output to stdout in case of fatal error. 300 | 301 | =item B<-m> t=I 302 | 303 | Set an alternative string for EOT message. 304 | 305 | =item B<-m> d=I 306 | 307 | Set an alternative string for tasks delimiter (for I<-g> mode). 308 | Delimiter should be at most one character. No delimiter means 309 | an entire input line is treated as a task. 310 | 311 | =item B<-m> w=I 312 | 313 | Set an alternative string for "weight:" message. 314 | 315 | =item B<-W> I 316 | 317 | When multiple machines or CPUs are used for tasks processing, it makes sense 318 | to start "heavier" tasks as soon as possible in order to minimize total 319 | calculatiion time. 320 | If B<-W> is specified, special weight is assigned to each tasks 321 | which is used for reordering tasks. 322 | If I is 0, weights themselves are used for reordering tasks. 323 | The bigger weight is, the more priority of the task is. 324 | If I is 1, the total weight of task is a sum of its own weight 325 | (specified on input) 326 | and weights of all tasks depending on it directly or indirectly. 327 | If I is 2, the total weight of task is a maximum value of task's own 328 | weight 329 | and weights of all tasks depending on it directly or indirectly. 330 | Weights are specified with a help of "weight:" keyword (unless I<-m w=> was specified). 331 | If 332 | weight is not specified, it defaults to 1. The following is the example 333 | for input graph of tasks with weights. 334 | 335 | weight: gtk2 30 336 | weight: glib2 20 337 | gtk2 firefox 338 | weight: firefox 200 339 | glib2 gtk2 340 | weight: qt4 200 341 | weight: kcachegrind 2 342 | qt4 kcachegrind 343 | qt4 djview4 344 | tiff djview4 345 | png djview4 346 | weight: twm 1 347 | weight: gqview 4 348 | 349 | =item B<-y> 350 | 351 | If applied, the magic string is used as an end-of-task marker instead of 352 | empty line. It is unlikely that this line appears on I's 353 | output. This option has higher priority than PAEXEC_EOT environment 354 | variable. 355 | 356 | =item B<-0> 357 | 358 | Change paexec to expect NUL character as a line separator 359 | instead of newline. This is expected to be used in 360 | concert with the -print0 function in find(1). 361 | 362 | =item B<-J> I 363 | 364 | Execute I for each I, replacing one or more occurrences 365 | of I with the entire I. Only 2-character I is 366 | allowed. Tale a note that such replacement works only if shell 367 | variable expansion is allowed in appropriate part of command (if B<-c> is applied) 368 | or if free argument is exactly a I (if B<-C> is applied). 369 | This option implies -x. 370 | 371 | =back 372 | 373 | =head1 EXAMPLES 374 | 375 | =over 6 376 | 377 | =item 1 378 | 379 | paexec -t '/usr/bin/ssh -x' -n 'host1 host2 host3' \ 380 | -le -g -c calculate-me < tasks.txt | 381 | paexec_reorder -Mf -Sl 382 | 383 | =item 2 384 | 385 | ls -1 *.wav | paexec -x -n +4 -c 'oggenc -Q' 386 | 387 | =item 3 388 | 389 | ls -1 *.wav | paexec -xCil -n+4 flac -f --silent 390 | 391 | =item 4 392 | 393 | { uname -s; uname -r; uname -m; } | 394 | paexec -x -lp -n+2 -c banner | 395 | paexec_reorder -l 396 | 397 | =item 5 398 | 399 | find . -name '*.dat' -print0 | 400 | paexec -0 -n+10 -C -J// scp // remoteserver:/remote/path 401 | 402 | =item 6 403 | 404 | ls -1 *.txt | paexec -n+10 -J%% -c 'awk "BEGIN {print toupper(\"%%\")}"' 405 | 406 | =back 407 | 408 | For more examples see paexec.pdf and examples/ subdirectory in the distribution. 409 | 410 | =head1 NOTES 411 | 412 | select(2) system call and non-blocking read(2) are used to read result 413 | lines from I. 414 | 415 | At the moment blocking write(2) is used to send I to the 416 | I. This may slow down an entire processing if I are 417 | too big. So, it is recommended to use shorter I, for example, 418 | filename or URI (several tens of bytes in size) instead of 419 | multi-megabyte content. Though this may be fixed in the future. 420 | 421 | Original paexec tarball contains a number of sample of use in 422 | presentation/paexec.pdf file. After installation you can find this 423 | file under share/doc/paexec/paexec.pdf or nearby. 424 | 425 | =head1 ENVIRONMENT 426 | 427 | =over 6 428 | 429 | =item I 430 | 431 | Overrides the compile time I size for internal buffers used 432 | to store tasks and the result lines. Versions of B prior to 433 | 0.9.0 used this value as a I buffer size. 434 | Now internal buffers are resized automatically. 435 | If unsure, do not set PAEXEC_BUFSIZE variable. 436 | See the default value in Makefile. 437 | 438 | =item I 439 | 440 | A list of variables passed to I. 441 | 442 | =item I 443 | 444 | This variable sets the end-of-task marker which is an empty line by default. 445 | Also, through this variable an end-of-task marker is passed to all Is. 446 | 447 | =item I 448 | 449 | Unless option B<-n> was applied, this variables specifies the nodes. 450 | 451 | =item I 452 | 453 | This variable sets the shell interpreter used inside B. 454 | By default it is /bin/sh. 455 | 456 | =item I 457 | 458 | Unless option B<-t> was applied, this variables specifies the transport. 459 | 460 | =back 461 | 462 | =head1 BUGS/FEEDBACK 463 | 464 | Please send any comments, questions, bug reports etc. to me by e-mail 465 | or (even better) register them at sourceforge project home. Feature 466 | requests are also welcomed. 467 | 468 | =head1 HOME 469 | 470 | L 471 | 472 | =head1 SEE ALSO 473 | L 474 | L 475 | L 476 | L 477 | L 478 | -------------------------------------------------------------------------------- /paexec/paexec_reorder: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env runawk 2 | 3 | # Copyright (c) 2009-2013 Aleksey Cheusov 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining 6 | # a copy of this software and associated documentation files (the 7 | # "Software"), to deal in the Software without restriction, including 8 | # without limitation the rights to use, copy, modify, merge, publish, 9 | # distribute, sublicense, and/or sell copies of the Software, and to 10 | # permit persons to whom the Software is furnished to do so, subject to 11 | # the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be 14 | # included in all copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | #env "LC_ALL=C" 25 | 26 | #use "alt_assert.awk" 27 | #use "init_getopt.awk" 28 | #use "tmpfile.awk" 29 | #use "xsystem.awk" 30 | 31 | ############################################################ 32 | #.begin-str help 33 | # paexec_reorder - takes output of 'paexec -le' or 'paexec -gle' on 34 | # input and outputs ordered results, that is without "slicing". 35 | # usage: paexec_reorder [OPTIONS] 36 | # OPTIONS: 37 | # -h display this help 38 | # -g expects output of "paexec -gle", 39 | # by default -- "paexec -le". 40 | # -S remove leading space 41 | # -l prepand output line with the task number 42 | # given on input 43 | # -x synonym for -y 44 | # -y output of "paexec -le [g] -y" is expected on input 45 | # =M method of resorting result line, where method is 46 | # m in-memory sort (the default) 47 | # s use sort(1) command 48 | # f use multiple temporary files 49 | # =m s= set alternative string for 'success', 'failure', 50 | # f= 'fatal' and '' (end of task). 51 | # F= 52 | # t= 53 | # 54 | # In -g -Mm and -g -Mf modes, portions of the result followed 55 | # by "fatal" marker are automatically cut off. 56 | #.end-str 57 | ############################################################ 58 | 59 | BEGIN { 60 | method = "m" 61 | 62 | msg_success = "success" 63 | msg_failure = "failure" 64 | msg_fatal = "fatal" 65 | msg_eot = ENVIRON ["PAEXEC_EOT"] 66 | 67 | while (getopt(short_opts)){ 68 | if (optopt == "h"){ 69 | print_help() 70 | exitnow(0) 71 | }else if (optopt == "g"){ 72 | graph_mode = 1 73 | }else if (optopt == "S"){ 74 | remove_spc = 1 75 | }else if (optopt == "l"){ 76 | output_task = 1 77 | }else if (optopt == "M"){ 78 | method = optarg 79 | }else if (optopt == "m"){ 80 | if (optarg ~ /^s=/){ 81 | msg_success = substr(optarg, 3) 82 | }else if (optarg ~ /^f=/){ 83 | msg_failure = substr(optarg, 3) 84 | }else if (optarg ~ /^F=/){ 85 | msg_fatal = substr(optarg, 3) 86 | }else if (optarg ~ /^t=/){ 87 | msg_eot = substr(optarg, 3) 88 | }else{ 89 | abort("bad argument for -m") 90 | } 91 | }else if (optopt == "y" || optopt == "x"){ 92 | msg_eot = "HG>&OSO@#;L8N;!&.U4ZC_9X:0AF,2Y>SRXAD_7U&QZ5S>N^?Y,I=W?@5" 93 | }else{ 94 | abort() 95 | } 96 | } 97 | 98 | bad_input_msg = "bad input, did you run paexec -sle?" 99 | 100 | # -Ms 101 | if (output_task) 102 | sort_cmd = "sort -k2n -k1n -t ' ' | cut -f 2- -d ' '" 103 | else 104 | sort_cmd = "sort -k2n -k1n -t ' ' | cut -f 3- -d ' '" 105 | } 106 | 107 | # -Mm 108 | function print_results (task, cnt,i){ 109 | if (graph_mode && !(task in pline)) 110 | return 111 | 112 | if (!graph_mode || pline [task] == msg_success || ppline [task] == msg_failure){ 113 | cnt = count [task] 114 | for (i=1; i <= cnt; ++i){ 115 | if (output_task) 116 | printf "%s ", task 117 | print line [task, i] 118 | delete line [task, i] 119 | } 120 | 121 | count [task] = 0 122 | }else if (pline [task] == msg_fatal){ 123 | count [task] = 0 124 | }else{ 125 | abort(bad_input_msg) 126 | } 127 | } 128 | 129 | # common code 130 | { 131 | assert(NF > 0 && $0 ~ /^[0-9]+ /, bad_input_msg) 132 | if (task > task_max) 133 | task_max = task 134 | } 135 | 136 | # -Ms 137 | method == "s" { 138 | if (! (match($0, /^[0-9]+ /) && substr($0, RSTART+RLENGTH) == msg_eot)){ 139 | if (remove_spc){ 140 | num = $1 141 | sub(/^[0-9]+ ?/, "", $0) 142 | print NR, num, $0 | sort_cmd 143 | }else{ 144 | print NR, $0 | sort_cmd 145 | } 146 | } 147 | 148 | next 149 | } 150 | 151 | # -Mf 152 | function print_results_file (task, cmd,fn){ 153 | if (task in count){ 154 | fn = runawk_tmpdir "/" task 155 | xclose(fn) 156 | cmd = sprintf("cat '%s' && rm '%s'", fn, fn) 157 | xsystem(cmd) 158 | delete count [task] 159 | } 160 | } 161 | 162 | method == "f" { 163 | if (match($0, /^[0-9]+ /) && substr($0, RSTART+RLENGTH) == msg_eot){ 164 | if (!graph_mode || pline [$1] != msg_fatal){ 165 | print_results_file(task) 166 | }else{ 167 | fn = runawk_tmpdir "/" task 168 | xclose(fn) 169 | xsystem("rm '" fn "'") 170 | delete count [task] 171 | } 172 | }else{ 173 | if (graph_mode){ 174 | ppline [$1] = pline [$1] 175 | pline [$1] = $2 176 | } 177 | 178 | task = $1 179 | count [task] = 1 180 | 181 | if (remove_spc) 182 | sub(/^[0-9]+ ?/, "") 183 | else 184 | sub(/^[0-9]+ /, "") 185 | 186 | fn = runawk_tmpdir "/" task 187 | if (output_task) 188 | print task, $0 > fn 189 | else 190 | print $0 > fn 191 | } 192 | 193 | next 194 | } 195 | 196 | # -Mm 197 | match($0, /^[0-9]+ /) && substr($0, RSTART+RLENGTH) == msg_eot { 198 | print_results($1) 199 | next 200 | } 201 | 202 | { 203 | if (graph_mode){ 204 | ppline [$1] = pline [$1] 205 | pline [$1] = $2 206 | } 207 | 208 | task = $1 209 | sub(/^[0-9]+ /, "", $0) 210 | cnt = ++count [task] 211 | 212 | if (remove_spc) 213 | sub(/^ /, "", $0) 214 | line [task, cnt] = $0 215 | } 216 | 217 | END { 218 | # output of "paexec -l" (without -e) is also allowed, obviously 219 | # this requires more memory for reordering results, so applying 220 | # option -e is strongly recomended. 221 | if (method == "m"){ 222 | # -Mm 223 | for (i=1; i <= task_max; ++i){ 224 | print_results(i) 225 | } 226 | }else if (method == "f"){ 227 | # -Mf 228 | for (i=1; i <= task_max; ++i){ 229 | print_results_file(i) 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /paexec/paexec_reorder.pod: -------------------------------------------------------------------------------- 1 | =head1 NAME 2 | 3 | paexec_reorder - reorder sliced output of "paexec -l" 4 | 5 | =head1 SYNOPSIS 6 | 7 | B [I] [I] 8 | 9 | =head1 DESCRIPTION 10 | 11 | B with -l option produces a sliced output where results of 12 | different tasks are intermixed. The intent of B is to 13 | produce ordered output where results for all tasks follow each other 14 | without intermixing. It is strongly recomended to send output of 15 | "B I<-le>" or "B I<-gle>" to the input of B. Otherwise 16 | more memory or disk space for temporary files will be required. 17 | 18 | =head1 OPTIONS 19 | 20 | =over 6 21 | 22 | =item B<-h> 23 | 24 | Display help information. 25 | 26 | =item B<-M> I 27 | 28 | If I is I, result is reordered in memory, this is the default. 29 | If it is I, temporary files are used for reordering. If I, B 30 | command is used. 31 | 32 | =item B<-l> 33 | 34 | Prepand output lines with the task number. 35 | 36 | =item B<-g> 37 | 38 | By default output of "B I<-le>" is expected on input. With I<-g> 39 | option, output of "B I<-gle>" is expected. In this case B 40 | will react on "fatal" B keyword. 41 | 42 | =item B<-x> 43 | 44 | If applied, output of "B -le [-g] I<-x>" is expected on input. 45 | This option has higher priority than PAEXEC_EOT environment variable. 46 | Actually B<-x> and B<-y> are synonyms. 47 | 48 | =item B<-y> 49 | 50 | If applied, output of "B -le [-g] I<-y>" is expected on input. 51 | This option has higher priority than PAEXEC_EOT environment variable. 52 | 53 | =item B<-S> 54 | 55 | Remove leading space character. 56 | 57 | =item B<-m> s=I 58 | 59 | =item B<-m> f=I 60 | 61 | =item B<-m> F=I 62 | 63 | =item B<-m> t=I 64 | 65 | Set alternative string 66 | for 'success', 'failure', 'fatal' and '' (end of task). 67 | 68 | =back 69 | 70 | =head1 EXAMPLES 71 | 72 | =over 6 73 | 74 | paexec -t '/usr/bin/ssh -x' -n 'host1 host2 host3' \ 75 | -l -c ~/bin/complex_task | paexec_reorder 76 | 77 | paexec -gEI -lr -n 'host1 host2 host3' \ 78 | -c command -t /usr/bin/rsh < tasks.txt | paexec_reorder -lgS -Mf 79 | 80 | For other examples, see examples/ directory 81 | 82 | =back 83 | 84 | =head1 ENVIRONMENT 85 | 86 | =over 6 87 | 88 | =item I 89 | 90 | I is used for creating temporary directory. 91 | See the appropriate man page. 92 | 93 | =item I 94 | 95 | The same as in B. 96 | 97 | =back 98 | 99 | =head1 BUGS/FEEDBACK 100 | 101 | Please send any comments, questions, bug reports etc. to me by e-mail 102 | or (even better) register them at sourceforge project home. Feature 103 | requests are also welcomed. 104 | 105 | =head1 HOME 106 | 107 | L 108 | 109 | =head1 SEE ALSO 110 | L 111 | L 112 | -------------------------------------------------------------------------------- /paexec/shquote.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | 26 | /* 27 | * Idea comes from NetBSD but implementations are not equal. 28 | * 29 | * http://netbsd.gw.com/cgi-bin/man-cgi?shquote+3+NetBSD-current 30 | * 31 | */ 32 | 33 | size_t 34 | shquote(const char *arg, char *buf, size_t bufsize); 35 | 36 | size_t 37 | shquote(const char *arg, char *buf, size_t bufsize) 38 | { 39 | int i; 40 | size_t len = 2; 41 | 42 | for (i=0; arg [i]; ++i){ 43 | ++len; 44 | if (arg [i] == '\'') 45 | len += 3; 46 | } 47 | 48 | if (!buf) 49 | return len; 50 | 51 | if (bufsize < len+1) 52 | return (size_t) -1; 53 | 54 | *buf++ = '\''; 55 | for (i=0; arg [i]; ++i){ 56 | if (arg [i] == '\''){ 57 | *buf++ = '\''; 58 | *buf++ = '\\'; 59 | *buf++ = '\''; 60 | *buf++ = '\''; 61 | }else{ 62 | *buf++ = arg [i]; 63 | } 64 | } 65 | *buf++ = '\''; 66 | *buf++ = '\0'; 67 | 68 | return len; 69 | } 70 | -------------------------------------------------------------------------------- /paexec/signals.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2011 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | 27 | #include "signals.h" 28 | #include "common.h" 29 | #include "wrappers.h" 30 | 31 | void block_signals(void) 32 | { 33 | sigset_t set; 34 | 35 | sigemptyset(&set); 36 | xsigaddset(&set, SIGALRM); 37 | xsigaddset(&set, SIGCHLD); 38 | 39 | xsigprocmask(SIG_BLOCK, &set, NULL); 40 | } 41 | 42 | void unblock_signals(void) 43 | { 44 | sigset_t set; 45 | 46 | sigemptyset(&set); 47 | xsigaddset(&set, SIGALRM); 48 | xsigaddset(&set, SIGCHLD); 49 | 50 | xsigprocmask(SIG_UNBLOCK, &set, NULL); 51 | } 52 | 53 | void set_sig_handler(int sig, void (*handler) (int)) 54 | { 55 | struct sigaction sa; 56 | 57 | sa.sa_handler = handler; 58 | sigemptyset(&sa.sa_mask); 59 | sa.sa_flags = 0; 60 | sigaction(sig, &sa, NULL); 61 | } 62 | 63 | void ignore_sigpipe(void) 64 | { 65 | set_sig_handler(SIGPIPE, SIG_IGN); 66 | } 67 | 68 | void wait_for_sigalrm(void) 69 | { 70 | sigset_t set; 71 | 72 | sigemptyset(&set); 73 | 74 | sigsuspend(&set); 75 | } 76 | 77 | void handler_sigchld(int dummy attr_unused) 78 | { 79 | int status; 80 | pid_t pid; 81 | 82 | while (pid = waitpid(-1, &status, WNOHANG), pid > 0){ 83 | } 84 | } 85 | 86 | void set_sigalrm_handler(void) 87 | { 88 | set_sig_handler(SIGALRM, handler_sigalrm); 89 | } 90 | 91 | void set_sigchld_handler(void) 92 | { 93 | set_sig_handler(SIGCHLD, handler_sigchld); 94 | } 95 | 96 | int sigalrm_tics = 0; 97 | void handler_sigalrm(int dummy attr_unused) 98 | { 99 | ++sigalrm_tics; 100 | alarm(1); 101 | } 102 | -------------------------------------------------------------------------------- /paexec/signals.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2011 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | extern int sigalrm_tics; 25 | 26 | void block_signals(void); 27 | void unblock_signals(void); 28 | void set_sig_handler(int sig, void (*handler) (int)); 29 | void ignore_sigpipe(void); 30 | void wait_for_sigalrm(void); 31 | void handler_sigchld(int dummy); 32 | void set_sigalrm_handler(void); 33 | void set_sigchld_handler(void); 34 | void handler_sigalrm(int dummy); 35 | -------------------------------------------------------------------------------- /paexec/tasks.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2014 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #ifdef HAVE_CONFIG_H 25 | /* if you need, add extra includes to config.h */ 26 | #include "config.h" 27 | #endif 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "decls.h" 35 | 36 | #include "tasks.h" 37 | #include "wrappers.h" 38 | 39 | extern char msg_delim; /* from paexec.c */ 40 | 41 | static int *deleted_tasks = NULL; 42 | 43 | struct task_entry { 44 | SLIST_ENTRY(task_entry) entries; /* List. */ 45 | int task_id; 46 | }; 47 | static SLIST_HEAD(task_head, task_entry) *arcs_outg = NULL, *arcs_inco = NULL; 48 | 49 | static int arcs_count = 0; 50 | 51 | static int *tasks_graph_deg = NULL; 52 | 53 | int tasks_count = 1; /* 0 - special meaning, not task ID */ 54 | int remained_tasks_count = 0; 55 | 56 | typedef struct task_struct { 57 | RB_ENTRY(task_struct) linkage; 58 | char *task; 59 | int task_id; 60 | } task_t; 61 | 62 | static int tasks_cmp(task_t *a, task_t *b) 63 | { 64 | return strcmp(a->task, b->task); 65 | } 66 | 67 | static RB_HEAD(tasks_entries, task_struct) tasks = RB_INITIALIZER(&tasks); 68 | 69 | RB_PROTOTYPE(tasks_entries, task_struct, linkage, tasks_cmp) 70 | RB_GENERATE(tasks_entries, task_struct, linkage, tasks_cmp) 71 | 72 | int graph_mode = 0; 73 | /* numeric task id to textual representation*/ 74 | static char ** id2task = NULL; 75 | /* numeric task id to weight */ 76 | static int *id2weight = NULL; 77 | static int *id2sum_weight = NULL; 78 | 79 | char *current_task = NULL; 80 | size_t current_task_sz = 0; 81 | 82 | int current_taskid = 0; 83 | 84 | static int *failed_taskids = NULL; 85 | int failed_taskids_count = 0; 86 | 87 | void tasks__init(void) 88 | { 89 | } 90 | 91 | void tasks__destroy(void) 92 | { 93 | task_t *data, *next; 94 | data = (task_t *) RB_MIN(tasks_entries, &tasks); 95 | while (data){ 96 | next = (task_t *) RB_NEXT(tasks_entries, &tasks, data); 97 | RB_REMOVE(tasks_entries, &tasks, data); 98 | free(data->task); 99 | free(data); 100 | 101 | data = next; 102 | } 103 | 104 | if (deleted_tasks) 105 | xfree(deleted_tasks); 106 | 107 | if (id2weight) 108 | xfree(id2weight); 109 | if (id2sum_weight) 110 | xfree(id2sum_weight); 111 | } 112 | 113 | void tasks__delete_task(int task, int print_task, int with_prefix) 114 | { 115 | struct task_entry *p; 116 | int to; 117 | 118 | assert(task < tasks_count); 119 | assert(task >= 0); 120 | 121 | if (!graph_mode){ 122 | if (id2task [task]){ 123 | xfree(id2task [task]); 124 | id2task [task] = NULL; 125 | } 126 | } 127 | 128 | if (graph_mode){ 129 | SLIST_FOREACH(p, &arcs_outg [task], entries){ 130 | to = p->task_id; 131 | if (tasks_graph_deg [to] > 0) 132 | --tasks_graph_deg [to]; 133 | } 134 | 135 | if(tasks_graph_deg [task] >= -1){ 136 | tasks_graph_deg [task] = -2; 137 | 138 | --remained_tasks_count; 139 | } 140 | } 141 | 142 | if (print_task){ 143 | if (!deleted_tasks [task]){ 144 | if (with_prefix) 145 | printf("%c", msg_delim); 146 | 147 | printf("%s", id2task [task]); 148 | 149 | deleted_tasks [task] = 1; 150 | } 151 | } 152 | } 153 | 154 | static void delete_task_rec2(int task, int with_prefix) 155 | { 156 | struct task_entry *p; 157 | int to; 158 | 159 | assert(task >= 0); 160 | 161 | tasks__delete_task(task, 1, with_prefix); 162 | 163 | SLIST_FOREACH(p, &arcs_outg [task], entries){ 164 | to = p->task_id; 165 | delete_task_rec2(to, 1); 166 | } 167 | } 168 | 169 | void tasks__delete_task_rec(int task) 170 | { 171 | memset(deleted_tasks, 0, tasks_count * sizeof(*deleted_tasks)); 172 | 173 | delete_task_rec2(task, 0); 174 | } 175 | 176 | static int get_new_task_num_from_graph(void) 177 | { 178 | int i; 179 | int best_weight = 0; 180 | int best_task = -1; 181 | 182 | for (i=1; i < tasks_count; ++i){ 183 | assert(tasks_graph_deg [i] >= -2); 184 | 185 | if (tasks_graph_deg [i] == 0){ 186 | if (id2sum_weight [i] > best_weight){ 187 | best_weight = id2sum_weight [i]; 188 | best_task = i; 189 | } 190 | } 191 | } 192 | 193 | return best_task; 194 | } 195 | 196 | static const char * get_new_task_from_graph(void) 197 | { 198 | /* topological sort of task graph */ 199 | int num = get_new_task_num_from_graph(); 200 | if (num == -1) 201 | return NULL; 202 | 203 | current_taskid = num; 204 | tasks_graph_deg [num] = -1; 205 | return id2task [num]; 206 | } 207 | 208 | static const char * get_new_task_from_stdin(void) 209 | { 210 | static char *task = NULL; 211 | static size_t task_sz = 0; 212 | 213 | ssize_t sz = 0; 214 | 215 | sz = xgetdelim(&task, &task_sz, eol_char, stdin); 216 | if (sz == -1) 217 | return NULL; 218 | 219 | if (sz > 0 && task [sz-1] == '\n') 220 | task [sz-1] = 0; 221 | 222 | current_taskid = tasks_count++; 223 | 224 | id2task = (char **) xrealloc( 225 | id2task, tasks_count * sizeof(*id2task)); 226 | id2task [current_taskid] = xstrdup(task); 227 | 228 | return task; 229 | } 230 | 231 | const char *tasks__get_new_task(void) 232 | { 233 | const char *task = NULL; 234 | size_t task_len = 0; 235 | 236 | if (failed_taskids_count > 0){ 237 | current_taskid = failed_taskids [--failed_taskids_count]; 238 | assert(current_taskid < tasks_count); 239 | task = id2task [current_taskid]; 240 | assert(task); 241 | }else if (graph_mode){ 242 | task = get_new_task_from_graph(); 243 | }else{ 244 | task = get_new_task_from_stdin(); 245 | } 246 | 247 | if (!task) 248 | return NULL; 249 | 250 | task_len = strlen(task); 251 | 252 | if (task_len >= current_task_sz){ 253 | current_task_sz = task_len+1; 254 | current_task = (char *) xrealloc(current_task, current_task_sz); 255 | } 256 | 257 | memcpy(current_task, task, task_len+1); 258 | 259 | return current_task; 260 | } 261 | 262 | int tasks__add_task(char *s, int weight) 263 | { 264 | task_t *n = malloc(sizeof(*n)); 265 | task_t *data; 266 | int task_id; 267 | 268 | n->task = s; 269 | 270 | data = RB_INSERT(tasks_entries, &tasks, n); 271 | if (data){ 272 | task_id = data->task_id; 273 | if (id2weight [task_id] < weight){ 274 | id2weight [task_id] = weight; 275 | id2sum_weight [task_id] = weight; 276 | } 277 | free(s); 278 | free(n); 279 | return task_id; 280 | }else{ 281 | n->task_id = tasks_count; 282 | 283 | ++tasks_count; 284 | ++remained_tasks_count; 285 | 286 | arcs_outg = (struct task_head *) xrealloc( 287 | arcs_outg, tasks_count * sizeof(*arcs_outg)); 288 | SLIST_INIT(&arcs_outg [tasks_count-1]); 289 | 290 | arcs_inco = (struct task_head *) xrealloc( 291 | arcs_inco, tasks_count * sizeof(*arcs_inco)); 292 | SLIST_INIT(&arcs_inco [tasks_count-1]); 293 | 294 | id2task = (char **) xrealloc( 295 | id2task, tasks_count * sizeof(*id2task)); 296 | id2task [tasks_count-1] = s; 297 | 298 | id2weight = (int *) xrealloc( 299 | id2weight, tasks_count * sizeof(*id2weight)); 300 | id2weight [tasks_count-1] = weight; 301 | 302 | id2sum_weight = (int *) xrealloc( 303 | id2sum_weight, tasks_count * sizeof(*id2sum_weight)); 304 | id2sum_weight [tasks_count-1] = weight; 305 | 306 | deleted_tasks = (int *) xrealloc( 307 | deleted_tasks, tasks_count * sizeof(*deleted_tasks)); 308 | deleted_tasks [tasks_count-1] = -1; 309 | 310 | tasks_graph_deg = (int *) xrealloc( 311 | tasks_graph_deg, tasks_count * sizeof(*tasks_graph_deg)); 312 | tasks_graph_deg [tasks_count-1] = 0; 313 | 314 | return tasks_count-1; 315 | } 316 | } 317 | 318 | void tasks__add_task_arc(int task_from, int task_to) 319 | { 320 | struct task_entry *p1,*p2; 321 | ++arcs_count; 322 | 323 | p1 = xmalloc(sizeof(*p1)); 324 | memset(p1, 0, sizeof(*p1)); 325 | p1->task_id = task_to; 326 | SLIST_INSERT_HEAD(&arcs_outg [task_from], p1, entries); 327 | 328 | p2 = xmalloc(sizeof(*p2)); 329 | memset(p2, 0, sizeof(*p2)); 330 | p2->task_id = task_from; 331 | SLIST_INSERT_HEAD(&arcs_inco [task_to], p2, entries); 332 | 333 | ++tasks_graph_deg [task_to]; 334 | } 335 | 336 | void tasks__mark_task_as_failed(int id) 337 | { 338 | failed_taskids = (int *) xrealloc( 339 | failed_taskids, 340 | (failed_taskids_count+1) * sizeof(*failed_taskids)); 341 | 342 | failed_taskids [failed_taskids_count++] = id; 343 | } 344 | 345 | static int *check_cycles__stack; 346 | static int *check_cycles__mark; 347 | 348 | static void check_cycles__outgoing(int stack_sz) 349 | { 350 | struct task_entry *p; 351 | int j; 352 | int s, t; 353 | int loop; 354 | int to; 355 | int from = check_cycles__stack [stack_sz-1]; 356 | 357 | assert(stack_sz > 0); 358 | 359 | assert(check_cycles__mark [from] == 0); 360 | check_cycles__mark [from] = 2; /* currently in the path */ 361 | 362 | SLIST_FOREACH(p, &arcs_outg [from], entries){ 363 | to = p->task_id; 364 | assert(stack_sz < tasks_count); 365 | 366 | check_cycles__stack [stack_sz] = to; 367 | 368 | switch (check_cycles__mark [to]){ 369 | case 2: 370 | loop = 0; 371 | fprintf(stderr, "Cyclic dependancy detected:\n"); 372 | for (j=1; j <= stack_sz; ++j){ 373 | s = check_cycles__stack [j-1]; 374 | t = check_cycles__stack [j]; 375 | 376 | if (!loop && s != to) 377 | continue; 378 | 379 | loop = 1; 380 | fprintf(stderr, " %s -> %s\n", id2task [s], id2task [t]); 381 | } 382 | 383 | exit(1); 384 | case 0: 385 | check_cycles__outgoing(stack_sz + 1); 386 | break; 387 | case 1: 388 | break; 389 | default: 390 | abort(); /* this should not happen */ 391 | } 392 | } 393 | 394 | check_cycles__mark [from] = 1; /* already seen */ 395 | } 396 | 397 | void tasks__check_for_cycles(void) 398 | { 399 | int i; 400 | 401 | check_cycles__stack = xmalloc( 402 | tasks_count * sizeof(check_cycles__stack [0])); 403 | 404 | check_cycles__mark = xmalloc( 405 | tasks_count * sizeof(check_cycles__mark [0])); 406 | memset(check_cycles__mark, 0, 407 | tasks_count * sizeof(check_cycles__mark [0])); 408 | 409 | /* */ 410 | for (i=1; i < tasks_count; ++i){ 411 | switch (check_cycles__mark [i]){ 412 | case 0: 413 | check_cycles__stack [0] = i; 414 | check_cycles__outgoing(1); 415 | break; 416 | case 1: 417 | break; 418 | case 2: 419 | abort(); /* this should not happen */ 420 | } 421 | } 422 | 423 | xfree(check_cycles__mark); 424 | xfree(check_cycles__stack); 425 | } 426 | 427 | static char *seen = NULL; 428 | 429 | static void tasks__make_sum_weights_rec(int *accu_w, int task) 430 | { 431 | struct task_entry *p; 432 | int to; 433 | 434 | seen [task] = 1; 435 | SLIST_FOREACH(p, &arcs_outg [task], entries){ 436 | to = p->task_id; 437 | if (seen [to]) 438 | continue; 439 | 440 | tasks__make_sum_weights_rec(accu_w, to); 441 | *accu_w += id2weight [to]; 442 | seen [to] = 1; 443 | } 444 | } 445 | 446 | void tasks__make_sum_weights(void) 447 | { 448 | int i; 449 | 450 | if (!graph_mode) 451 | return; 452 | 453 | seen = (char *) xmalloc(tasks_count); 454 | 455 | for (i=1; i < tasks_count; ++i){ 456 | memset(seen, 0, tasks_count); 457 | tasks__make_sum_weights_rec(&id2sum_weight [i], i); 458 | } 459 | 460 | xfree(seen); 461 | } 462 | 463 | static void tasks__make_max_weights_rec(int *accu_w, int task) 464 | { 465 | struct task_entry *p; 466 | int to; 467 | int curr_w; 468 | 469 | seen [task] = 1; 470 | SLIST_FOREACH(p, &arcs_outg [task], entries){ 471 | to = p->task_id; 472 | if (seen [to]) 473 | continue; 474 | 475 | tasks__make_max_weights_rec(accu_w, to); 476 | curr_w = id2weight [to]; 477 | if (*accu_w < curr_w) 478 | *accu_w = curr_w; 479 | seen [to] = 1; 480 | } 481 | } 482 | 483 | void tasks__make_max_weights(void) 484 | { 485 | int i; 486 | 487 | if (!graph_mode) 488 | return; 489 | 490 | seen = (char *) xmalloc(tasks_count); 491 | 492 | for (i=1; i < tasks_count; ++i){ 493 | memset(seen, 0, tasks_count); 494 | 495 | tasks__make_max_weights_rec(&id2sum_weight [i], i); 496 | } 497 | 498 | xfree(seen); 499 | } 500 | 501 | void tasks__print_sum_weights(void) 502 | { 503 | int i; 504 | 505 | if (!graph_mode) 506 | return; 507 | 508 | for (i=1; i < tasks_count; ++i){ 509 | fprintf(stderr, "weight [%s]=%d\n", id2task [i], id2weight [i]); 510 | fprintf(stderr, "sum_weight [%s]=%d\n", id2task [i], id2sum_weight [i]); 511 | } 512 | } 513 | -------------------------------------------------------------------------------- /paexec/tasks.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2013 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #ifndef _TASKS_H_ 25 | #define _TASKS_H_ 26 | 27 | /* The following variables are read-only, do set them directly! */ 28 | 29 | /* a number of tasks */ 30 | extern int tasks_count; 31 | /* true "paexec -s" */ 32 | extern int graph_mode; 33 | /* last read task and its id */ 34 | extern char *current_task; 35 | extern int current_taskid; 36 | /* a number of tasks to be done */ 37 | extern int remained_tasks_count; 38 | /* a number of tasks with FATAL failure (e.g. connection lost) */ 39 | extern int failed_taskids_count; 40 | 41 | void tasks__init(void); 42 | int tasks__add_task(char *s, int weight); 43 | void tasks__add_task_arc(int task_from, int task_to); 44 | void tasks__check_for_cycles(void); 45 | void tasks__delete_task(int task, int print_task, int with_prefix); 46 | void tasks__delete_task_rec(int task); 47 | void tasks__destroy(void); 48 | const char *tasks__get_new_task(void); 49 | void tasks__mark_task_as_failed(int taskid); 50 | void tasks__make_sum_weights(void); 51 | void tasks__make_max_weights(void); 52 | void tasks__print_sum_weights(void); 53 | 54 | #endif // _TASKS_H_ 55 | -------------------------------------------------------------------------------- /paexec/wrappers.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2019 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | 36 | #include "decls.h" 37 | #include "wrappers.h" 38 | #include "common.h" 39 | 40 | void nonblock(int fd) 41 | { 42 | int ret = fcntl(fd, F_GETFL, 0); 43 | if (ret == -1){ 44 | perror("fcntl failed(2)"); 45 | exit(1); 46 | } 47 | 48 | ret = fcntl(fd, F_SETFL, ret | O_NONBLOCK); 49 | if (ret == -1){ 50 | perror("fcntl failed(2)"); 51 | exit(1); 52 | } 53 | } 54 | 55 | void xsigaddset(sigset_t *set, int signo) 56 | { 57 | if (sigaddset(set, signo)){ 58 | perror("sigaddset(2) failed"); 59 | exit(1); 60 | } 61 | } 62 | 63 | void xsigprocmask(int how, const sigset_t *set, sigset_t *oset) 64 | { 65 | if (sigprocmask(how, set, oset)){ 66 | perror("sigprocmask(2) failed"); 67 | exit(1); 68 | } 69 | } 70 | 71 | ssize_t xgetdelim(char **lineptr, size_t *n, int delimiter, FILE *stream) 72 | { 73 | ssize_t ret = getdelim(lineptr, n, delimiter, stream); 74 | 75 | if (ret == (ssize_t) -1 && ferror(stdin)){ 76 | perror("getdelim(3) failed"); 77 | exit(1); 78 | } 79 | 80 | return ret; 81 | } 82 | 83 | char *xstrdup(const char *s) 84 | { 85 | char *ret = strdup(s); 86 | if (!ret){ 87 | perror("strdup(3) failed"); 88 | exit(1); 89 | } 90 | 91 | return ret; 92 | } 93 | 94 | void *xmalloc(size_t size) 95 | { 96 | void *ret = malloc(size); 97 | if (!ret){ 98 | perror("malloc(3) failed"); 99 | exit(1); 100 | } 101 | 102 | return ret; 103 | } 104 | 105 | void *xcalloc(size_t number, size_t size) 106 | { 107 | void *ret = calloc(number, size); 108 | if (!ret){ 109 | perror("calloc(3) failed"); 110 | exit(1); 111 | } 112 | 113 | return ret; 114 | } 115 | 116 | void *xrealloc(void *ptr, size_t size) 117 | { 118 | void *ret = realloc(ptr, size); 119 | if (!ret){ 120 | perror("realloc(3) failed"); 121 | exit(1); 122 | } 123 | 124 | return ret; 125 | } 126 | 127 | void xfree(void *p) 128 | { 129 | if (p) 130 | free(p); 131 | } 132 | 133 | void xshquote(const char *arg, char *buf, size_t bufsize) 134 | { 135 | size_t ret = shquote(arg, buf, bufsize); 136 | if ((size_t)-1 == ret){ 137 | err__internal(__func__, "paexec: shquote(3) failed"); 138 | exit(1); 139 | } 140 | } 141 | 142 | void err__fatal(const char *routine, const char *m) 143 | { 144 | kill_childs(); 145 | wait_for_childs(); 146 | 147 | fflush(stdout); 148 | 149 | err_fatal(routine, "%s", m); 150 | } 151 | 152 | void err__fatal_errno(const char *routine, const char *m) 153 | { 154 | kill_childs(); 155 | wait_for_childs(); 156 | 157 | fflush(stdout); 158 | 159 | err_fatal_errno(routine, "%s", m); 160 | } 161 | 162 | void err__internal(const char *routine, const char *m) 163 | { 164 | kill_childs(); 165 | wait_for_childs(); 166 | 167 | fflush(stdout); 168 | 169 | err_internal(routine, "%s", m); 170 | } 171 | -------------------------------------------------------------------------------- /paexec/wrappers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007-2013 Aleksey Cheusov 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining 5 | * a copy of this software and associated documentation files (the 6 | * "Software"), to deal in the Software without restriction, including 7 | * without limitation the rights to use, copy, modify, merge, publish, 8 | * distribute, sublicense, and/or sell copies of the Software, and to 9 | * permit persons to whom the Software is furnished to do so, subject to 10 | * the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be 13 | * included in all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | */ 23 | 24 | #ifndef _WRAPPERS_H_ 25 | #define _WRAPPERS_H_ 26 | 27 | #if HAVE_HEADER_SYS_SELECT_H 28 | #include 29 | #endif 30 | #include /* On ancient HP-UX select(2) is declared here */ 31 | #include 32 | #include 33 | #include 34 | 35 | void nonblock(int fd); 36 | void xsigprocmask(int how, const sigset_t *set, sigset_t *oset); 37 | void xsigaddset(sigset_t *set, int signo); 38 | ssize_t xgetdelim(char **lineptr, size_t *n, int delimiter, FILE *stream); 39 | char *xstrdup(const char *s); 40 | void *xmalloc(size_t size); 41 | void *xcalloc(size_t number, size_t size); 42 | void *xrealloc(void *ptr, size_t size); 43 | void xfree(void *p); 44 | void xshquote(const char *arg, char *buf, size_t bufsize); 45 | 46 | void err__fatal(const char *routine, const char *m); 47 | void err__fatal_errno(const char *routine, const char *m); 48 | void err__internal(const char *routine, const char *m); 49 | 50 | void kill_childs(void); /* paexec.c */ 51 | void wait_for_childs(void); /* paexec.c */ 52 | 53 | #endif /* _WRAPPERS_H_ */ 54 | -------------------------------------------------------------------------------- /presentation/Makefile: -------------------------------------------------------------------------------- 1 | ################################################## 2 | 3 | FILES = paexec.pdf 4 | FILESDIR = ${DOCDIR} 5 | 6 | .PHONE : all 7 | all : paexec.pdf 8 | 9 | .PHONY : pdf dvi 10 | pdf : paexec.pdf 11 | ps : paexec.ps 12 | dvi : paexec.dvi 13 | 14 | .SUFFIXES: .ps .eps .pdf .dvi .tex .dot 15 | 16 | paexec.ps paexec.dvi: dep-graph.eps 17 | 18 | .ps.pdf: 19 | ps2pdf "$<" "$@" 20 | 21 | .dot.eps: 22 | dot -Tps ${.IMPSRC} > ${.TARGET} 23 | 24 | .dvi.ps: 25 | dvips $< 26 | 27 | .tex.dvi: 28 | latex "${.IMPSRC}" && latex "${.IMPSRC}" 29 | 30 | GARBAGE = *.dvi *.aux *.vrb *.toc *.snm *.log *.nav *.out *.eps *.ps _mkc_* 31 | CLEANFILES += ${GARBAGE} *.pdf 32 | 33 | .PHONY: _clean_garbage 34 | _clean_garbage: 35 | rm -f ${GARBAGE} 36 | 37 | _prepdist: all _clean_garbage 38 | 39 | .include 40 | -------------------------------------------------------------------------------- /presentation/dep-graph.dot: -------------------------------------------------------------------------------- 1 | digraph FSA { 2 | node [ shape=box fontsize=14 fontface="Arial"]; 3 | 4 | "audio/cd-discid" -> "audio/abcde" [ fontsize = 14]; 5 | "textproc/gsed" -> "audio/abcde" [ fontsize = 14]; 6 | "audio/cdparanoia" -> "audio/abcde" [ fontsize = 14]; 7 | "audio/id3v2" -> "audio/abcde" [ fontsize = 14]; 8 | "audio/id3" -> "audio/abcde" [ fontsize = 14]; 9 | "misc/mkcue" -> "audio/abcde" [ fontsize = 14]; 10 | "shells/bash" -> "audio/abcde" [ fontsize = 14]; 11 | "devel/libtool-base" -> "audio/cdparanoia" [ fontsize = 14]; 12 | "devel/gmake" -> "audio/cdparanoia" [ fontsize = 14]; 13 | "devel/libtool-base" -> "audio/id3lib" [ fontsize = 14]; 14 | "devel/gmake" -> "audio/id3v2" [ fontsize = 14]; 15 | "audio/id3lib" -> "audio/id3v2" [ fontsize = 14]; 16 | "devel/m4" -> "devel/bison" [ fontsize = 14]; 17 | "lang/f2c" -> "devel/libtool-base" [ fontsize = 14]; 18 | "devel/gmake" -> "misc/mkcue" [ fontsize = 14]; 19 | "devel/bison" -> "shells/bash" [ fontsize = 14]; 20 | node [ shape = box ]; 21 | "devel/m4" [ ]; 22 | "shells/bash" [ ]; 23 | "audio/id3lib" [ ]; 24 | "misc/mkcue" [ ]; 25 | "audio/id3" [ ]; 26 | "lang/f2c" [ ]; 27 | "devel/gmake" [ ]; 28 | "textproc/gsed" [ ]; 29 | "audio/id3v2" [ ]; 30 | "audio/abcde" [ ]; 31 | "devel/bison" [ ]; 32 | "audio/cdparanoia" [ ]; 33 | "devel/libtool-base" [ ]; 34 | "audio/cd-discid" [ ]; 35 | } 36 | -------------------------------------------------------------------------------- /presentation/paexec.tex: -------------------------------------------------------------------------------- 1 | %%%begin-myprojects 2 | \documentclass[hyperref={colorlinks=true}]{beamer} 3 | 4 | \usepackage{fancyvrb,relsize} 5 | \usepackage{graphicx} 6 | 7 | \setbeamertemplate{navigation symbols}{} 8 | 9 | %\usetheme{Boadilla} 10 | %\usetheme{CambridgeUS} 11 | %\usetheme{Malmoe} 12 | %\usetheme{Singapore} 13 | %\usetheme{boxes} 14 | 15 | %\usecolortheme{crane} 16 | %\usecolortheme{dove} 17 | \usecolortheme{seagull} % very cool with \usetheme{default} 18 | %\usefonttheme{professionalfonts} 19 | %\useinnertheme{rectangles} 20 | 21 | \mode 22 | \title{paexec -- distributes tasks over network or CPUs} 23 | \author{Aleksey Cheusov \\ \texttt{vle@gmx.net}} 24 | \date{Minsk, Belarus, 2013} 25 | 26 | \begin{document} 27 | 28 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 29 | 30 | \newenvironment{CodeSmall}[1]% 31 | {\Verbatim[label=\bf{#1},frame=single,% 32 | fontsize=\footnotesize,% 33 | commandchars=\\\{\}]}% 34 | {\endVerbatim} 35 | \newenvironment{CodeSmallNoLabel}% 36 | {\Verbatim[frame=single,% 37 | fontsize=\footnotesize,% 38 | commandchars=\\\{\}]}% 39 | {\endVerbatim} 40 | 41 | \newenvironment{Code}[1]% 42 | {\Verbatim[label=\bf{#1},frame=single,% 43 | fontsize=\small,% 44 | commandchars=\\\{\}]}% 45 | {\endVerbatim} 46 | \newenvironment{CodeNoLabel}% 47 | {\Verbatim[frame=single,% 48 | fontsize=\small,% 49 | commandchars=\\\{\}]}% 50 | {\endVerbatim} 51 | 52 | \newenvironment{CodeLarge}[1]% 53 | {\Verbatim[label=\bf{#1},frame=single,% 54 | fontsize=\large,% 55 | commandchars=\\\{\}]}% 56 | {\endVerbatim} 57 | \newenvironment{CodeLargeNoLabel}% 58 | {\Verbatim[frame=single,% 59 | fontsize=\large,% 60 | commandchars=\\\{\}]}% 61 | {\endVerbatim} 62 | 63 | %\newcommand{\prompt}[1]{\textcolor{blue}{#1}} 64 | %\newcommand{\prompt}[1]{\textbf{#1}\textnormal{}} 65 | \newcommand{\prompt}[1]{{\bf{#1}}} 66 | %\newcommand{\h}[1]{\textbf{#1}} 67 | %\newcommand{\h}[1]{\bf{#1}\textnormal{}} 68 | \newcommand{\h}[1]{{\bf{#1}}} 69 | \newcommand{\name}[1]{{\tt{#1}}} 70 | \newcommand{\URL}[1]{\textbf{#1}} 71 | \newcommand{\AutohellFile}[1]{\textcolor{red}{#1}} 72 | \newcommand{\MKCfile}[1]{\textcolor{green}{#1}} 73 | \newcommand{\ModuleName}[1]{\textbf{#1}\textnormal{}} 74 | \newcommand{\ProgName}[1]{\textbf{#1}\textnormal{}} 75 | \newcommand{\ProjectName}[1]{\textbf{#1}\textnormal{}} 76 | \newcommand{\PackageName}[1]{\textbf{#1}\textnormal{}} 77 | \newcommand{\MKC}[1]{\large\textsf{#1}\textnormal{}\normalsize} 78 | 79 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 80 | %% \begin{frame} 81 | %% \frametitle{qqq} 82 | %% \begin{code}{files in the directory} 83 | %% bla bla bla 84 | %% \end{code} 85 | %% \end{frame} 86 | 87 | %%%end-myprojects 88 | 89 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 90 | \begin{frame} 91 | \titlepage 92 | \end{frame} 93 | 94 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 95 | \begin{frame}{} 96 | \frametitle{Problem} 97 | \begin{block}{} 98 | \begin{itemize} 99 | \item Huge amount of data to process 100 | \item Typical desktop machines have more than one CPU 101 | \item Unlimited resources are available on the Internet 102 | \item Heterogeneous environment (*BSD, Linux, Windows...) 103 | \end{itemize} 104 | \end{block} 105 | \end{frame} 106 | 107 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 108 | \begin{frame}[fragile] 109 | \frametitle{Solution} 110 | 111 | \begin{block}{} 112 | \begin{CodeLarge}{Usage} 113 | paexec [OPTIONS] \textbackslash 114 | -n 'machines or CPUs' \textbackslash 115 | -t 'transport program' \textbackslash 116 | -c 'calculator' < tasks 117 | \end{CodeLarge} 118 | \begin{CodeLarge}{example} 119 | ls -1 *.wav | \textbackslash 120 | paexec -x -c 'flac -s' -n +4 > /dev/null 121 | \end{CodeLarge} 122 | \begin{CodeLarge}{example} 123 | paexec \textbackslash 124 | -n 'host1 host2 host3' \textbackslash 125 | -t /usr/bin/ssh \textbackslash 126 | -c ~/bin/toupper < tasks 127 | \end{CodeLarge} 128 | \end{block} 129 | \end{frame} 130 | 131 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 132 | \begin{frame}[fragile] 133 | \frametitle{Example 1: toupper} 134 | Our goal is to convert strings to upper case in parallel 135 | \begin{block}{} 136 | \begin{Code}{\~{}/bin/toupper} 137 | #!/usr/bin/awk -f 138 | \{ 139 | print " ", toupper(\$0) 140 | print "" # empty line -- end-of-task marker! 141 | fflush() # We must flush stdout! 142 | \} 143 | \end{Code} 144 | \end{block} 145 | \begin{block}{} 146 | \begin{Code}{\~{}/tmp/tasks} 147 | apple 148 | bananas 149 | orange 150 | \end{Code} 151 | \end{block} 152 | \end{frame} 153 | 154 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 155 | \begin{frame}[fragile] 156 | \frametitle{Example 1: toupper} 157 | \name{\~{}/bin/toupper} script will be run only once on remote servers 158 | ``server1'' and ``server2''. It takes tasks from stdin (one task per 159 | line). Transport program is \name{ssh(1)}. 160 | \begin{block}{} 161 | \begin{CodeLarge}{paexec invocation} 162 | \prompt{\$} paexec \h{-t} ssh \h{-c} ~/bin/toupper \textbackslash 163 | \h{-n} 'server1 server2' < tasks > results 164 | \prompt{\$} cat results 165 | BANANAS 166 | ORANGE 167 | APPLE 168 | \prompt{\$} 169 | \end{CodeLarge} 170 | \end{block} 171 | \end{frame} 172 | 173 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 174 | \begin{frame}[fragile] 175 | \frametitle{Example 1: toupper} 176 | Options \h{-l} and \h{-r} add the task number and server where this 177 | task is processed to stdout. 178 | \begin{block}{} 179 | \begin{CodeLarge}{paexec -lr invocation} 180 | \prompt{\$} paexec \h{-lr} -t ssh -c ~/bin/toupper \textbackslash 181 | -n 'server1 server2' < tasks > results 182 | \prompt{\$} cat results 183 | \h{server2 2} BANANAS 184 | \h{server2 3} ORANGE 185 | \h{server1 1} APPLE 186 | \prompt{\$} 187 | \end{CodeLarge} 188 | \end{block} 189 | \end{frame} 190 | 191 | 192 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 193 | \begin{frame}[fragile] 194 | \frametitle{Example 1: toupper} 195 | The same as above but four instances of \name{\~{}/bin/toupper} are ran locally. 196 | \begin{block}{} 197 | \begin{CodeLarge}{paexec invocation} 198 | \prompt{\$} paexec \h{-n} +4 -c ~/bin/toupper 199 | < tasks > results 200 | \prompt{\$} cat results 201 | BANANAS 202 | ORANGE 203 | APPLE 204 | \prompt{\$} 205 | \end{CodeLarge} 206 | \end{block} 207 | \end{frame} 208 | 209 | 210 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 211 | \begin{frame}[fragile] 212 | \frametitle{Example 1: toupper} 213 | The same as above but without \name{\~{}/bin/toupper}. In this example we 214 | run AWK program for each individual task. At the same time we still 215 | make only one ssh connection to each server regardless of a number of 216 | tasks given on input. 217 | \begin{block}{} 218 | \begin{CodeLarge}{paexec invocation} 219 | \prompt{\$} paexec \h{-x} -t ssh -n 'server1 server2' \textbackslash 220 | \h{-c} "awk 'BEGIN \{print toupper(ARGV[1])\}' " \textbackslash 221 | < tasks > results 222 | \prompt{\$} cat results 223 | ORANGE 224 | BANANAS 225 | APPLE 226 | \prompt{\$} 227 | \end{CodeLarge} 228 | \end{block} 229 | \end{frame} 230 | 231 | 232 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 233 | \begin{frame}[fragile] 234 | \frametitle{Example 1: toupper} 235 | If we want to "shquate" less one can use option \h{-C} instead of \h{-c} 236 | and specify command after options. 237 | \begin{block}{} 238 | \begin{CodeLarge}{paexec invocation} 239 | \prompt{\$} paexec -x \h{-C} -t ssh -n 'server1 server2' \textbackslash 240 | awk 'BEGIN \{print toupper(ARGV[1])\}' \textbackslash 241 | < tasks > results 242 | \prompt{\$} cat results 243 | ORANGE 244 | BANANAS 245 | APPLE 246 | \prompt{\$} 247 | \end{CodeLarge} 248 | \end{block} 249 | \end{frame} 250 | 251 | 252 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 253 | \begin{frame}[fragile] 254 | \frametitle{Example 1: toupper} 255 | With options \h{-z} or \h{-Z} we can easily run our tasks on 256 | unreliable hosts. If we cannot connect or lost connection to the 257 | server, paexec will redistribute failed tasks to another server. 258 | \begin{block}{} 259 | \begin{CodeLarge}{paexec invocation} 260 | \prompt{\$} paexec \h{-Z}240 -x -t ssh \textbackslash 261 | -n 'server1 badhostname server2' \textbackslash 262 | -c "awk 'BEGIN \{print toupper(ARGV[1])\}' " \textbackslash 263 | < tasks > results 264 | {\it ssh: Could not resolve hostname badhostname:} 265 | {\it No address associated with hostname} 266 | {\it badhostname 1 fatal} 267 | \prompt{\$} cat results 268 | ORANGE 269 | BANANAS 270 | APPLE 271 | \prompt{\$} 272 | \end{CodeLarge} 273 | \end{block} 274 | \end{frame} 275 | 276 | 277 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 278 | \linespread{0.5} 279 | \begin{frame}[fragile] 280 | \frametitle{Example 2: parallel banner(1)} 281 | 282 | \begin{block}{} 283 | \begin{CodeLarge}{what is banner(1)?} 284 | \prompt{\$} banner -f @ NetBSD 285 | 286 | @ @ @@@@@@ @@@@@ @@@@@@ 287 | @@ @ @@@@@@ @@@@@ @ @ @ @ @ @ 288 | @ @ @ @ @ @ @ @ @ @ 289 | @ @ @ @@@@@ @ @@@@@@ @@@@@ @ @ 290 | @ @ @ @ @ @ @ @ @ @ 291 | @ @@ @ @ @ @ @ @ @ @ 292 | @ @ @@@@@@ @ @@@@@@ @@@@@ @@@@@@ 293 | 294 | 295 | \prompt{\$} 296 | \end{CodeLarge} 297 | \end{block} 298 | \end{frame} 299 | \linespread{1} 300 | 301 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 302 | \begin{frame}[fragile] 303 | \frametitle{Example 2: parallel banner(1)} 304 | \name{\~{}/bin/pbanner} is wrapper for \name{banner(1)} for reading tasks line by line. 305 | Magic line is used instead empty line as an end-of-task marker. 306 | \begin{block}{} 307 | \begin{Code}{\~{}/bin/pbanner} 308 | #!/usr/bin/env sh 309 | 310 | while read task; do 311 | banner -f M "\$task" | 312 | echo \h{"\$PAEXEC\_EOT"} # end-of-task marker 313 | done 314 | \end{Code} 315 | \begin{Code}{tasks} 316 | pae 317 | xec 318 | \end{Code} 319 | 320 | \begin{Code}{paexec invocation} 321 | \prompt{\$} paexec -l \h{-mt='SE@X-L0S0!&'} -c ~/bin/pbanner \textbackslash 322 | -n +2 < tasks > result 323 | \prompt{\$} 324 | \end{Code} 325 | \end{block} 326 | \end{frame} 327 | 328 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 329 | \linespread{0.5} 330 | \begin{frame}[fragile] 331 | \frametitle{Example 2: parallel banner(1)} 332 | \name{paexec(1)} reads calculator's output asynchronously. 333 | So, its output is sliced. 334 | \begin{block}{} 335 | \begin{CodeSmall}{Sliced result} 336 | \prompt{\$} cat result 337 | 2 338 | 2 339 | 2 M M MMMMMM MMMM 340 | 2 M M M M M 341 | 1 342 | 1 343 | 1 MMMMM MM MMMMMM 344 | 1 M M M M M 345 | 1 M M M M MMMMM 346 | 1 MMMMM MMMMMM M 347 | 2 MM MMMMM M 348 | 2 MM M M 349 | 2 M M M M M 350 | 2 M M MMMMMM MMMM 351 | 2 352 | 1 M M M M 353 | 1 M M M MMMMMM 354 | 1 355 | \prompt{\$} 356 | \end{CodeSmall} 357 | \end{block} 358 | \end{frame} 359 | \linespread{1} 360 | 361 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 362 | \linespread{0.5} 363 | \begin{frame}[fragile] 364 | \frametitle{Example 2: parallel banner(1)} 365 | \name{paexec\_reorder(1)} normalizes \name{paexec(1)}'s output. 366 | \begin{block}{} 367 | \begin{CodeSmall}{Ordered result} 368 | \prompt{\$} paexec_reorder \h{-mt='SE@X-L0S0!&'} results 369 | 370 | 371 | MMMMM MM MMMMMM 372 | M M M M M 373 | M M M M MMMMM 374 | MMMMM MMMMMM M 375 | M M M M 376 | M M M MMMMMM 377 | 378 | 379 | 380 | M M MMMMMM MMMM 381 | M M M M M 382 | MM MMMMM M 383 | MM M M 384 | M M M M M 385 | M M MMMMMM MMMM 386 | 387 | 388 | \prompt{\$} 389 | \end{CodeSmall} 390 | \end{block} 391 | \end{frame} 392 | \linespread{1} 393 | 394 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 395 | \linespread{0.5} 396 | \begin{frame}[fragile] 397 | \frametitle{Example 2: parallel banner(1)} 398 | The same as above but using magic end-of-task marker provided by \name{paexec(1)}. 399 | \begin{block}{} 400 | \begin{CodeSmall}{Ordered result} 401 | \prompt{\$} paexec \h{-y} -lc ~/bin/pbanner -n+2 < tasks | paexec_reorder \h{-y} 402 | 403 | 404 | MMMMM MM MMMMMM 405 | M M M M M 406 | M M M M MMMMM 407 | MMMMM MMMMMM M 408 | M M M M 409 | M M M MMMMMM 410 | 411 | 412 | 413 | M M MMMMMM MMMM 414 | M M M M M 415 | MM MMMMM M 416 | MM M M 417 | M M M M M 418 | M M MMMMMM MMMM 419 | 420 | 421 | \prompt{\$} 422 | \end{CodeSmall} 423 | \end{block} 424 | \end{frame} 425 | \linespread{1} 426 | 427 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 428 | \linespread{0.5} 429 | \begin{frame}[fragile] 430 | \frametitle{Example 2: parallel banner(1)} 431 | For this trivial task wrapper like \name{\~{}/bin/pbanner} is not needed. 432 | We can easily run \name{banner(1)} directly. 433 | \begin{block}{} 434 | \begin{CodeSmall}{Sliced result} 435 | \prompt{\$} paexec -l \h{-x} -c banner -n+2 < tasks 436 | 2 437 | 2 438 | 2 M M MMMMMM MMMM 439 | 2 M M M M M 440 | 2 MM MMMMM M 441 | 2 MM M M 442 | 2 M M M M M 443 | 1 444 | 1 445 | 1 MMMMM MM MMMMMM 446 | 1 M M M M M 447 | 1 M M M M MMMMM 448 | 1 MMMMM MMMMMM M 449 | 2 M M MMMMMM MMMM 450 | 2 451 | 1 M M M M 452 | 1 M M M MMMMMM 453 | 1 454 | \prompt{\$} 455 | \end{CodeSmall} 456 | \end{block} 457 | \end{frame} 458 | \linespread{1} 459 | 460 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 461 | \begin{frame}[fragile] 462 | \frametitle{Example 3: dependency graph of tasks} 463 | \name{paexec(1)} is able to build tasks taking into account their ``dependencies''. 464 | Here ``devel/gmake'' and others are pkgsrc packages. Our goal in this example 465 | is to build pkgsrc package audio/abcde and all its build-time and compile-time 466 | dependencies. 467 | \begin{block}{} 468 | \begin{figure} 469 | \includegraphics[width=\textwidth, height=\textheight, keepaspectratio=true]{dep-graph.eps} 470 | \end{figure} 471 | \end{block} 472 | \end{frame} 473 | 474 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 475 | \begin{frame}[fragile] 476 | \frametitle{Example 3: dependency graph of tasks} 477 | \name{``paexec \h{-g}''} takes a dependency graph on input (in \name{tsort(1)} format). 478 | Tasks are separated by space (\name{paexec -md=}). 479 | \begin{block}{} 480 | \begin{CodeSmall}{\~{}/tmp/packages\_to\_build} 481 | audio/cd-discid audio/abcde 482 | textproc/gsed audio/abcde 483 | audio/cdparanoia audio/abcde 484 | audio/id3v2 audio/abcde 485 | audio/id3 audio/abcde 486 | misc/mkcue audio/abcde 487 | shells/bash audio/abcde 488 | devel/libtool-base audio/cdparanoia 489 | devel/gmake audio/cdparanoia 490 | devel/libtool-base audio/id3lib 491 | devel/gmake audio/id3v2 492 | audio/id3lib audio/id3v2 493 | devel/m4 devel/bison 494 | lang/f2c devel/libtool-base 495 | devel/gmake misc/mkcue 496 | devel/bison shells/bash 497 | \end{CodeSmall} 498 | \end{block} 499 | \end{frame} 500 | 501 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 502 | \begin{frame}[fragile] 503 | \frametitle{Example 3: dependency graph of tasks} 504 | If option -g is applied, every task may succeed or fail. In case of failure 505 | all dependants fail recursively. 506 | For this to work we have to slightly adapt ``calculator''. 507 | \begin{block}{} 508 | \begin{CodeSmall}{\~{}/bin/pkg\_builder} 509 | #!/usr/bin/awk -f 510 | 511 | \{ 512 | print "build " \$0 513 | \h{print "success"} # build succeeded! (paexec -ms=) 514 | print "" # end-of-task marker 515 | fflush() # we must flush stdout 516 | \} 517 | \end{CodeSmall} 518 | \end{block} 519 | \end{frame} 520 | 521 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 522 | \begin{frame}[fragile] 523 | \frametitle{Example 3: dependency graph of tasks} 524 | 525 | \begin{block}{} 526 | \begin{CodeSmall}{paexec -g invocation (no failures)} 527 | \prompt{\$} paexec \h{-g} -l -c ~/bin/pkg_builder -n 'server2 server1' \textbackslash 528 | -t ssh < ~/tmp/packages_to_build | paexec_reorder > result 529 | \prompt{\$} cat result 530 | build textproc/gsed 531 | \h{success} 532 | build devel/gmake 533 | success 534 | build misc/mkcue 535 | success 536 | build devel/m4 537 | success 538 | build devel/bison 539 | success 540 | ... 541 | build audio/id3v2 542 | success 543 | build audio/abcde 544 | success 545 | \prompt{\$} 546 | \end{CodeSmall} 547 | \end{block} 548 | \end{frame} 549 | 550 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 551 | \begin{frame}[fragile] 552 | \frametitle{Example 3: dependency graph of tasks} 553 | Let's suppose that ``devel/gmake'' fails to build. 554 | 555 | \begin{block}{} 556 | \begin{CodeSmall}{\~{}/bin/pkg\_builder} 557 | #!/usr/bin/awk -f 558 | 559 | \{ 560 | print "build " \$0 561 | if (\$0 == "devel/gmake") 562 | \h{print "failure"} # Oh no... 563 | else 564 | print "success" # build succeeded! 565 | 566 | print "" # end-of-task marker 567 | fflush() # we must flush stdout 568 | \} 569 | \end{CodeSmall} 570 | \end{block} 571 | \end{frame} 572 | 573 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 574 | \begin{frame}[fragile] 575 | \frametitle{Example 3: dependency graph of tasks} 576 | Package ``devel/gmake'' and all dependants are marked as failed. 577 | Even if failures happen, the build continues. 578 | \begin{block}{} 579 | \begin{CodeSmall}{paexec -g invocation (with failures)} 580 | \prompt{\$} paexec -gl -c ~/bin/pkg_builder -n 'server2 server1' \textbackslash 581 | -t ssh < ~/tmp/packages_to_build | paexec_reorder > result 582 | \prompt{\$} cat result 583 | build audio/cd-discid 584 | success 585 | build audio/id3 586 | success 587 | build devel/gmake 588 | \h{failure} 589 | \h{devel/gmake audio/cdparanoia audio/abcde audio/id3v2 misc/mkcue} 590 | build devel/m4 591 | success 592 | build textproc/gsed 593 | success 594 | ... 595 | \prompt{\$} 596 | \end{CodeSmall} 597 | \end{block} 598 | \end{frame} 599 | 600 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 601 | \begin{frame}[fragile] 602 | \frametitle{Example 3: dependency graph of tasks} 603 | paexec is resistant not only to network failures but also to 604 | \h{unexpected} calculator \h{exits or crashes}. 605 | \begin{block}{} 606 | \begin{CodeSmall}{\~{}/bin/pkg\_builder} 607 | #!/usr/bin/awk -f 608 | 609 | \{ 610 | "hostname -s" | getline hostname 611 | print "build " \$0 " on " hostname 612 | 613 | if (hostname == "server1" && \$0 == "textproc/gsed") 614 | \h{exit 139} 615 | # Damn it, I'm dying... 616 | # Take a note that exit status doesn't matter. 617 | else 618 | print "success" # Yes! :-) 619 | 620 | print "" # end-of-task marker 621 | fflush() # we must flush stdout 622 | \} 623 | \end{CodeSmall} 624 | \end{block} 625 | \end{frame} 626 | 627 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 628 | \begin{frame}[fragile] 629 | \frametitle{Example 3: dependency graph of tasks} 630 | ``textproc/gsed'' failed on ``server1'' but then succeeded on ``server2''. 631 | Every 300 seconds we try to reconnect to ``server1''. Keywords ``success'', 632 | ``failure'' and ``fatal'' may be changed with a help of -ms=, -mf= and 633 | -mF= options respectively. 634 | \begin{block}{} 635 | \begin{CodeSmall}{paexec -Z300 invocation (with failure)} 636 | \prompt{\$} paexec -gl -Z300 -t ssh -c ~/bin/pkg_builder \textbackslash 637 | -n 'server2 server1' < ~/tmp/packages_to_build \textbackslash 638 | | paexec_reorder > result 639 | \prompt{\$} cat result 640 | build audio/cd-discid on server2 641 | success 642 | \h{build textproc/gsed on server1} 643 | \h{fatal} 644 | \h{build textproc/gsed on server2} 645 | \h{success} 646 | build audio/id3 on server2 647 | success 648 | ... 649 | \prompt{\$} 650 | \end{CodeSmall} 651 | \end{block} 652 | \end{frame} 653 | 654 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 655 | \begin{frame}[fragile] 656 | \frametitle{Example 4: Converting .wav files to .flac or .ogg} 657 | In trivial cases we don't need relatively complex ``calculator''. 658 | Running a trivial command may be enough. 659 | Below we run three .wav to .flac/.ogg convertors in parallel. 660 | If \h{-x} is applied, task is passed to calculator as an argument. 661 | \begin{block}{} 662 | \begin{CodeSmall}{paexec -x invocation} 663 | \prompt{\$} ls -1 *.wav | paexec \h{-x} -c 'flac -s' -n+3 >/dev/null 664 | \prompt{\$} 665 | \end{CodeSmall} 666 | \end{block} 667 | \begin{block}{} 668 | \begin{CodeSmall}{paexec -x invocation} 669 | \prompt{\$} ls -1 *.wav | paexec -ixC -n+3 oggenc -Q | grep . 670 | 01-Crying_Wolf.wav 671 | 02-Autumn.wav 672 | 03-Time_Heals.wav 673 | 04-Alice_(Letting_Go).wav 674 | 05-This_Side_Of_The_Looking_Glass.wav 675 | ... 676 | \prompt{\$} 677 | \end{CodeSmall} 678 | \end{block} 679 | \end{frame} 680 | 681 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 682 | \begin{frame}[fragile] 683 | \frametitle{Example 5: paexec -W} 684 | If different tasks take different amount of time to process, than it 685 | makes sense to process ``heavier'' ones earlier in order to minimize 686 | total calculation time. For this to work one can weigh each tasks. 687 | Note that this mode enables ``graph'' mode automatically. 688 | \begin{block}{} 689 | \begin{CodeSmall}{\~{}/bin/calc} 690 | #!/bin/sh 691 | # \$1 -- task given on input 692 | if test \$1 = huge; then 693 | sleep 6 694 | else 695 | sleep 1 696 | fi 697 | 698 | echo "task \$1 done" 699 | \end{CodeSmall} 700 | \end{block} 701 | \end{frame} 702 | 703 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 704 | \begin{frame}[fragile] 705 | \frametitle{Example 5: paexec -W} 706 | This is how we run unweighted tasks. The whole process takes 8 seconds. 707 | \begin{block}{} 708 | \begin{CodeSmall}{paexec invocation} 709 | \prompt{\$} printf 'small1\textbackslash{nsmall2}\textbackslash{nsmall3}\textbackslash{nsmall4}\textbackslash{nsmall5}\textbackslash{nhuge}\textbackslash{n}' | 710 | time -p paexec -c \~{}/bin/calc -n +2 -xg | grep -v success 711 | task small2 done 712 | task small1 done 713 | task small3 done 714 | task small4 done 715 | task small5 done 716 | task huge done 717 | real 8.04 718 | user 0.03 719 | sys 0.03 720 | \prompt{\$} 721 | \end{CodeSmall} 722 | \end{block} 723 | \end{frame} 724 | 725 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 726 | \begin{frame}[fragile] 727 | \frametitle{Example 5: paexec -W} 728 | If we say paexec that the task ``huge'' is performed 6 times longer than others, 729 | it starts``huge'' first and then others. In total we spend 6 seconds for all tasks. 730 | \begin{block}{} 731 | \begin{CodeSmall}{paexec -W1 invocation} 732 | \prompt{\$} printf 'small1\textbackslash{nsmall2}\textbackslash{nsmall3}\textbackslash{nsmall4}\textbackslash{nweight: huge 6}\textbackslash{n}' | 733 | time -p paexec -c \~{}/bin/calc -n +2 -x \h{-W1} | grep -v success 734 | task small1 done 735 | task small2 done 736 | task small3 done 737 | task small4 done 738 | task small5 done 739 | task huge done 740 | real 6.02 741 | user 0.03 742 | sys 0.02 743 | \prompt{\$} 744 | \end{CodeSmall} 745 | \end{block} 746 | \end{frame} 747 | 748 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 749 | \begin{frame}[fragile] 750 | \frametitle{} 751 | For details see the manual page. 752 | \begin{center} 753 | \huge 754 | The End 755 | \end{center} 756 | \end{frame} 757 | 758 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 759 | 760 | \end{document} 761 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2007-2024 Aleksey Cheusov 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | MKC_CHECK_CUSTOM += sleep_fract 23 | MKC_CUSTOM_FN.sleep_fract = ../checks/sleep_fract 24 | 25 | CLEANFILES += _test.in _tasks.tmp _test.tmp 26 | 27 | USE_TEST_BUFSIZES ?= 1 2 3 4 5 6 7 8 9 10 11 12 13 14 1000 10000 28 | 29 | all: 30 | @echo 'running tests...'; \ 31 | set -e; \ 32 | cd ${.CURDIR}; \ 33 | export OBJDIR=${.OBJDIR}; \ 34 | export PATH=${.CURDIR}/broken_echo:$$PATH; \ 35 | export PATH=${.CURDIR}/scripts:$$PATH; \ 36 | export PATH=${.CURDIR}/../paexec:$$PATH; \ 37 | export PATH=${.CURDIR}/../examples/all_substr:$$PATH; \ 38 | export PATH=${.CURDIR}/../examples/cc_wrapper:$$PATH; \ 39 | export PATH=${.CURDIR}/../examples/cc_wrapper2:$$PATH; \ 40 | export PATH=${.CURDIR}/../examples/dirtest:$$PATH; \ 41 | export PATH=${.CURDIR}/../examples/divide:$$PATH; \ 42 | export PATH=${.CURDIR}/../examples/make_package:$$PATH; \ 43 | export PATH=${.CURDIR}/../examples/toupper:$$PATH; \ 44 | export PATH=${.CURDIR}/../examples/wav2flac:$$PATH; \ 45 | export PATH=${OBJDIR_paexec}:$$PATH; \ 46 | export PATH=${OBJDIR_paargs}:$$PATH; \ 47 | export PATH=${OBJDIR_scripts}:$$PATH; \ 48 | export PATH=${OBJDIR_transp_closed_stdin}:$$PATH; \ 49 | export PATH=${OBJDIR_all_substr}:$$PATH; \ 50 | export PATH=${OBJDIR_cc_wrapper}:$$PATH; \ 51 | export PATH=${OBJDIR_cc_wrapper2}:$$PATH; \ 52 | export PATH=${OBJDIR_dirtest}:$$PATH; \ 53 | export PATH=${OBJDIR_divide}:$$PATH; \ 54 | export PATH=${OBJDIR_make_package}:$$PATH; \ 55 | export PATH=${OBJDIR_toupper}:$$PATH; \ 56 | export PATH=${OBJDIR_wav2flac}:$$PATH; \ 57 | export PATH=${OBJDIR_delayed_output}:$$PATH; \ 58 | export SLEEP_FRACT=${CUSTOM.sleep_fract:S/0//}; \ 59 | export USE_TEST_BUFSIZES=${USE_TEST_BUFSIZES:Q}; \ 60 | if ./test.sh; \ 61 | then echo 'SUCCEEDED'; \ 62 | else echo 'FAILED'; false; \ 63 | fi 64 | 65 | .include 66 | -------------------------------------------------------------------------------- /tests/Makefile.inc: -------------------------------------------------------------------------------- 1 | .include "../Makefile.inc" 2 | -------------------------------------------------------------------------------- /tests/delayed_output/Makefile: -------------------------------------------------------------------------------- 1 | # Written by Aleksey Cheusov 2 | # Public domain 3 | 4 | PROG = delayed_output 5 | 6 | WARNS = 4 7 | 8 | .include 9 | -------------------------------------------------------------------------------- /tests/delayed_output/delayed_output.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Written by Aleksey Cheusov 3 | * Public domain 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | static void alarm_handler(int sig) 13 | { 14 | if (sig != SIGALRM) 15 | _exit(2); 16 | } 17 | 18 | int main(int argc, char **argv) 19 | { 20 | --argc; 21 | ++argv; 22 | assert(argc == 1); 23 | 24 | unsigned int remaining; 25 | signal(SIGALRM, alarm_handler); 26 | alarm(1); 27 | 28 | remaining = sleep(1000); 29 | assert(remaining > 0); 30 | 31 | printf("prefix "); 32 | puts(argv[0]); 33 | 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /tests/fakeflac/fake1.wav: -------------------------------------------------------------------------------- 1 | This is fake wav1 2 | -------------------------------------------------------------------------------- /tests/fakeflac/fake2.wav: -------------------------------------------------------------------------------- 1 | This is fake wav2 2 | -------------------------------------------------------------------------------- /tests/fakeflac/fake3.wav: -------------------------------------------------------------------------------- 1 | This is fake wav3 2 | -------------------------------------------------------------------------------- /tests/fakeflac/fake4.wav: -------------------------------------------------------------------------------- 1 | This is fake wav4 2 | -------------------------------------------------------------------------------- /tests/fakeflac/fake5.wav: -------------------------------------------------------------------------------- 1 | This is fake wav5 2 | -------------------------------------------------------------------------------- /tests/fakeflac/flac: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | set -e 7 | 8 | while test $# -ne 0; do 9 | case "$1" in 10 | --silent) 11 | ;; 12 | -o) 13 | dst_fn=$2 14 | shift;; 15 | *) 16 | break;; 17 | esac 18 | shift 19 | done 20 | 21 | test $# -eq 1 22 | 23 | src_fn="$1" 24 | : ${dst_fn:=${src_fn%%.wav}.flac} 25 | 26 | cat "$src_fn" > "$dst_fn" 27 | echo "converted to flac" >> "$dst_fn" 28 | -------------------------------------------------------------------------------- /tests/scripts/Makefile: -------------------------------------------------------------------------------- 1 | # Written by Aleksey Cheusov 2 | # Public domain 3 | 4 | INSCRIPTS = big_result_cmd transport_broken_rnd \ 5 | transport_broken_echo transport_broken_toupper 6 | SCRIPTS = paexec_notransport ${INSCRIPTS} 7 | 8 | .include 9 | -------------------------------------------------------------------------------- /tests/scripts/big_result_cmd.in: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env runawk 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | #env "LC_ALL=C" 7 | 8 | #use "power_getopt.awk" 9 | 10 | ############################################################ 11 | #.begin-str help 12 | # big_result_cmd -- test command 13 | # OPTIONS: 14 | # -h display this help 15 | # =l the number of lines to print, 16 | # =s print the specified status before EOT, 17 | # 18 | # In -g -Mm and -g -Mf modes, portions of the result followed 19 | # by "fatal" marker are automatically cut off. 20 | #.end-str 21 | ############################################################ 22 | 23 | BEGIN { 24 | eot = ENVIRON ["PAEXEC_EOT"] 25 | lines = getarg("l") + 0 26 | status = getarg("s") 27 | } 28 | 29 | { 30 | for (i=0; i < lines; ++i){ 31 | # works fine for empty input too 32 | print "line #" i, toupper($0) 33 | } 34 | 35 | if (status != "") 36 | print status 37 | 38 | print eot # end of task marker 39 | fflush() # end of task MUST BE FLUSHED!!! 40 | } 41 | -------------------------------------------------------------------------------- /tests/scripts/paexec_notransport: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | shift # skip node 7 | 8 | unset PAEXEC_EOT 9 | 10 | # unset variables used in regression tests 11 | unset ZZZZ 12 | unset YYYY 13 | unset CCCC 14 | 15 | # 16 | eval "$@" 17 | -------------------------------------------------------------------------------- /tests/scripts/transport_broken_echo.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | BEGIN { 7 | if (ARGC != 3){ 8 | print "usage: transport_broken_echo id ''" > "/dev/stderr" 9 | exit 77 10 | } 11 | 12 | id = ARGV [1] 13 | ARGV [1] = "-" 14 | 15 | ARGV [2] = "" 16 | 17 | # emulate totally broken NODE==4 18 | if (id == 4){ 19 | system("sleep 1") 20 | exit 3 21 | } 22 | } 23 | 24 | { 25 | print "I'll output " $1 26 | if ($1 == id && id >= 1){ 27 | # emulate broken TASK==NODE where NODE >= 1 28 | exit 1 29 | } 30 | 31 | print $1 # echoing input 32 | print "success" # This is a keyword! Not a random word ;-) 33 | 34 | print "" # end of task marker 35 | fflush() # end of task MUST BE FLUSHED!!! 36 | } 37 | -------------------------------------------------------------------------------- /tests/scripts/transport_broken_echo2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | # 7 | # usage: transport 8 | # 9 | 10 | fn=$1 11 | shift 12 | 13 | id=$1 14 | shift 15 | 16 | if test "$id" -eq 1 && ! test -f "$fn"; then 17 | # fails if there is not file and = 1 18 | touch "$fn" 19 | exit 12 20 | fi 21 | 22 | if test "$id" -eq 2; then 23 | # fails = 2 with 3 secs timeout 24 | read task 25 | sleep 3 26 | echo task 27 | exit 11 28 | fi 29 | 30 | # succeeds in all other cases 31 | while read task; do 32 | echo "output $task" 33 | echo success 34 | echo '' 35 | done 36 | -------------------------------------------------------------------------------- /tests/scripts/transport_broken_rnd.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | function sleep (secs){ 7 | if (!nosleep && 0 != system("sleep " secs)){ 8 | exit 10 9 | } 10 | } 11 | 12 | function my_rnd (){ 13 | if (rand() < threshold) 14 | return -1 # return unusual value for exiting with error 15 | else 16 | return rand() 17 | } 18 | 19 | BEGIN { 20 | threshold = ARGV [1] + 0.0 21 | fake_cmd = ARGV [2] 22 | 23 | nosleep = ARGV [1] ~ /ns/ 24 | nopostfail = ARGV [1] ~ /nopostfail/ 25 | 26 | ARGV [1] = "-" 27 | ARGV [2] = "" 28 | 29 | srand() 30 | 31 | # FreeBSD's /usr/bin/awk generates very bad first random value. 32 | # This is why we run rand() twice. 33 | rand() 34 | } 35 | 36 | { 37 | rnd = my_rnd() 38 | if (rnd < 0.0) { 39 | sleep(0.1) 40 | exit 1 41 | } 42 | 43 | sleep(rnd) 44 | 45 | print " " $0 46 | 47 | rnd = my_rnd() 48 | if (!nopostfail && rnd < 0.0) { 49 | sleep(0.1) 50 | exit 1 51 | } 52 | 53 | print "success" 54 | print "" 55 | fflush() 56 | } 57 | -------------------------------------------------------------------------------- /tests/scripts/transport_broken_toupper.in: -------------------------------------------------------------------------------- 1 | #!@awk@ -f 2 | 3 | # Written by Aleksey Cheusov 4 | # Public domain 5 | 6 | BEGIN { 7 | if (ARGC != 3){ 8 | print "usage: transport_broken_toupper id ''" > "/dev/stderr" 9 | exit 77 10 | } 11 | 12 | id = ARGV [1] 13 | ARGV [1] = "-" 14 | 15 | ARGV [2] = "" 16 | 17 | # emulate totally broken NODE==4 18 | if (id == 4){ 19 | system("sleep 1") 20 | exit 3 21 | } 22 | } 23 | 24 | { 25 | if ($1 == id && id >= 1){ 26 | # emulate broken TASK==NODE where NODE >= 1 27 | exit 1 28 | } 29 | 30 | print toupper($1) # uppering input 31 | 32 | print "" # end of task marker 33 | fflush() # end of task MUST BE FLUSHED!!! 34 | } 35 | -------------------------------------------------------------------------------- /tests/transp_closed_stdin/Makefile: -------------------------------------------------------------------------------- 1 | # Written by Aleksey Cheusov 2 | # Public domain 3 | 4 | PROG = transp_closed_stdin 5 | 6 | WARNS = 4 7 | 8 | .include 9 | -------------------------------------------------------------------------------- /tests/transp_closed_stdin/transp_closed_stdin.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Written by Aleksey Cheusov 3 | * Public domain 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | static const char *id; 12 | 13 | int main (int argc, char **argv) 14 | { 15 | char task [1000]; 16 | 17 | --argc, ++argv; 18 | 19 | id = argv [0]; 20 | 21 | while (fgets (task, sizeof (task), stdin)){ 22 | task [strlen (task)-1] = 0; 23 | 24 | printf ("I'll output %s\n%s\nsuccess\n\n", task, task); 25 | 26 | fflush (stdout); 27 | 28 | if (!strcmp (task, id)){ 29 | fclose (stdin); 30 | sleep (2); 31 | exit (1); 32 | } 33 | } 34 | 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /use.mk: -------------------------------------------------------------------------------- 1 | USE_VARIABLES += USE_BUFSIZE USE_COPTS USE_TEST_BUFSIZES 2 | USE_BUFSIZE.descr = "Internal buffer size for read(2) and write(2)" 3 | USE_BUFSIZE.0 = "unset: 4096" 4 | USE_BUFSIZE.1 = "value: other value" 5 | USE_COPTS.descr = "Optimization flags" 6 | USE_COPTS.0 = "unset: -O2 -g" 7 | USE_COPTS.1 = "value: other value" 8 | USE_TEST_BUFSIZES.descr = "Initial buffer sizes for testing" 9 | USE_TEST_BUFSIZES.0 = "unset: 1 2 3 ... 14 1000 10000" 10 | USE_TEST_BUFSIZES.1 = "value: other test sizes" 11 | --------------------------------------------------------------------------------