├── test-special.input
├── test-normal.input
├── ppss-wav2mp3.cfg
├── wav2mp3.sh
├── flac2mp3.sh
├── transcode2mp4.sh
├── ppss-test.sh
├── README.md
├── shunit2
└── ppss


/test-special.input:
--------------------------------------------------------------------------------
1 | \'file-!@#$%^&*()_ +=-0987654321~\'
2 | \'file-/\<>?:;'{}[]\'
3 | file-/\/\:\/!@#$%^&*()_+=-0987654321~
4 | file-42>424>424<2424>424?24<24>24
5 | file-/\<>?:;'{}[]
6 | http://www.google.nl
7 | ftp://storage.nl
8 | ./flac/Bééthoven Overtures CD2/01 - Beethoven, Lv - Leonore I - Op.138.flac
9 | 


--------------------------------------------------------------------------------
/test-normal.input:
--------------------------------------------------------------------------------
 1 | test-a
 2 | test-b
 3 | test-c
 4 | test-d
 5 | test-e
 6 | test-f
 7 | test-g
 8 | test-h
 9 | test-i
10 | test-j
11 | test-k
12 | test-l
13 | test-m
14 | test-n
15 | test-o
16 | test-p
17 | test-q
18 | test-r
19 | test-s
20 | test-t
21 | test-u
22 | test-v
23 | test-w
24 | test-x
25 | test-y
26 | test-z
27 | 


--------------------------------------------------------------------------------
/ppss-wav2mp3.cfg:
--------------------------------------------------------------------------------
 1 | REMOTE_OUTPUT_DIR=/mnt/mp3
 2 | SSH_KEY=ppss-key.dsa
 3 | SSH_KNOWN_HOSTS=known_hosts
 4 | SRC_DIR=/mnt/wav
 5 | COMMAND='./wav2mp3.sh "$ITEM" "$OUTPUT_DIR"'
 6 | NODES_FILE=nodes.txt
 7 | SSH_SERVER=10.0.1.110
 8 | USER=ppss
 9 | SCRIPT=wav2mp3.sh
10 | RANDOMIZE=1
11 | DOWNLOAD_TO_NODE=0
12 | UPLOAD_TO_SERVER=0
13 | SECURE_COPY=1
14 | PPSS_DEBUG=1
15 | 


--------------------------------------------------------------------------------
/wav2mp3.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | SRC="$1"
 4 | DEST="$2"
 5 | 
 6 |             
 7 | TYPE=`file -b "$SRC"`
 8 | RES=`echo "$TYPE" | grep "WAVE audio"`
 9 | if [ ! "$?" == "0" ]
10 | then
11 |     echo "File $SRC is not a wav file..."
12 |     echo "Type is $TYPE"
13 |     exit 0
14 | fi
15 | 
16 | BASENAME=`basename "$SRC"`
17 | MP3FILE="`echo ${BASENAME%wav}mp3`"
18 | lame --quiet --preset insane "$SRC" "$DEST/$MP3FILE"
19 | exit "$?"
20 | 


--------------------------------------------------------------------------------
/flac2mp3.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | INPUT="$1"
 4 | 
 5 | METATAGS="--export-tags-to="
 6 | LAMEOPTS=""
 7 | ERROR_STATUS="0"
 8 | 
 9 | function usage () {
10 | 
11 | 	echo 
12 | 	echo "Usage: $0 <flac file name>"
13 | 	echo
14 | 	exit 1
15 | }
16 | 
17 | function error () {
18 | 
19 | 	ERROR="$1"
20 | 	MSG="$2"
21 | 
22 | 	echo "Error: $MSG"
23 | 	exit 1
24 | }
25 | 
26 | 
27 | if [ -z "$INPUT" ]
28 | then
29 | 	usage
30 | fi
31 | 
32 | if [ ! -e "$INPUT" ]
33 | then
34 | 	echo "File $INPUT does not exist!"
35 | 	exit 1
36 | fi
37 | 
38 | FILETYPE="`file -b "$INPUT" | awk '{ print $1 }'`"
39 | if [ ! "$FILETYPE" == "FLAC" ]
40 | then
41 | 	echo "File $FILE is not a flac file..."
42 | 	exit 0
43 | fi
44 | 
45 | checkvar () {
46 | 
47 | 	VAR="$1"
48 | 
49 | 	if [ -z "$VAR" ] || [ "$VAR" == "" ] 
50 | 	then
51 | 		echo "Unknown"
52 | 	else
53 | 		echo "$VAR"
54 | 	fi
55 | }
56 | 
57 | 
58 | METATAGS="TITLE ARTIST ALBUM GENRE COMPOSER CONDUCTOR ENSEMBLE TRACKNUMBER DATE ALBUM ARTIST DISCNUMBER DISC"
59 | 
60 | function convert () {
61 | 
62 | 	FILE="$1"
63 | 	META="$FILE.meta"
64 | 	MP3FILE="`echo ${FILE%flac}mp3`"
65 | 	DIR="`dirname "$FILE"`"
66 | 
67 | 	metaflac --export-tags-to="$META" "$FILE"
68 | 
69 | 	ARTIST="`metaflac "$FILE" --show-tag=ARTIST | sed s/.*=//g`"
70 |     	TITLE="`metaflac "$FILE" --show-tag=TITLE | sed s/.*=//g`"
71 |     	ALBUM="`metaflac "$FILE" --show-tag=ALBUM | sed s/.*=//g`"
72 |     	GENRE="`metaflac "$FILE" --show-tag=GENRE | sed s/.*=//g`"
73 |     	TRACKNUMBER="`metaflac "$FILE" --show-tag=TRACKNUMBER | sed s/.*=//g`"
74 | 
75 | 	for x in $METATAGS
76 | 	do
77 | 		declare $x="`grep "$x" "$META" | cut -d "=" -f 2`"
78 | 
79 | 		VAR=$(eval echo " \$$x")
80 | 		VAR="`checkvar $VAR`"
81 | 	done
82 | 
83 | 
84 | 	flac -s -c -d "$FILE" | lame --tt "$TITLE" --tn "$TRACKNUMBER"  --tg "$GENRE"  --ty "$DATE"  --ta "$ARTIST" --tl "$ALBUM"  --ty "$YEAR"  --preset insane - "$MP3FILE"
85 | 	ERROR_STATUS="$?"
86 | 	if [ -e "$META" ]
87 | 	then
88 | 		rm "$META"
89 | 	fi
90 | }
91 | 
92 | convert "$INPUT"
93 | 
94 | exit "$ERROR_STATUS"
95 | 


--------------------------------------------------------------------------------
/transcode2mp4.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | INPUT="$1"
  4 | RESOLUTION="$3"
  5 | SEPARATE="$4"
  6 | TITLES=0
  7 | OPTS_HIGHRES="-e x264 -q 20.0 -r 29.97 --pfr  -a 1 -E faac -B 160 -6 dpl2 -R Auto -D 0.0 -f mp4 -4 -X 1024 --strict-anamorphic -m"
  8 | OPTS_LOWRES="-e x264 -q 20.0 -a 1 -E faac -B 128 -6 dpl2 -R 48 -D 0.0 -f mp4 -X 480 -m -x cabac=0:ref=2:me=umh:bframes=0:subme=6:8x8dct=0:trellis=0"
  9 | OPTS_SOURCE="-e x264  -q 20.0 -a 1,1 -E faac,ac3 -l 576 -B 160,160 -6 dpl2,auto -R Auto,Auto -D 0.0,0.0 -f mp4 --detelecine --decomb --strict-anamorphic -m -x b-adapt=2:rc-lookahead=50"
 10 | MODE=""
 11 | HANDBRAKE=HandBrakeCLI
 12 | DIRNAME=`dirname "$INPUT"`
 13 | BASENAME=`basename "$INPUT"`
 14 | OPTS=""
 15 | OUTPUT_DIR="$2"
 16 | OUTPUT_FILE_NAME=""
 17 | 
 18 | if [ -z "$INPUT" ] 
 19 | then
 20 | 	echo "usage $0 <input file / folder> <output folder> <highres|lowres> <separate>"	
 21 | 	echo
 22 | 	echo "Input either file, VIDEO_TS directory or .ISO"
 23 | 	echo 
 24 | 	echo -e "highres:\t1024 x 576"
 25 | 	echo -e "lowres:\t\t480 x 320"
 26 | 	echo -e "source:\t\tsame as source."
 27 | 	echo 
 28 | 	echo -e "separate:\tseparate files for episodes of a serie."
 29 | 	exit 1
 30 | fi
 31 | 
 32 | if [ ! -z "$OUTPUT_DIR" ]
 33 | then
 34 | 	if [ ! -e "$OUTPUT_DIR" ] || [ ! -d "$OUTPUT_DIR" ]
 35 | 	then
 36 | 		echo "Output directory does not exist or is not a directory."
 37 | 		exit 1
 38 | 	fi
 39 | else
 40 | 	echo "Output to current directory."
 41 | 	OUTPUT_DIR="."
 42 | fi
 43 | 
 44 | 
 45 | if [ ! -e "$INPUT" ]
 46 | then
 47 | 	echo "$INPUT does not exist!"
 48 | 	exit 1
 49 | fi
 50 | 
 51 | if [ -d "$INPUT" ] 
 52 | then
 53 | 	MODE=DIR
 54 | else
 55 | 	MODE=FILE
 56 | fi
 57 | 
 58 | echo "Input type is $MODE"
 59 | 
 60 | case "$RESOLUTION" in 
 61 | 	highres|HIGHRES	) 
 62 | 			OPTS="$OPTS_HIGHRES" ;;
 63 | 	lowres|LOWRES  	)
 64 | 			OPTS="$OPTS_LOWRES" ;;
 65 | 	source|SOURCE	)
 66 | 			OPTS="$OPTS_SOURCE" ;;
 67 | 			*) 	
 68 | 			echo "Resolution must be 'highres', 'source' or 'lowres'." 
 69 | 			exit 1 
 70 | 			;;
 71 | esac
 72 | 
 73 | function titles () {
 74 | 
 75 | 	TITLES=`./$HANDBRAKE -t 0 -i "$INPUT" 2>&1 | grep "+ title" | awk '{ print $3 }'  | sed s/://g`
 76 | 	echo $TITLES
 77 | }
 78 | 
 79 | if [ "$MODE" = "FILE" ]
 80 | then
 81 | 	mkdir -p "$OUTPUT_DIR/$DIRNAME" 
 82 | 	OUTPUT_FILE_NAME="$OUTPUT_DIR/$DIRNAME/${BASENAME%.*}"
 83 | elif [ "$MODE" = "DIR" ]
 84 | then
 85 | 	echo "$INPUT" | grep -i video_ts >> /dev/null 2>&1
 86 | 	if [ "$?" = "0" ]
 87 | 	then
 88 | 		INTERMEDIATE2=`basename "$DIRNAME"`
 89 | 		mkdir -p "$OUTPUT_DIR/$DIRNAME"
 90 | 		OUTPUT_FILE_NAME="$OUTPUT_DIR/$DIRNAME/$INTERMEDIATE2"
 91 | 	else
 92 | 		INTERMEDIATE2="$BASENAME"
 93 | 		mkdir -p "$OUTPUT_DIR/$DIRNAME/$INTERMEDIATE2"
 94 | 		OUTPUT_FILE_NAME="$OUTPUT_DIR/$DIRNAME/$INTERMEDIATE2/$INTERMEDIATE2"
 95 | 	fi
 96 | 	echo "INTERMEDIATE2 = $INTERMEDIATE2"
 97 | else
 98 | 	echo "Mode is not determined..."
 99 | 	exit 1
100 | fi
101 | 
102 | if [ "$SEPARATE" = "separate" ]
103 | then
104 | 	TITLES=`titles $INPUT`
105 | 	echo "TITLES = $TITLES"
106 | 	ERROR=0
107 | 
108 | 	for x in $TITLES
109 | 	do
110 | 		HandBrakeCLI $OPTS -i "$INPUT" -o "$OUTPUT_FILE_NAME-$x.mp4" 
111 | 		if [ ! "$?" = "0" ]
112 | 		then 
113 | 			ERROR="1"
114 | 		fi 
115 | 	done
116 | 	exit "$ERROR"
117 | else
118 | 	echo "Creating a single file."
119 | 	HandBrakeCLI $OPTS -i "$INPUT" -o "$OUTPUT_FILE_NAME.mp4" 
120 | fi
121 | 
122 | 
123 | 
124 | 
125 | 


--------------------------------------------------------------------------------
/ppss-test.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | DEBUG="$1"
  4 | VERSION="2.98"
  5 | TMP_DIR="/tmp/ppss"
  6 | PPSS=./ppss
  7 | PPSS_DIR=ppss_dir
  8 | export PPSS_DEBUG=1
  9 | HOST_ARCH=`uname`
 10 | SPECIAL_DIR=$TMP_DIR/root/special
 11 | . "$PPSS"
 12 | 
 13 | cleanup () {
 14 | 
 15 |         unset RES1
 16 |         unset RES2
 17 |         GLOBAL_COUNTER=1
 18 |     if [ ! "$DEBUG" = "debug" ]
 19 |     then
 20 |         for x in $REMOVEFILES
 21 |         do
 22 |             if [ -e ./$x ]
 23 |             then
 24 |                 rm -r ./$x
 25 |             fi
 26 |         done
 27 |     fi
 28 | 
 29 |        if [ ! -z "$TMP_DIR" ] && [ -e "$TMP_DIR" ]
 30 |        then
 31 |            rm -rf "$TMP_DIR"   
 32 |        fi
 33 | }
 34 | 
 35 | parseJobStatus () {
 36 | 
 37 |     TMP_FILE="$1"
 38 | 
 39 |     RES=`grep "Status:" "$JOBLOG/$TMP_FILE"`
 40 |     STATUS=`echo "$RES" | awk '{ print $2 }'`
 41 |     echo "$STATUS"
 42 | }
 43 | 
 44 | get_item_count_of_input_file () {
 45 | 
 46 |     if [ -e "$PPSS_DIR/INPUT_FILE-$$" ]
 47 |     then
 48 |         CONTENTS_OF_INPUTFILE=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
 49 |         echo "$CONTENTS_OF_INPUTFILE"
 50 |     else
 51 |         echo "Error, file $PPSS_DIR/INPUT_FILE-$$ does not exist."
 52 |     fi
 53 | }
 54 | 
 55 | oneTimeSetUp () {
 56 | 
 57 | 	JOBLOG=./$PPSS_DIR/job_log
 58 | 	INPUTFILENORMAL=test-normal.input
 59 |     INPUTFILESPECIAL_DIR=test-special.input
 60 |     LOCALOUTPUT=ppss_dir/PPSS_LOCAL_OUTPUT
 61 | 	REMOVEFILES="$PPSS_DIR test-ppss-*"
 62 | 
 63 |     if [ ! -e "$TMP_DIR" ]
 64 |     then
 65 |         mkdir -p "$TMP_DIR"
 66 |     fi
 67 | 
 68 |     cleanup
 69 | }
 70 | 
 71 | testVersion () {
 72 | 
 73 |     assertEquals "Version mismatch!" "$VERSION" "$SCRIPT_VERSION"
 74 | }
 75 | 
 76 | rename-ppss-dir () {
 77 | 
 78 | 	TEST="$1"
 79 | 
 80 | 	if [ -e "$PPSS_DIR" ] && [ -d "$PPSS_DIR" ] && [ ! -z "$TEST" ]
 81 | 	then
 82 | 		mv "$PPSS_DIR" test-ppss-"$TEST"
 83 | 	fi
 84 | }
 85 | 
 86 | oneTimeTearDown () {
 87 | 
 88 | 	if [ ! "$DEBUG" == "debug" ]
 89 | 	then
 90 |         cleanup 
 91 |     fi
 92 | }
 93 | 
 94 | createDirectoryWithSomeFiles () {
 95 | 
 96 |     ROOT_DIR=$TMP_DIR/root
 97 |     CHILD_1=$ROOT_DIR/child_1
 98 |     CHILD_2=$ROOT_DIR/child_2
 99 |     
100 |     if [ ! -e "$ROOT_DIR" ]
101 |     then
102 |         mkdir -p "$ROOT_DIR" 
103 |     fi
104 | 
105 |     if [ ! -e "$CHILD_1" ]
106 |     then
107 |         mkdir -p "$CHILD_1"
108 |     fi
109 | 
110 |     if [ ! -e "$CHILD_2" ]
111 |     then
112 |         mkdir -p "$CHILD_2"
113 |     fi
114 | 
115 |     for x in {1..10}
116 |     do
117 |         touch "$ROOT_DIR/file-$x" 
118 |         touch "$CHILD_1/file-$x"
119 |         touch "$CHILD_2/file-$x"
120 |     done
121 | 
122 |     ln -s /etc/resolve.conf "$ROOT_DIR" 2> /dev/null
123 |     ln -s /etc/hosts "$ROOT_DIR" 2> /dev/null
124 | }
125 | 
126 | createSpecialFilenames () {
127 | 
128 |     ERROR=0
129 |     mkdir -p "$SPECIAL_DIR"
130 | 
131 |     touch "$SPECIAL_DIR/a file with spaces" 
132 |     touch "$SPECIAL_DIR/a\\'file\\'with\\'quotes"
133 |     touch "$SPECIAL_DIR/a{file}with{curly}brackets}"
134 |     touch "$SPECIAL_DIR/a(file)with(parenthesis)"
135 |     touch "$SPECIAL_DIR/a\\file\\with\\backslashes"
136 |     touch "$SPECIAL_DIR/a!file!with!exclamationmarks"
137 |     touch "$SPECIAL_DIR/a filé with special characters"
138 |     touch "$SPECIAL_DIR/a\"file\"with\"double\"quotes"
139 | }
140 | 
141 | testMD5 () {
142 | 
143 |     export USE_MD5=1
144 |     init_vars > /dev/null 2>&1
145 |     ARCH=Darwin
146 |     set_md5
147 |     assertEquals "MD5 executable not set properly - $MD5" "$MD5" "md5" 
148 |     ARCH=Linux
149 |     set_md5
150 |     assertEquals "MD5 executable not set properly - $MD5" "$MD5" "md5sum" 
151 |     ARCH=$HOST_ARCH
152 | }
153 | 
154 | init_get_all_items () {
155 | 
156 |     DIR="$1"
157 |     TRAVERSAL="$2"
158 |     createDirectoryWithSomeFiles
159 |     create_working_directory
160 |     export SRC_DIR=$DIR
161 |     init_vars > /dev/null 2>&1
162 |     get_all_items
163 | }
164 | 
165 | testRecursion () {
166 | 
167 |     init_get_all_items $TMP_DIR/root 1
168 |     RESULT=`get_item_count_of_input_file`
169 |     EXPECTED=32
170 |     assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
171 | 
172 |     rename-ppss-dir $FUNCNAME
173 | }
174 | 
175 | testNoRecursion () {
176 | 
177 |     init_get_all_items $TMP_DIR/root 0    
178 |     RESULT=`get_item_count_of_input_file`
179 |     EXPECTED=12
180 | 
181 |     assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
182 | 
183 |     rename-ppss-dir $FUNCNAME
184 | }
185 | 
186 | testGetItem () {
187 | 
188 |     createSpecialFilenames
189 |     init_get_all_items $TMP_DIR/root 1
190 |     get_item
191 |     if [ -z "$ITEM" ]
192 |     then
193 |         ERROR=1
194 |     else 
195 |         ERROR=0
196 |     fi
197 |     EXPECTED=0
198 |     assertEquals "Get item failed." "$EXPECTED" "$ERROR"
199 | 
200 |     i=1
201 |     ERROR=0
202 |     while get_item
203 |     do
204 |         ((i++))
205 |     done
206 |     EXPECTED=40
207 |     assertEquals "Got wrong number of items." "$EXPECTED" "$i"
208 | 
209 |     rename-ppss-dir $FUNCNAME
210 |     cleanup
211 | }
212 | 
213 | return_all_items () {
214 | 
215 |     while get_item
216 |     do
217 |         ALL_ITEMS="$ALL_ITEMS$ITEM"$'\n'
218 |     done
219 |     echo "$ALL_ITEMS"
220 | }
221 | 
222 | testNumberOfItems () {
223 | 
224 |     createSpecialFilenames
225 |     RESULT=`init_get_all_items $TMP_DIR/root 1`
226 | 
227 |     RES1=`find $TMP_DIR/root/ ! -type d`
228 | 
229 |     RES2=`return_all_items`
230 | 
231 |     echo "$RES1" > a
232 |     echo "$RES2" > b
233 | 
234 |     assertEquals "Input file and actual files not the same!" "$RES1" "$RES2"
235 |     rename-ppss-dir $FUNCNAME
236 | }
237 | 
238 | 
239 | testInvalidProcessingOfitemVariable() {
240 | 
241 |     createSpecialFilenames
242 |     init_get_all_items $TMP_DIR/root 1
243 |     COMMAND='echo $ITEM'
244 |     while get_item
245 |     do
246 |         commando "$ITEM"
247 |     done
248 |     RESULT=$(grep '$ITEM' $PPSS_DIR/job_log/*)
249 |     EXPECTED=""
250 |     assertEquals "Got incorrect processing of ITEM variable." "$EXPECTED" "$RESULT"
251 |     rename-ppss-dir $FUNCNAME
252 | }
253 | 
254 | 
255 | testNumberOfLogfiles () {
256 | 
257 |     createSpecialFilenames
258 |     init_get_all_items $TMP_DIR/root 1
259 |     COMMAND='echo hoi' 
260 |     while get_item
261 |     do
262 |         commando "$ITEM"
263 |     done
264 |     RESULT=`ls -1 $PPSS_DIR/job_log/ | wc -l | awk '{ print $1}'`
265 |     EXPECTED=40
266 |     assertEquals "Got wrong number of log files." "$EXPECTED" "$RESULT"
267 |     rename-ppss-dir $FUNCNAME
268 | }
269 | 
270 | testUserInputFile () {
271 | 
272 |     cleanup
273 |     INPUT_FILE=test-special.input
274 |     create_working_directory
275 |     init_vars > /dev/null 2>&1
276 |     get_all_items    
277 |     RESULT=`return_all_items`
278 |     ORIGINAL=`cat $INPUT_FILE`
279 |     assertEquals "User input processing not ok." "$RESULT" "$ORIGINAL"
280 |     rename-ppss-dir $FUNCNAME
281 | }
282 | 
283 | . ./shunit2
284 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ### |P|P|S|S| - (Distributed) Parallel Processing Shell Script
  2 | 
  3 |     I've moved this project from Google code to Github including its wiki pages.
  4 |     
  5 |     >>> THIS PROJECT IS NO LONGER UNDER ANY DEVELOPMENT OR MAINTENANCE <<<
  6 | 
  7 | PPSS is a Bash shell script that executes commands, scripts or programs in parallel. It is designed to make full use of current multi-core CPUs. It will detect the number of available CPUs and start a separate job for each CPU core. It will also use hyper threading by default. 
  8 | 
  9 | PPSS can be run on multiple hosts, processing a single group of items, like a cluster. 
 10 | 
 11 | PPSS provides you with examples that will make it obvious how it is used:
 12 | 
 13 |     bash-3.2$ ppss
 14 |     |P|P|S|S| Distributed Parallel Processing Shell Script 2.60
 15 | 
 16 |     usage: ./ppss [ -d <sourcedir> | -f <sourcefile> ]  [ -c '<command> "$ITEM"' ]
 17 |                      [ -C <configfile> ] [ -j ] [ -l <logfile> ] [ -p <# jobs> ]
 18 |                      [ -D <delay> ] [ -h ] [ --help ] [ -r ] 
 19 |     
 20 |     Examples:
 21 |                      ./ppss -d /dir/with/some/files -c 'gzip '
 22 |                      ./ppss -d /dir/with/some/files -c 'cp "$ITEM" /tmp' -p 2
 23 |                      ./ppss -f <file> -c 'wget -q -P /destination/directory "$ITEM"' -p 10
 24 | 
 25 | 
 26 | 
 27 | Basically, just provide PPSS with a source of items (a directory with files, for example) and a command that must be applied to these items.
 28 | 
 29 | For a quick demonstration of it's standalone usage, see the video below.
 30 | 
 31 | <wiki:video width=600px url="http://www.youtube.com/watch?v=32PwsARbePw"/>
 32 | 
 33 | A bit more advanced (better quality): 
 34 | 
 35 | <wiki:video width=600px url="http://www.youtube.com/watch?v=AdwZlW1eZ6A"/>
 36 | 
 37 | PPSS will take a list of items as input. Items can be files within a directory or entries in a text file. PPSS 
 38 | executes a user-specified command for each item in this list. The item is supplied as an argument to this command. At any point in time, there are never more items processed in parallel as there are cores available.
 39 | 
 40 | An example how this script is used:
 41 | 
 42 | 
 43 |     user@host:~/ppss$ ./ppss.sh -d /wavs -c './encode.sh ' 
 44 |     Mar 30 23:21:10: INFO  =========================================================
 45 |     Mar 30 23:21:10: INFO                         |P|P|S|S|                         
 46 |     Mar 30 23:21:10: INFO  Distributed Parallel Processing Shell Script version 2.18
 47 |     Mar 30 23:21:10: INFO  =========================================================
 48 |     Mar 30 23:21:10: INFO  Hostname:	Core i7
 49 |     Mar 30 23:21:10: INFO  ---------------------------------------------------------
 50 |     Mar 30 23:21:10: INFO  Found 8 logic processors.
 51 |     Mar 30 23:21:10: INFO  CPU: Intel(R) Core(TM) i7 CPU         920  @ 2.67GHz
 52 |     Mar 30 23:21:10: INFO  Starting 8 workers.
 53 |     Mar 30 23:21:10: INFO  ---------------------------------------------------------
 54 |     Mar 30 23:21:17: INFO  Currently 76 percent complete. Processed 172 of 226 items.
 55 | 
 56 | 
 57 | In this example, the script detects that four CPU-cores are available. Hyper-threading is used as the core i7 920 supports it, so 8 workers are started. Don't miss the trailing space within the command section. 
 58 | 
 59 | #### Logging
 60 | 
 61 | One of the nice features of PPSS is logging. The output of every command on every item that is executed is logged into a single file. Below is an example of such a file:
 62 | 
 63 |     ===== PPSS Item Log File =====
 64 |     Host:		imac-2.local
 65 |     Item:		PPSS_LOCAL_TMPDIR/20080602.wav
 66 |     Start date:	Mar 03 00:10:32
 67 |     
 68 |     Encode of PPSS_LOCAL_TMPDIR/20080602.wav successful.
 69 |     
 70 |     Status:		Succes - item has been processed.
 71 |     Elapsed time (h:m:s): 0:4:48
 72 | 
 73 | 
 74 | As you can see, a lot of information is logged by PPSS about the processed item, including the time it took to process it. Of particular interest is the status line: it is based on the exit status of the executed command, so error detection is build-in.
 75 | 
 76 | This script is build with the goal to be very easy to use. It runs on Linux and Mac OS X. It should work on other Unix-like operating systems, such as Solaris, that support the Bash shell.
 77 | 
 78 | This script is (only) useful for jobs that can be easily broken down in separate tasks that can be executed in parallel. For example, encoding a bunch of wav-files to mp3-format, downloading a large number of files, resizing images, anything you can think of.
 79 | 
 80 | Please note that this script is _even useful on a single-core host_. Certain jobs, such as downloading files and processing these downloaded files can often be optimized by executing these processes in parallel. 
 81 | 
 82 | *_PPSS is always a work in progress and although it seems to work for me, it might not for you for reasons I'm currently not aware of. I would very much appreciate it if you try it out and create an issue if you find a bug. Thanks!_*
 83 | 
 84 | #### Distributed PPSS
 85 | 
 86 | From version 2.0 and onward, PPSS supports distributed computing. With this version, it is possible to run PPSS on multiple host that each process a part of the same queue of items. Nodes communicate with each other through a single SSH server. 
 87 | 
 88 | This script has already been used to convert 400 GB of WAV files to MP3 with 4 hosts, a Core i7 running Ubuntu, two Macs based on 1.8 and 2 ghz Core Duos running Leopard, and an 2,2 Ghz AMD system running Debian. 
 89 | 
 90 | The remarkable thing is that the Core 7i @ 3,6 Ghz processed 380 files, while the other three systems _combined_ only processed 199. Keep in mind that the Core 7i has only 4 physical cores...
 91 | 
 92 | http://chart.apis.google.com/chart?cht=p3&chd=t:66,11,11,12&chs=350x150&chl=Core%20i7%20|AMD|iMac|Mac%20Mini&noncense=test.png
 93 | 
 94 | It is difficult to give an impression how PPSS works in distributed mode, however maybe the status screen can give you an idea.
 95 | 
 96 | 
 97 |     mrt 29 22:18:27: INFO  =========================================================
 98 |     mrt 29 22:18:27: INFO                         |P|P|S|S|                         
 99 |     mrt 29 22:18:27: INFO  Distributed Parallel Processing Shell Script version 2.17
100 |     mrt 29 22:18:27: INFO  =========================================================
101 |     mrt 29 22:18:27: INFO  Hostname:	MacBoek.local
102 |     mrt 29 22:18:27: INFO  ---------------------------------------------------------
103 |     mrt 29 22:18:28: INFO  Status:		100 percent complete.
104 |     mrt 29 22:18:28: INFO  Nodes:	        7
105 |     mrt 29 22:18:28: INFO  ---------------------------------------------------------
106 |     mrt 29 22:18:28: INFO  IP-address       Hostname            Processed     Status
107 |     mrt 29 22:18:28: INFO  ---------------------------------------------------------
108 |     mrt 29 22:18:28: INFO  192.168.0.4      Corei7                    155   FINISHED
109 |     mrt 29 22:18:29: INFO  192.168.0.2      MINI.local                 34   FINISHED
110 |     mrt 29 22:18:29: INFO  192.168.0.5      server                     29   FINISHED
111 |     mrt 29 22:18:30: INFO  192.168.0.63     host3                       6   FINISHED
112 |     mrt 29 22:18:31: INFO  192.168.0.64     host4                       6   FINISHED
113 |     mrt 29 22:18:31: INFO  192.168.0.20     imac-2.local               34   FINISHED
114 |     mrt 29 22:18:32: INFO  192.168.0.1      router                      7   FINISHED
115 |     mrt 29 22:18:32: INFO  ---------------------------------------------------------
116 |     mrt 29 22:18:32: INFO  Total processed:                           271
117 | 
118 | 


--------------------------------------------------------------------------------
/shunit2:
--------------------------------------------------------------------------------
   1 | # $Id: shunit2 277 2008-10-29 21:20:22Z kate.ward@forestent.com $
   2 | # vim:et:ft=sh:sts=2:sw=2
   3 | # vim:foldmethod=marker:foldmarker=/**,*/
   4 | #
   5 | #/**
   6 | # <?xml version="1.0" encoding="UTF-8"?>
   7 | # <s:shelldoc xmlns:s="http://www.forestent.com/projects/shelldoc/xsl/2005.0">
   8 | # <s:header>
   9 | # shUnit 2.1.5
  10 | # Shell Unit Test Framework
  11 | #
  12 | # http://shunit2.googlecode.com/
  13 | #
  14 | # written by Kate Ward &lt;kate.ward@forestent.com&gt;
  15 | # released under the LGPL
  16 | #
  17 | # This module implements a xUnit based unit test framework similar to JUnit.
  18 | # </s:header>
  19 | #*/
  20 | 
  21 | SHUNIT_VERSION='2.1.5'
  22 | 
  23 | SHUNIT_TRUE=0
  24 | SHUNIT_FALSE=1
  25 | SHUNIT_ERROR=2
  26 | 
  27 | _shunit_warn() { echo "shunit2:WARN $@" >&2; }
  28 | _shunit_error() { echo "shunit2:ERROR $@" >&2; }
  29 | _shunit_fatal() { echo "shunit2:FATAL $@" >&2; }
  30 | 
  31 | # specific shell checks
  32 | if [ -n "${ZSH_VERSION:-}" ]; then
  33 |   setopt |grep "^shwordsplit$" >/dev/null
  34 |   if [ $? -ne ${SHUNIT_TRUE} ]; then
  35 |     _shunit_fatal 'zsh shwordsplit option is required for proper operation'
  36 |     exit ${SHUNIT_ERROR}
  37 |   fi
  38 |   if [ -z "${SHUNIT_PARENT:-}" ]; then
  39 |     _shunit_fatal "zsh does not pass \$0 through properly. please declare \
  40 | \"SHUNIT_PARENT=\$0\" before calling shUnit2"
  41 |     exit ${SHUNIT_ERROR}
  42 |   fi
  43 | fi
  44 | 
  45 | #
  46 | # constants
  47 | #
  48 | 
  49 | __SHUNIT_ASSERT_MSG_PREFIX='ASSERT:'
  50 | __SHUNIT_PARENT=${SHUNIT_PARENT:-$0}
  51 | 
  52 | # set the constants readonly
  53 | shunit_constants_=`set |grep '^__SHUNIT_' |cut -d= -f1`
  54 | echo "${shunit_constants_}" |grep '^Binary file' >/dev/null \
  55 |     && shunit_constants_=`set |grep -a '^__SHUNIT_' |cut -d= -f1`
  56 | for shunit_constant_ in ${shunit_constants_}; do
  57 |   shunit_ro_opts_=''
  58 |   case ${ZSH_VERSION:-} in
  59 |     '') ;;  # this isn't zsh
  60 |     [123].*) ;;  # early versions (1.x, 2.x, 3.x)
  61 |     *) shunit_ro_opts_='-g' ;;  # all later versions. declare readonly globally
  62 |   esac
  63 |   readonly ${shunit_ro_opts_} ${shunit_constant_}
  64 | done
  65 | unset shunit_constant_ shunit_constants_ shunit_ro_opts_
  66 | 
  67 | # variables
  68 | __shunit_skip=${SHUNIT_FALSE}
  69 | __shunit_suite=''
  70 | 
  71 | # counts of tests
  72 | __shunit_testSuccess=${SHUNIT_TRUE}
  73 | __shunit_testsTotal=0
  74 | __shunit_testsPassed=0
  75 | __shunit_testsFailed=0
  76 | 
  77 | # counts of asserts
  78 | __shunit_assertsTotal=0
  79 | __shunit_assertsPassed=0
  80 | __shunit_assertsFailed=0
  81 | __shunit_assertsSkipped=0
  82 | 
  83 | __shunit_lineno=''
  84 | __shunit_reportGenerated=${SHUNIT_FALSE}
  85 | 
  86 | # macros
  87 | _SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi'
  88 | 
  89 | #-----------------------------------------------------------------------------
  90 | # assert functions
  91 | #
  92 | 
  93 | #/**
  94 | # <s:function group="asserts">
  95 | # <entry align="right">
  96 | #   <emphasis>void</emphasis>
  97 | # </entry>
  98 | # <entry>
  99 | #   <funcsynopsis>
 100 | #     <funcprototype>
 101 | #       <funcdef><function>assertEquals</function></funcdef>
 102 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 103 | #       <paramdef>string <parameter>expected</parameter></paramdef>
 104 | #       <paramdef>string <parameter>actual</parameter></paramdef>
 105 | #     </funcprototype>
 106 | #   </funcsynopsis>
 107 | #   <para>Asserts that <emphasis>expected</emphasis> and
 108 | #   <emphasis>actual</emphasis> are equal to one another. The message is
 109 | #   optional.</para>
 110 | # </entry>
 111 | # </s:function>
 112 | #*/
 113 | assertEquals()
 114 | {
 115 |   ${_SHUNIT_LINENO_}
 116 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 117 |     _shunit_error "assertEquals() requires two or three arguments; $# given"
 118 |     _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}"
 119 |     return ${SHUNIT_ERROR}
 120 |   fi
 121 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 122 | 
 123 |   shunit_message_=${__shunit_lineno}
 124 |   if [ $# -eq 3 ]; then
 125 |     shunit_message_="${shunit_message_}$1"
 126 |     shift
 127 |   fi
 128 |   shunit_expected_=$1
 129 |   shunit_actual_=$2
 130 | 
 131 |   shunit_return=${SHUNIT_TRUE}
 132 |   if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then
 133 |     _shunit_assertPass
 134 |   else
 135 |     failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}"
 136 |     shunit_return=${SHUNIT_FALSE}
 137 |   fi
 138 | 
 139 |   unset shunit_message_ shunit_expected_ shunit_actual_
 140 |   return ${shunit_return}
 141 | }
 142 | _ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"'
 143 | 
 144 | #/**
 145 | # <s:function group="asserts">
 146 | # <entry align="right">
 147 | #   <emphasis>void</emphasis>
 148 | # </entry>
 149 | # <entry>
 150 | #   <funcsynopsis>
 151 | #     <funcprototype>
 152 | #       <funcdef><function>assertNotEquals</function></funcdef>
 153 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 154 | #       <paramdef>string <parameter>unexpected</parameter></paramdef>
 155 | #       <paramdef>string <parameter>actual</parameter></paramdef>
 156 | #     </funcprototype>
 157 | #   </funcsynopsis>
 158 | #   <para>Asserts that <emphasis>unexpected</emphasis> and
 159 | #   <emphasis>actual</emphasis> are <emphasis role="strong">not</emphasis>
 160 | #   equal to one another. The message is optional.</para>
 161 | # </entry>
 162 | # </s:function>
 163 | #*/
 164 | assertNotEquals()
 165 | {
 166 |   ${_SHUNIT_LINENO_}
 167 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 168 |     _shunit_error "assertNotEquals() requires two or three arguments; $# given"
 169 |     return ${SHUNIT_ERROR}
 170 |   fi
 171 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 172 | 
 173 |   shunit_message_=${__shunit_lineno}
 174 |   if [ $# -eq 3 ]; then
 175 |     shunit_message_="${shunit_message_}$1"
 176 |     shift
 177 |   fi
 178 |   shunit_unexpected_=$1
 179 |   shunit_actual_=$2
 180 | 
 181 |   shunit_return=${SHUNIT_TRUE}
 182 |   if [ "${shunit_unexpected_}" != "${shunit_actual_}" ]; then
 183 |     _shunit_assertPass
 184 |   else
 185 |     failSame "${shunit_message_}" "$@"
 186 |     shunit_return=${SHUNIT_FALSE}
 187 |   fi
 188 | 
 189 |   unset shunit_message_ shunit_unexpected_ shunit_actual_
 190 |   return ${shunit_return}
 191 | }
 192 | _ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"'
 193 | 
 194 | #/**
 195 | # <s:function group="asserts">
 196 | # <entry align="right">
 197 | #   <emphasis>void</emphasis>
 198 | # </entry>
 199 | # <entry>
 200 | #   <funcsynopsis>
 201 | #     <funcprototype>
 202 | #       <funcdef><function>assertNull</function></funcdef>
 203 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 204 | #       <paramdef>string <parameter>value</parameter></paramdef>
 205 | #     </funcprototype>
 206 | #   </funcsynopsis>
 207 | #   <para>Asserts that <emphasis>value</emphasis> is <literal>null</literal>,
 208 | #   or in shell terms a zero-length string. The message is optional.</para>
 209 | # </entry>
 210 | # </s:function>
 211 | #*/
 212 | assertNull()
 213 | {
 214 |   ${_SHUNIT_LINENO_}
 215 |   if [ $# -lt 1 -o $# -gt 2 ]; then
 216 |     _shunit_error "assertNull() requires one or two arguments; $# given"
 217 |     return ${SHUNIT_ERROR}
 218 |   fi
 219 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 220 | 
 221 |   shunit_message_=${__shunit_lineno}
 222 |   if [ $# -eq 2 ]; then
 223 |     shunit_message_="${shunit_message_}$1"
 224 |     shift
 225 |   fi
 226 |   assertTrue "${shunit_message_}" "[ -z '$1' ]"
 227 |   shunit_return=$?
 228 | 
 229 |   unset shunit_message_
 230 |   return ${shunit_return}
 231 | }
 232 | _ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"'
 233 | 
 234 | #/**
 235 | # <s:function group="asserts">
 236 | # <entry align="right">
 237 | #   <emphasis>void</emphasis>
 238 | # </entry>
 239 | # <entry>
 240 | #   <funcsynopsis>
 241 | #     <funcprototype>
 242 | #       <funcdef><function>assertNotNull</function></funcdef>
 243 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 244 | #       <paramdef>string <parameter>value</parameter></paramdef>
 245 | #     </funcprototype>
 246 | #   </funcsynopsis>
 247 | #   <para>Asserts that <emphasis>value</emphasis> is <emphasis
 248 | #   role="strong">not</emphasis> <literal>null</literal>, or in shell terms not
 249 | #   a zero-length string. The message is optional.</para>
 250 | # </entry>
 251 | # </s:function>
 252 | #*/
 253 | assertNotNull()
 254 | {
 255 |   ${_SHUNIT_LINENO_}
 256 |   if [ $# -gt 2 ]; then  # allowing 0 arguments as $1 might actually be null
 257 |     _shunit_error "assertNotNull() requires one or two arguments; $# given"
 258 |     return ${SHUNIT_ERROR}
 259 |   fi
 260 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 261 | 
 262 |   shunit_message_=${__shunit_lineno}
 263 |   if [ $# -eq 2 ]; then
 264 |     shunit_message_="${shunit_message_}$1"
 265 |     shift
 266 |   fi
 267 |   assertTrue "${shunit_message_}" "[ -n '${1:-}' ]"
 268 |   shunit_return=$?
 269 | 
 270 |   unset shunit_message_
 271 |   return ${shunit_return}
 272 | }
 273 | _ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"'
 274 | 
 275 | #/**
 276 | # <s:function group="asserts">
 277 | # <entry align="right">
 278 | #   <emphasis>void</emphasis>
 279 | # </entry>
 280 | # <entry>
 281 | #   <funcsynopsis>
 282 | #     <funcprototype>
 283 | #       <funcdef><function>assertSame</function></funcdef>
 284 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 285 | #       <paramdef>string <parameter>expected</parameter></paramdef>
 286 | #       <paramdef>string <parameter>actual</parameter></paramdef>
 287 | #     </funcprototype>
 288 | #   </funcsynopsis>
 289 | #   <para>This function is functionally equivalent to
 290 | #   <function>assertEquals</function>.</para>
 291 | # </entry>
 292 | # </s:function>
 293 | #*/
 294 | assertSame()
 295 | {
 296 |   ${_SHUNIT_LINENO_}
 297 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 298 |     _shunit_error "assertSame() requires one or two arguments; $# given"
 299 |     return ${SHUNIT_ERROR}
 300 |   fi
 301 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 302 | 
 303 |   shunit_message_=${__shunit_lineno}
 304 |   if [ $# -eq 3 ]; then
 305 |     shunit_message_="${shunit_message_}$1"
 306 |     shift
 307 |   fi
 308 |   assertEquals "${shunit_message_}" "$1" "$2"
 309 |   shunit_return=$?
 310 | 
 311 |   unset shunit_message_
 312 |   return ${shunit_return}
 313 | }
 314 | _ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"'
 315 | 
 316 | #/**
 317 | # <s:function group="asserts">
 318 | # <entry align="right">
 319 | #   <emphasis>void</emphasis>
 320 | # </entry>
 321 | # <entry>
 322 | #   <funcsynopsis>
 323 | #     <funcprototype>
 324 | #       <funcdef><function>assertNotSame</function></funcdef>
 325 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 326 | #       <paramdef>string <parameter>unexpected</parameter></paramdef>
 327 | #       <paramdef>string <parameter>actual</parameter></paramdef>
 328 | #     </funcprototype>
 329 | #   </funcsynopsis>
 330 | #   <para>Asserts that <emphasis>unexpected</emphasis> and
 331 | #   <emphasis>actual</emphasis> are <emphasis role="strong">not</emphasis>
 332 | #   equal to one another. The message is optional.</para>
 333 | # </entry>
 334 | # </s:function>
 335 | #*/
 336 | assertNotSame()
 337 | {
 338 |   ${_SHUNIT_LINENO_}
 339 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 340 |     _shunit_error "assertNotSame() requires two or three arguments; $# given"
 341 |     return ${SHUNIT_ERROR}
 342 |   fi
 343 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 344 | 
 345 |   shunit_message_=${__shunit_lineno}
 346 |   if [ $# -eq 3 ]; then
 347 |     shunit_message_="${shunit_message_:-}$1"
 348 |     shift
 349 |   fi
 350 |   assertNotEquals "${shunit_message_}" "$1" "$2"
 351 |   shunit_return=$?
 352 | 
 353 |   unset shunit_message_
 354 |   return ${shunit_return}
 355 | }
 356 | _ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"'
 357 | 
 358 | #/**
 359 | # <s:function group="asserts">
 360 | # <entry align="right">
 361 | #   <emphasis>void</emphasis>
 362 | # </entry>
 363 | # <entry>
 364 | #   <funcsynopsis>
 365 | #     <funcprototype>
 366 | #       <funcdef><function>assertTrue</function></funcdef>
 367 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 368 | #       <paramdef>string <parameter>condition</parameter></paramdef>
 369 | #     </funcprototype>
 370 | #   </funcsynopsis>
 371 | #   <para>Asserts that a given shell test condition is true. The message is
 372 | #   optional.</para>
 373 | #   <para>Testing whether something is true or false is easy enough by using
 374 | #   the assertEquals/assertNotSame functions. Shell supports much more
 375 | #   complicated tests though, and a means to support them was needed. As such,
 376 | #   this function tests that conditions are true or false through evaluation
 377 | #   rather than just looking for a true or false.</para>
 378 | #   <funcsynopsis>
 379 | #     The following test will succeed: <funcsynopsisinfo>assertTrue "[ 34 -gt 23 ]"</funcsynopsisinfo>
 380 | #     The folloing test will fail with a message: <funcsynopsisinfo>assertTrue "test failed" "[ -r '/non/existant/file' ]"</funcsynopsisinfo>
 381 | #   </funcsynopsis>
 382 | # </entry>
 383 | # </s:function>
 384 | #*/
 385 | assertTrue()
 386 | {
 387 |   ${_SHUNIT_LINENO_}
 388 |   if [ $# -gt 2 ]; then
 389 |     _shunit_error "assertTrue() takes one two arguments; $# given"
 390 |     return ${SHUNIT_ERROR}
 391 |   fi
 392 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 393 | 
 394 |   shunit_message_=${__shunit_lineno}
 395 |   if [ $# -eq 2 ]; then
 396 |     shunit_message_="${shunit_message_}$1"
 397 |     shift
 398 |   fi
 399 |   shunit_condition_=$1
 400 | 
 401 |   # see if condition is an integer, i.e. a return value
 402 |   shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'`
 403 |   shunit_return=${SHUNIT_TRUE}
 404 |   if [ -z "${shunit_condition_}" ]; then
 405 |     # null condition
 406 |     shunit_return=${SHUNIT_FALSE}
 407 |   elif [ "${shunit_condition_}" = "${shunit_match_}" ]; then
 408 |     # possible return value. treating 0 as true, and non-zero as false.
 409 |     [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE}
 410 |   else
 411 |     # (hopefully) a condition
 412 |     ( eval ${shunit_condition_} ) >/dev/null 2>&1
 413 |     [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE}
 414 |   fi
 415 | 
 416 |   # record the test
 417 |   if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then
 418 |     _shunit_assertPass
 419 |   else
 420 |     _shunit_assertFail "${shunit_message_}"
 421 |   fi
 422 | 
 423 |   unset shunit_message_ shunit_condition_ shunit_match_
 424 |   return ${shunit_return}
 425 | }
 426 | _ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"'
 427 | 
 428 | #/**
 429 | # <s:function group="asserts">
 430 | # <entry align="right">
 431 | #   <emphasis>void</emphasis>
 432 | # </entry>
 433 | # <entry>
 434 | #   <funcsynopsis>
 435 | #     <funcprototype>
 436 | #       <funcdef><function>assertFalse</function></funcdef>
 437 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 438 | #       <paramdef>string <parameter>condition</parameter></paramdef>
 439 | #     </funcprototype>
 440 | #   </funcsynopsis>
 441 | #   <para>Asserts that a given shell test condition is false. The message is
 442 | #   optional.</para>
 443 | #   <para>Testing whether something is true or false is easy enough by using
 444 | #   the assertEquals/assertNotSame functions. Shell supports much more
 445 | #   complicated tests though, and a means to support them was needed. As such,
 446 | #   this function tests that conditions are true or false through evaluation
 447 | #   rather than just looking for a true or false.</para>
 448 | #   <funcsynopsis>
 449 | #     The following test will succeed: <funcsynopsisinfo>assertFalse "[ 'apples' = 'oranges' ]"</funcsynopsisinfo>
 450 | #     The folloing test will fail with a message: <funcsynopsisinfo>assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]"</funcsynopsisinfo>
 451 | #   </funcsynopsis>
 452 | # </entry>
 453 | # </s:function>
 454 | #*/
 455 | assertFalse()
 456 | {
 457 |   ${_SHUNIT_LINENO_}
 458 |   if [ $# -lt 1 -o $# -gt 2 ]; then
 459 |     _shunit_error "assertFalse() quires one or two arguments; $# given"
 460 |     return ${SHUNIT_ERROR}
 461 |   fi
 462 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 463 | 
 464 |   shunit_message_=${__shunit_lineno}
 465 |   if [ $# -eq 2 ]; then
 466 |     shunit_message_="${shunit_message_}$1"
 467 |     shift
 468 |   fi
 469 |   shunit_condition_=$1
 470 | 
 471 |   # see if condition is an integer, i.e. a return value
 472 |   shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'`
 473 |   shunit_return=${SHUNIT_TRUE}
 474 |   if [ -z "${shunit_condition_}" ]; then
 475 |     # null condition
 476 |     shunit_return=${SHUNIT_FALSE}
 477 |   elif [ "${shunit_condition_}" = "${shunit_match_}" ]; then
 478 |     # possible return value. treating 0 as true, and non-zero as false.
 479 |     [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE}
 480 |   else
 481 |     # (hopefully) a condition
 482 |     ( eval ${shunit_condition_} ) >/dev/null 2>&1
 483 |     [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE}
 484 |   fi
 485 | 
 486 |   # record the test
 487 |   if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then
 488 |     _shunit_assertPass
 489 |   else
 490 |     _shunit_assertFail "${shunit_message_}"
 491 |   fi
 492 | 
 493 |   unset shunit_message_ shunit_condition_ shunit_match_
 494 |   return ${shunit_return}
 495 | }
 496 | _ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"'
 497 | 
 498 | #-----------------------------------------------------------------------------
 499 | # failure functions
 500 | #
 501 | 
 502 | #/**
 503 | # <s:function group="failures">
 504 | # <entry align="right">
 505 | #   <emphasis>void</emphasis>
 506 | # </entry>
 507 | # <entry>
 508 | #   <funcsynopsis>
 509 | #     <funcprototype>
 510 | #       <funcdef><function>fail</function></funcdef>
 511 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 512 | #     </funcprototype>
 513 | #   </funcsynopsis>
 514 | #   <para>Fails the test immediately, with the optional message.</para>
 515 | # </entry>
 516 | # </s:function>
 517 | #*/
 518 | fail()
 519 | {
 520 |   ${_SHUNIT_LINENO_}
 521 |   if [ $# -gt 1 ]; then
 522 |     _shunit_error "fail() requires one or two arguments; $# given"
 523 |     return ${SHUNIT_ERROR}
 524 |   fi
 525 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 526 | 
 527 |   shunit_message_=${__shunit_lineno}
 528 |   if [ $# -eq 1 ]; then
 529 |     shunit_message_="${shunit_message_}$1"
 530 |     shift
 531 |   fi
 532 | 
 533 |   _shunit_assertFail "${shunit_message_}"
 534 | 
 535 |   unset shunit_message_
 536 |   return ${SHUNIT_FALSE}
 537 | }
 538 | _FAIL_='eval fail --lineno "${LINENO:-}"'
 539 | 
 540 | #/**
 541 | # <s:function group="failures">
 542 | # <entry align="right">
 543 | #   <emphasis>void</emphasis>
 544 | # </entry>
 545 | # <entry>
 546 | #   <funcsynopsis>
 547 | #     <funcprototype>
 548 | #       <funcdef><function>failNotEquals</function></funcdef>
 549 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 550 | #       <paramdef>string <parameter>unexpected</parameter></paramdef>
 551 | #       <paramdef>string <parameter>actual</parameter></paramdef>
 552 | #     </funcprototype>
 553 | #   </funcsynopsis>
 554 | #   <para>Fails the test if <emphasis>unexpected</emphasis> and
 555 | #   <emphasis>actual</emphasis> are <emphasis role="strong">not</emphasis>
 556 | #   equal to one another. The message is optional.</para>
 557 | # </entry>
 558 | # </s:function>
 559 | #*/
 560 | failNotEquals()
 561 | {
 562 |   ${_SHUNIT_LINENO_}
 563 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 564 |     _shunit_error "failNotEquals() requires one or two arguments; $# given"
 565 |     return ${SHUNIT_ERROR}
 566 |   fi
 567 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 568 | 
 569 |   shunit_message_=${__shunit_lineno}
 570 |   if [ $# -eq 3 ]; then
 571 |     shunit_message_="${shunit_message_}$1"
 572 |     shift
 573 |   fi
 574 |   shunit_unexpected_=$1
 575 |   shunit_actual_=$2
 576 | 
 577 |   _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_unexpected_}> but was:<${shunit_actual_}>"
 578 | 
 579 |   unset shunit_message_ shunit_unexpected_ shunit_actual_
 580 |   return ${SHUNIT_FALSE}
 581 | }
 582 | _FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"'
 583 | 
 584 | #/**
 585 | # <s:function group="failures">
 586 | # <entry align="right">
 587 | #   <emphasis>void</emphasis>
 588 | # </entry>
 589 | # <entry>
 590 | #   <funcsynopsis>
 591 | #     <funcprototype>
 592 | #       <funcdef><function>failSame</function></funcdef>
 593 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 594 | #     </funcprototype>
 595 | #   </funcsynopsis>
 596 | #   <para>Indicate test failure because arguments were the same. The message is
 597 | #   optional.</para>
 598 | # </entry>
 599 | # </s:function>
 600 | #*/
 601 | failSame()
 602 | {
 603 |   ${_SHUNIT_LINENO_}
 604 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 605 |     _shunit_error "failSame() requires two or three arguments; $# given"
 606 |     return ${SHUNIT_ERROR}
 607 |   fi
 608 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 609 | 
 610 |   shunit_message_=${__shunit_lineno}
 611 |   if [ $# -eq 3 ]; then
 612 |     shunit_message_="${shunit_message_}$1"
 613 |     shift
 614 |   fi
 615 | 
 616 |   _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same"
 617 | 
 618 |   unset shunit_message_
 619 |   return ${SHUNIT_FALSE}
 620 | }
 621 | _FAIL_SAME_='eval failSame --lineno "${LINENO:-}"'
 622 | 
 623 | #/**
 624 | # <s:function group="failures">
 625 | # <entry align="right">
 626 | #   <emphasis>void</emphasis>
 627 | # </entry>
 628 | # <entry>
 629 | #   <funcsynopsis>
 630 | #     <funcprototype>
 631 | #       <funcdef><function>failNotSame</function></funcdef>
 632 | #       <paramdef>string <parameter>[message]</parameter></paramdef>
 633 | #       <paramdef>string <parameter>expected</parameter></paramdef>
 634 | #       <paramdef>string <parameter>actual</parameter></paramdef>
 635 | #     </funcprototype>
 636 | #   </funcsynopsis>
 637 | #   <para>Indicate test failure because arguments were not the same. The
 638 | #   message is optional.</para>
 639 | # </entry>
 640 | # </s:function>
 641 | #*/
 642 | failNotSame()
 643 | {
 644 |   ${_SHUNIT_LINENO_}
 645 |   if [ $# -lt 2 -o $# -gt 3 ]; then
 646 |     _shunit_error "failNotEquals() requires one or two arguments; $# given"
 647 |     return ${SHUNIT_ERROR}
 648 |   fi
 649 |   _shunit_shouldSkip && return ${SHUNIT_TRUE}
 650 | 
 651 |   shunit_message_=${__shunit_lineno}
 652 |   if [ $# -eq 3 ]; then
 653 |     shunit_message_="${shunit_message_}$1"
 654 |     shift
 655 |   fi
 656 |   failNotEquals "${shunit_message_}" "$1" "$2"
 657 |   shunit_return=$?
 658 | 
 659 |   unset shunit_message_
 660 |   return ${shunit_return}
 661 | }
 662 | _FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"'
 663 | 
 664 | #-----------------------------------------------------------------------------
 665 | # skipping functions
 666 | #
 667 | 
 668 | #/**
 669 | # <s:function group="skipping">
 670 | # <entry align="right">
 671 | #   <emphasis>void</emphasis>
 672 | # </entry>
 673 | # <entry>
 674 | #   <funcsynopsis>
 675 | #     <funcprototype>
 676 | #       <funcdef><function>startSkipping</function></funcdef>
 677 | #       <paramdef />
 678 | #     </funcprototype>
 679 | #   </funcsynopsis>
 680 | #   <para>This function forces the remaining assert and fail functions to be
 681 | #   "skipped", i.e. they will have no effect. Each function skipped will be
 682 | #   recorded so that the total of asserts and fails will not be altered.</para>
 683 | # </entry>
 684 | # </s:function>
 685 | #*/
 686 | startSkipping()
 687 | {
 688 |   __shunit_skip=${SHUNIT_TRUE}
 689 | }
 690 | 
 691 | #/**
 692 | # <s:function group="skipping">
 693 | # <entry align="right">
 694 | #   <emphasis>void</emphasis>
 695 | # </entry>
 696 | # <entry>
 697 | #   <funcsynopsis>
 698 | #     <funcprototype>
 699 | #       <funcdef><function>endSkipping</function></funcdef>
 700 | #       <paramdef />
 701 | #     </funcprototype>
 702 | #   </funcsynopsis>
 703 | #   <para>This function returns calls to the assert and fail functions to their
 704 | #   default behavior, i.e. they will be called.</para>
 705 | # </entry>
 706 | # </s:function>
 707 | #*/
 708 | endSkipping()
 709 | {
 710 |   __shunit_skip=${SHUNIT_FALSE}
 711 | }
 712 | 
 713 | #/**
 714 | # <s:function group="skipping">
 715 | # <entry align="right">
 716 | #   <emphasis>boolean</emphasis>
 717 | # </entry>
 718 | # <entry>
 719 | #   <funcsynopsis>
 720 | #     <funcprototype>
 721 | #       <funcdef><function>isSkipping</function></funcdef>
 722 | #       <paramdef />
 723 | #     </funcprototype>
 724 | #   </funcsynopsis>
 725 | #   <para>This function returns the state of skipping.</para>
 726 | # </entry>
 727 | # </s:function>
 728 | #*/
 729 | isSkipping()
 730 | {
 731 |   return ${__shunit_skip}
 732 | }
 733 | 
 734 | #-----------------------------------------------------------------------------
 735 | # suite functions
 736 | #
 737 | 
 738 | #/**
 739 | # <s:function group="suites">
 740 | # <entry align="right">
 741 | #   <emphasis>void</emphasis>
 742 | # </entry>
 743 | # <entry>
 744 | #   <funcsynopsis>
 745 | #     <funcprototype>
 746 | #       <funcdef><function>suite</function></funcdef>
 747 | #       <paramdef />
 748 | #     </funcprototype>
 749 | #   </funcsynopsis>
 750 | #   <para>This function can be optionally overridden by the user in their test
 751 | #   suite.</para>
 752 | #   <para>If this function exists, it will be called when
 753 | #   <command>shunit2</command> is sourced. If it does not exist, shUnit2 will
 754 | #   search the parent script for all functions beginning with the word
 755 | #   <literal>test</literal>, and they will be added dynamically to the test
 756 | #   suite.</para>
 757 | # </entry>
 758 | # </s:function>
 759 | #*/
 760 | # Note: see _shunit_mktempFunc() for actual implementation
 761 | # suite() { :; }
 762 | 
 763 | #/**
 764 | # <s:function group="suites">
 765 | # <entry align="right">
 766 | #   <emphasis>void</emphasis>
 767 | # </entry>
 768 | # <entry>
 769 | #   <funcsynopsis>
 770 | #     <funcprototype>
 771 | #       <funcdef><function>suite_addTest</function></funcdef>
 772 | #       <paramdef>string <parameter>function</parameter></paramdef>
 773 | #     </funcprototype>
 774 | #   </funcsynopsis>
 775 | #   <para>This function adds a function name to the list of tests scheduled for
 776 | #   execution as part of this test suite. This function should only be called
 777 | #   from within the <function>suite()</function> function.</para>
 778 | # </entry>
 779 | # </s:function>
 780 | #*/
 781 | suite_addTest()
 782 | {
 783 |   shunit_func_=${1:-}
 784 | 
 785 |   __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}"
 786 |   __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1`
 787 | 
 788 |   unset shunit_func_
 789 | }
 790 | 
 791 | #/**
 792 | # <s:function group="suites">
 793 | # <entry align="right">
 794 | #   <emphasis>void</emphasis>
 795 | # </entry>
 796 | # <entry>
 797 | #   <funcsynopsis>
 798 | #     <funcprototype>
 799 | #       <funcdef><function>oneTimeSetUp</function></funcdef>
 800 | #       <paramdef />
 801 | #     </funcprototype>
 802 | #   </funcsynopsis>
 803 | #   <para>This function can be be optionally overridden by the user in their
 804 | #   test suite.</para>
 805 | #   <para>If this function exists, it will be called once before any tests are
 806 | #   run. It is useful to prepare a common environment for all tests.</para>
 807 | # </entry>
 808 | # </s:function>
 809 | #*/
 810 | # Note: see _shunit_mktempFunc() for actual implementation
 811 | # oneTimeSetUp() { :; }
 812 | 
 813 | #/**
 814 | # <s:function group="suites">
 815 | # <entry align="right">
 816 | #   <emphasis>void</emphasis>
 817 | # </entry>
 818 | # <entry>
 819 | #   <funcsynopsis>
 820 | #     <funcprototype>
 821 | #       <funcdef><function>oneTimeTearDown</function></funcdef>
 822 | #       <paramdef />
 823 | #     </funcprototype>
 824 | #   </funcsynopsis>
 825 | #   <para>This function can be be optionally overridden by the user in their
 826 | #   test suite.</para>
 827 | #   <para>If this function exists, it will be called once after all tests are
 828 | #   completed. It is useful to clean up the environment after all tests.</para>
 829 | # </entry>
 830 | # </s:function>
 831 | #*/
 832 | # Note: see _shunit_mktempFunc() for actual implementation
 833 | # oneTimeTearDown() { :; }
 834 | 
 835 | #/**
 836 | # <s:function group="suites">
 837 | # <entry align="right">
 838 | #   <emphasis>void</emphasis>
 839 | # </entry>
 840 | # <entry>
 841 | #   <funcsynopsis>
 842 | #     <funcprototype>
 843 | #       <funcdef><function>setUp</function></funcdef>
 844 | #       <paramdef />
 845 | #     </funcprototype>
 846 | #   </funcsynopsis>
 847 | #   <para>This function can be be optionally overridden by the user in their
 848 | #   test suite.</para>
 849 | #   <para>If this function exists, it will be called before each test is run.
 850 | #   It is useful to reset the environment before each test.</para>
 851 | # </entry>
 852 | # </s:function>
 853 | #*/
 854 | # Note: see _shunit_mktempFunc() for actual implementation
 855 | # setUp() { :; }
 856 | 
 857 | #/**
 858 | # <s:function group="suites">
 859 | # <entry align="right">
 860 | #   <emphasis>void</emphasis>
 861 | # </entry>
 862 | # <entry>
 863 | #   <funcsynopsis>
 864 | #     <funcprototype>
 865 | #       <funcdef><function>tearDown</function></funcdef>
 866 | #       <paramdef />
 867 | #     </funcprototype>
 868 | #   </funcsynopsis>
 869 | #   <para>This function can be be optionally overridden by the user in their
 870 | #   test suite.</para>
 871 | #   <para>If this function exists, it will be called after each test completes.
 872 | #   It is useful to clean up the environment after each test.</para>
 873 | # </entry>
 874 | # </s:function>
 875 | #*/
 876 | # Note: see _shunit_mktempFunc() for actual implementation
 877 | # tearDown() { :; }
 878 | 
 879 | #------------------------------------------------------------------------------
 880 | # internal shUnit2 functions
 881 | #
 882 | 
 883 | # this function is a cross-platform temporary directory creation tool. not all
 884 | # OSes have the mktemp function, so one is included here.
 885 | _shunit_mktempDir()
 886 | {
 887 |   # try the standard mktemp function
 888 |   ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return
 889 | 
 890 |   # the standard mktemp didn't work.  doing our own.
 891 |   if [ -r '/dev/urandom' ]; then
 892 |     _shunit_random_=`od -vAn -N4 -tx4 </dev/urandom |sed 's/^[^0-9a-f]*//'`
 893 |   elif [ -n "${RANDOM:-}" ]; then
 894 |     # $RANDOM works
 895 |     _shunit_random_=${RANDOM}${RANDOM}${RANDOM}$$
 896 |   else
 897 |     # $RANDOM doesn't work
 898 |     _shunit_date_=`date '+%Y%m%d%H%M%S'`
 899 |     _shunit_random_=`expr ${_shunit_date_} / $$`
 900 |   fi
 901 | 
 902 |   _shunit_tmpDir_="${TMPDIR:-/tmp}/shunit.${_shunit_random_}"
 903 |   ( umask 077 && mkdir "${_shunit_tmpDir_}" ) || {
 904 |     _shunit_fatal 'could not create temporary directory! exiting'
 905 |     exit ${SHUNIT_FALSE}
 906 |   }
 907 | 
 908 |   echo ${_shunit_tmpDir_}
 909 |   unset _shunit_date_ _shunit_random_ _shunit_tmpDir_
 910 | }
 911 | 
 912 | # this function is here to work around issues in Cygwin
 913 | _shunit_mktempFunc()
 914 | {
 915 |   for _shunit_func_ in oneTimeSetUp oneTimeTearDown setUp tearDown suite; do
 916 |     _shunit_file_="${__shunit_tmpDir}/${_shunit_func_}"
 917 |     cat <<EOF >"${_shunit_file_}"
 918 | #! /bin/sh
 919 | exit ${SHUNIT_TRUE}
 920 | EOF
 921 |     chmod +x "${_shunit_file_}"
 922 |   done
 923 | 
 924 |   unset _shunit_file_
 925 | }
 926 | 
 927 | _shunit_cleanup()
 928 | {
 929 |   _shunit_name_=$1
 930 | 
 931 |   case ${_shunit_name_} in
 932 |     EXIT) _shunit_signal_=0 ;;
 933 |     INT) _shunit_signal_=2 ;;
 934 |     TERM) _shunit_signal_=15 ;;
 935 |     *)
 936 |       _shunit_warn "unrecognized trap value (${_shunit_name_})"
 937 |       _shunit_signal_=0
 938 |       ;;
 939 |   esac
 940 | 
 941 |   # do our work
 942 |   rm -fr "${__shunit_tmpDir}"
 943 | 
 944 |   # exit for all non-EXIT signals
 945 |   if [ ${_shunit_name_} != 'EXIT' ]; then
 946 |     _shunit_warn "trapped and now handling the (${_shunit_name_}) signal"
 947 |     # disable EXIT trap
 948 |     trap 0
 949 |     # add 128 to signal and exit
 950 |     exit `expr ${_shunit_signal_} + 128`
 951 |   elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then
 952 |     _shunit_assertFail 'Unknown failure encountered running a test'
 953 |     _shunit_generateReport
 954 |     exit ${SHUNIT_ERROR}
 955 |   fi
 956 | 
 957 |   unset _shunit_name_ _shunit_signal_
 958 | }
 959 | 
 960 | # The actual running of the tests happens here.
 961 | _shunit_execSuite()
 962 | {
 963 |   for _shunit_test_ in ${__shunit_suite}; do
 964 |     __shunit_testSuccess=${SHUNIT_TRUE}
 965 | 
 966 |     # disable skipping
 967 |     endSkipping
 968 | 
 969 |     # execute the per-test setup function
 970 |     setUp
 971 | 
 972 |     # execute the test
 973 |     echo "${_shunit_test_}"
 974 |     eval ${_shunit_test_}
 975 | 
 976 |     # execute the per-test tear-down function
 977 |     tearDown
 978 | 
 979 |     # update stats
 980 |     if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then
 981 |       __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1`
 982 |     else
 983 |       __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1`
 984 |     fi
 985 |   done
 986 | 
 987 |   unset _shunit_test_
 988 | }
 989 | 
 990 | # This function exits shUnit2 with the appropriate error code and OK/FAILED
 991 | # message.
 992 | _shunit_generateReport()
 993 | {
 994 |   _shunit_ok_=${SHUNIT_TRUE}
 995 | 
 996 |   # if no exit code was provided one, determine an appropriate one
 997 |   [ ${__shunit_testsFailed} -gt 0 \
 998 |       -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \
 999 |           && _shunit_ok_=${SHUNIT_FALSE}
1000 | 
1001 |   echo
1002 |   if [ ${__shunit_testsTotal} -eq 1 ]; then
1003 |     echo "Ran ${__shunit_testsTotal} test."
1004 |   else
1005 |     echo "Ran ${__shunit_testsTotal} tests."
1006 |   fi
1007 | 
1008 |   _shunit_failures_=''
1009 |   _shunit_skipped_=''
1010 |   [ ${__shunit_assertsFailed} -gt 0 ] \
1011 |       && _shunit_failures_="failures=${__shunit_assertsFailed}"
1012 |   [ ${__shunit_assertsSkipped} -gt 0 ] \
1013 |       && _shunit_skipped_="skipped=${__shunit_assertsSkipped}"
1014 | 
1015 |   if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then
1016 |     _shunit_msg_='OK'
1017 |     [ -n "${_shunit_skipped_}" ] \
1018 |         && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})"
1019 |   else
1020 |     _shunit_msg_="FAILED (${_shunit_failures_}"
1021 |     [ -n "${_shunit_skipped_}" ] \
1022 |         && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}"
1023 |     _shunit_msg_="${_shunit_msg_})"
1024 |   fi
1025 | 
1026 |   echo
1027 |   echo ${_shunit_msg_}
1028 |   __shunit_reportGenerated=${SHUNIT_TRUE}
1029 | 
1030 |   unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_
1031 | }
1032 | 
1033 | _shunit_shouldSkip()
1034 | {
1035 |   [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE}
1036 |   _shunit_assertSkip
1037 | }
1038 | 
1039 | _shunit_assertPass()
1040 | {
1041 |   __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1`
1042 |   __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1`
1043 | }
1044 | 
1045 | _shunit_assertFail()
1046 | {
1047 |   _shunit_msg_=$1
1048 | 
1049 |   __shunit_testSuccess=${SHUNIT_FALSE}
1050 |   __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1`
1051 |   __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1`
1052 |   echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}"
1053 | 
1054 |   unset _shunit_msg_
1055 | }
1056 | 
1057 | _shunit_assertSkip()
1058 | {
1059 |   __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1`
1060 |   __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1`
1061 | }
1062 | 
1063 | #------------------------------------------------------------------------------
1064 | # main
1065 | #
1066 | 
1067 | # create a temporary storage location
1068 | __shunit_tmpDir=`_shunit_mktempDir`
1069 | 
1070 | # provide a public temporary directory for unit test scripts
1071 | # TODO(kward): document this
1072 | shunit_tmpDir="${__shunit_tmpDir}/tmp"
1073 | mkdir "${shunit_tmpDir}"
1074 | 
1075 | # setup traps to clean up after ourselves
1076 | trap '_shunit_cleanup EXIT' 0
1077 | trap '_shunit_cleanup INT' 2
1078 | trap '_shunit_cleanup TERM' 15
1079 | 
1080 | # create phantom functions to work around issues with Cygwin
1081 | _shunit_mktempFunc
1082 | PATH="${__shunit_tmpDir}:${PATH}"
1083 | 
1084 | # execute the oneTimeSetUp function (if it exists)
1085 | oneTimeSetUp
1086 | 
1087 | # execute the suite function defined in the parent test script
1088 | # deprecated as of 2.1.0
1089 | suite
1090 | 
1091 | # if no suite function was defined, dynamically build a list of functions
1092 | if [ -z "${__shunit_suite}" ]; then
1093 |   shunit_funcs_=`grep "^[ \t]*test[A-Za-z0-9_]* *()" ${__SHUNIT_PARENT} \
1094 |       |sed 's/[^A-Za-z0-9_]//g'`
1095 |   for shunit_func_ in ${shunit_funcs_}; do
1096 |     suite_addTest ${shunit_func_}
1097 |   done
1098 | fi
1099 | unset shunit_func_ shunit_funcs_
1100 | 
1101 | # execute the tests
1102 | _shunit_execSuite
1103 | 
1104 | # execute the oneTimeTearDown function (if it exists)
1105 | oneTimeTearDown
1106 | 
1107 | # generate the report
1108 | _shunit_generateReport
1109 | 
1110 | # that's it folks
1111 | [ ${__shunit_testsFailed} -eq 0 ]
1112 | exit $?
1113 | 
1114 | #/**
1115 | # </s:shelldoc>
1116 | #*/
1117 | 


--------------------------------------------------------------------------------
/ppss:
--------------------------------------------------------------------------------
   1 | #!/usr/bin/env bash
   2 | #
   3 | # PPSS, the Parallel Processing Shell Script
   4 | # 
   5 | # Copyright (c) 2010, Louwrentius
   6 | #
   7 | # This program is free software: you can redistribute it and/or modify
   8 | #    it under the terms of the GNU General Public License as published by
   9 | #    the Free Software Foundation, either version 3 of the License, or
  10 | #    (at your option) any later version.
  11 | #
  12 | #    This program is distributed in the hope that it will be useful,
  13 | #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 | #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 | #    GNU General Public License for more details.
  16 | #
  17 | #    See <http://www.gnu.org/licenses/>
  18 | #    for a copy of the GNU General Public License
  19 | #
  20 | # "Patches or other contributions are always welcome!"
  21 | #
  22 | 
  23 | #
  24 | # Handling control-c for a clean shutdown.
  25 | #
  26 | trap 'kill_process' SIGINT 
  27 | 
  28 | SCRIPT_NAME="Distributed Parallel Processing Shell Script"
  29 | SCRIPT_VERSION="2.98"
  30 | 
  31 | #
  32 | # The first argument to this script can be a mode.
  33 | #
  34 | MODES="node start config stop pause continue deploy status erase kill"
  35 | for x in $MODES
  36 | do
  37 |     if [[ "$x" == "$1" ]] 
  38 |     then
  39 |         MODE="$1"
  40 |         shift
  41 |         break
  42 |     fi
  43 | done
  44 | 
  45 | #
  46 | # The working directory of PPSS can be set with
  47 | # export PPSS_DIR=/path/to/workingdir
  48 | #
  49 | if [[ -z "$PPSS_DIR" ]] 
  50 | then
  51 |     PPSS_DIR="ppss_dir"
  52 | fi
  53 | 
  54 | CONFIG=""
  55 | HOSTNAME="$(hostname)"
  56 | ARCH="$(uname)"
  57 | PPSS_HOME_DIR="ppss-home"
  58 | SOURCED="$0"
  59 | 
  60 | PID="$$"
  61 | PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal"      # Pause processing if this file is present.
  62 | PAUSE_DELAY="60"                                          # Polling every 1 minutes by default.
  63 | STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal"        # Stop processing if this file is present.
  64 | GLOBAL_COUNTER=1
  65 | LISTOFITEMS="$PPSS_DIR/INPUT_FILE-$PID"
  66 | JOB_LOG_DIR="$PPSS_DIR/job_log"                           # Directory containing log files of processed items.
  67 | LOGFILE="$PPSS_DIR/ppss-log-$PID.txt"                     # General PPSS log file. Contains lots of info.
  68 | FAILED_ITEMS_COUNTER=0
  69 | if [[ -z "$QUIET" ]] 
  70 | then
  71 |     QUIET="0"
  72 | fi
  73 | STOP="0"                                                  # STOP job.
  74 | MAX_DELAY="0"                                             # MAX DELAY between jobs.
  75 | MAX_LOCK_DELAY="9"                                        #
  76 | PERCENT="0"
  77 | LISTENER_PID=""
  78 | IFS_BACKUP="$IFS"
  79 | CPUINFO="/proc/cpuinfo"
  80 | PROCESSORS=""
  81 | START_KEY="start-$RANDOM$RANDOM$RANDOM$RANDOM"           # If this key is received by listener, start a new process
  82 | FAIL_KEY="fail-$RANDOM$RANDOM$RANDOM$RANDOM"            # if this key is received by listener, increase error count              
  83 | KILL_KEY="kill-$RANDOM$RANDOM$RANDOM$RANDOM"            # This is a signal to stop immediately and kill PPSS
  84 | QUEUE=""
  85 | INOTIFY=""
  86 | TRAVERSAL="1"                                           # all running processes.
  87 | START_PPSS=""
  88 | STOP_PPSS=""
  89 | SIZE_OF_INPUT=""
  90 | LOCAL_LOCKING="1"
  91 | LIST_OF_PROCESSED_ITEMS="$PPSS_DIR/LIST_OF_PROCESSED_ITEMS"
  92 | PROCESSED_ITEMS=""
  93 | UNPROCESSED_ITEMS=""
  94 | FAILED_ITEMS="$PPSS_DIR/LIST_OF_FAILED_ITEMS"
  95 | ACTIVE_WORKERS="0"
  96 | DAEMON_POLLING_INTERVAL="10"
  97 | STAT=""
  98 | DAEMON_FILE_AGE="4"
  99 | ENABLE_INPUT_LOCK="0"
 100 | PROCESSING_TIME=""
 101 | NODE_ID="NODE_ID"
 102 | USE_MD5="0"
 103 | RANDOMIZE="0"
 104 | 
 105 | SSH_SERVER=""                           # Remote server or 'master'.
 106 | SSH_KEY=""                              # SSH key for ssh account.
 107 | SSH_KNOWN_HOSTS=""
 108 | SSH_SOCKET="$PPSS_DIR/ppss_ssh_socket-$$"          # Multiplex multiple SSH connections over 1 master.
 109 | SSH_OPTS="-o BatchMode=yes -o ControlPath=$SSH_SOCKET \
 110 |                            -o GlobalKnownHostsFile=./known_hosts \
 111 |                            -o ControlMaster=auto \
 112 |                            -o Cipher=blowfish \
 113 |                            -o ConnectTimeout=10 "
 114 | 
 115 | SSH_OPTS_NOMP="-o BatchMode=yes -o GlobalKnownHostsFile=./known_hosts \
 116 |                            -o Cipher=blowfish \
 117 |                            -o ConnectTimeout=10 "
 118 |                                         # Blowfish is faster but still secure. 
 119 | SSH_MASTER_PID=""
 120 | 
 121 | ITEM_LOCK_DIR="$PPSS_DIR/PPSS_ITEM_LOCK_DIR"      # Remote directory on master used for item locking.
 122 | PPSS_LOCAL_TMPDIR="$PPSS_DIR/PPSS_LOCAL_TMPDIR"   # Local directory on slave for local processing.
 123 | PPSS_LOCAL_OUTPUT="$PPSS_DIR/PPSS_LOCAL_OUTPUT"   # Local directory on slave for local output.
 124 | DOWNLOAD_TO_NODE="0"                   # Transfer item to slave via (s)cp.
 125 | UPLOAD_TO_SERVER="0"                   # Transfer output back to server via (s)cp.
 126 | SECURE_COPY="1"                         # If set, use SCP, Otherwise, use cp.
 127 | REMOTE_OUTPUT_DIR=""                    # Remote directory to which output must be uploaded.
 128 | SCRIPT=""                               # Custom user script that is executed by ppss.
 129 | ITEM_ESCAPED=""
 130 | DISABLE_SKIPPING=0
 131 | PPSS_NODE_STATUS="$PPSS_DIR/NODE_STATUS"
 132 | NODE_STATUS_FILE="$PPSS_NODE_STATUS/$HOSTNAME-status.txt"
 133 | DAEMON=0
 134 | EMAIL=""
 135 | 
 136 | REGISTER=""                               # For STACK
 137 | STACK=""
 138 | TMP_STACK=""
 139 | 
 140 | showusage_short () {
 141 | 
 142 |     echo
 143 |     echo "|P|P|S|S| $SCRIPT_NAME $SCRIPT_VERSION"
 144 |     echo
 145 |     echo "usage: $0 [[ -d <sourcedir> | -f <sourcefile> ]]  [[ -c '<command> \"\$ITEM\"' ]] "
 146 |     echo "                 [[ -C <configfile> ]] [[ -j ]] [[ -l <logfile> ]] [[ -p <# jobs> ]] "
 147 |     echo "                 [[ -q ]] [[ -D <delay> ]] [[ -h ]] [[ --help ]] [[ -r ]] [[ --daemon ]] " 
 148 |     echo
 149 |     echo "Examples:"
 150 |     echo "                 $0 -d /dir/with/some/files -c 'gzip '" 
 151 |     echo "                 $0 -d /dir/with/some/files -c 'cp \"\$ITEM\" /tmp' -p 2" 
 152 |     echo "                 $0 -f <file> -c 'wget -q -P /destination/directory \"\$ITEM\"' -p 10" 
 153 |     echo 
 154 | }
 155 | 
 156 | showusage_normal () {
 157 | 
 158 |     showusage_head
 159 |     showusage_basic
 160 |     showusage_basic_example
 161 | 
 162 | }
 163 | 
 164 | showusage_long () {
 165 | 
 166 |     showusage_extended_head
 167 |     showusage_basic
 168 |     showusage_extended_body
 169 | }
 170 | 
 171 | showusage_head () {
 172 | 
 173 |     echo 
 174 |     echo "|P|P|S|S| $SCRIPT_NAME $SCRIPT_VERSION"
 175 |     echo 
 176 |     echo "PPSS is a Bash shell script that executes commands in parallel on a set"
 177 |     echo "of items, such as files in a directory, or lines in a file. The purpose" 
 178 |     echo "of PPSS is to make it simple to benefit from multiple CPUs or CPU cores."
 179 |     echo 
 180 |     echo "This short summary only discusses options for stand-alone mode. For a"
 181 |     echo "full listing of all options, run PPSS with the options --help"
 182 |     echo
 183 | }
 184 | 
 185 | showusage_basic () {
 186 | 
 187 |     echo "Usage $0 [[ options ]] "
 188 |     echo
 189 |     echo -e "--command | -c     Command to execute. Syntax: '<command> ' including the single quotes."
 190 |     echo -e "                   Example: -c 'ls -alh '. It is also possible to specify where an item "
 191 |     echo -e "                   must be inserted: 'cp \"\$ITEM\" /somedir'."
 192 |     echo 
 193 |     echo -e "--sourcedir | -d   Directory that contains files that must be processed. Individual files" 
 194 |     echo -e "                   are fed as an argument to the command that has been specified with -c." 
 195 |     echo 
 196 |     echo -e "--sourcefile | -f  Each single line of the supplied file will be fed as an item to the"
 197 |     echo -e "                   command that has been specified with -c. Read input from stdin with"
 198 |     echo -e "                   -f -"
 199 |     echo 
 200 |     echo -e "--config | -C      If the mode is config, a config file with the specified name will be"
 201 |     echo -e "                   generated based on all the options specified. In the other modes". 
 202 |     echo -e "                   this option will result in PPSS reading the config file and start"
 203 |     echo -e "                   processing items based on the settings of this file."
 204 |     echo
 205 |     echo -e "--disable-ht | -j  Disable hyper threading. Is enabled by default."
 206 |     echo 
 207 |     echo -e "--log | -l         Sets the name of the log file. The default is ppss-log.txt."
 208 |     echo
 209 |     echo -e "--processes | -p   Start the specified number of processes. Ignore the number of available"
 210 |     echo -e "                   CPUs."
 211 |     echo
 212 |     echo -e "--quiet | -q       Shows no output except for a progress indication using percents."
 213 |     echo
 214 |     echo -e "--delay | -D       Adds an initial random delay to the start of all parallel jobs to spread"
 215 |     echo -e "                   the load. The delay (seconds) is only used at the start of all 'threads'."
 216 |     echo
 217 |     echo -e "--daemon           Daemon mode. Do not exit after items are professed, but keep looking "
 218 |     echo -e "                   for new items and process them. Read the manual how to use this!"
 219 |     echo -e "                   See --help for important additional options regarding daemon mode."
 220 |     echo
 221 |     echo -e "--disable-inotify  Linux users can use real-time inotify filesystem events when using"
 222 |     echo -e "                   daemon mode. Requires inotify-tools. Enabled by default if available."
 223 |     echo -e "                   Automatically disabled if NFS is used as the daeon source dir."
 224 |     echo
 225 |     echo -e "--no-traversal|-r  By default, PPSS uses the regular 'find' command to list all files"
 226 |     echo -e "                   within the directory specified by the -d option. If you do not wish"
 227 |     echo -e "                   for PPSS to process files in sub directories, use this option."
 228 |     echo -e "                   Only files within the specified directory will be processed. Any"
 229 |     echo -e "                   subdirectories will then be ignored."
 230 |     echo 
 231 |     echo -e "--email | -e       PPSS sends an e-mail if PPSS has finished. It is also used if processing"
 232 |     echo -e "                   of an item has failed (configurable, see -h). "
 233 |     echo
 234 |     echo -e "--debug            Enable debugging output to the |P|P|S|S| log file."
 235 |     echo 
 236 |     echo -e "--help             Extended help, including options for distributed mode."
 237 | }
 238 | 
 239 | showusage_basic_example () {
 240 | 
 241 |     echo 
 242 |     echo -e "Example: encoding some wav files to mp3 using lame:"
 243 |     echo 
 244 |     echo -e "$0 -d /path/to/wavfiles -c 'lame '" 
 245 |     echo 
 246 |     echo -e "Extended usage: use --help"
 247 |     echo
 248 | }
 249 | 
 250 | showusage_extended_head () {
 251 |     
 252 |     echo 
 253 |     echo "|P|P|S|S| $SCRIPT_NAME $SCRIPT_VERSION"
 254 |     echo 
 255 |     echo "PPSS is a Bash shell script that executes commands in parallel on a set  "
 256 |     echo "of items, such as files in a directory, or lines in a file."
 257 |     echo 
 258 |     echo "Usage: $0 [[ MODE ]] [[ options ]] "
 259 |     echo 
 260 |     echo "Modes are optional and mainly used for running in distributed mode. Modes are:"
 261 |     echo 
 262 |     echo " config       Generate a config file based on the supplied option parameters."
 263 |     echo " deploy       Deploy PPSS and related files on the specified nodes."
 264 |     echo " erase        Erase PPSS and related files from the specified nodes."
 265 |     echo
 266 |     echo " start        Starting PPSS on nodes."
 267 |     echo " pause        Pausing PPSS on all nodes."
 268 |     echo " stop         Stopping PPSS on all nodes."
 269 |     echo " continue     Continuing PPSS on all nodes."
 270 |     echo " node         Running PPSS as a node, requires additional options."
 271 | }
 272 | 
 273 | showusage_extended_body () {
 274 | 
 275 |     echo 
 276 |     echo -e "The following options are used for distributed execution of PPSS."
 277 |     echo 
 278 |     echo -e "--master | -m      Specifies the SSH server that is used for communication between nodes."
 279 |     echo -e "                   Using SSH, file locks are created, informing other nodes that an item "
 280 |     echo -e "                   is locked. If items are files that must be processed, they must reside" 
 281 |     echo -e "                   on this host. SCP is used to transfer files from this host to nodes"
 282 |     echo -e "                   for local procesing."
 283 |     echo
 284 |     echo -e "--node | -n        File containig a list of nodes that act as PPSS clients. One IP / DNS"
 285 |     echo -e "                   name per line."
 286 |     echo
 287 |     echo -e "--key | -k         The SSH key that a node uses to connect to the master."
 288 |     echo
 289 |     echo -e "--known-hosts | -K The file that contains the server public key. Can often be found on  "
 290 |     echo -e "                   hosts that already once connected to the server. See the file "
 291 |     echo -e "                   ~/.ssh/known_hosts or else, manualy connect once and check this file."
 292 |     echo
 293 |     echo -e "--user | -u        The SSH user name that is used by the node when logging in into the"
 294 |     echo -e "                   master SSH server."
 295 |     echo 
 296 |     echo -e "--script | -S      Specifies the script/program that must be copied to the nodes for"
 297 |     echo -e "                   execution through PPSS. Only used in the deploy mode."
 298 |     echo -e "                   This option should be specified if necessary when generating a config."
 299 |     echo
 300 |     echo -e "--download         This option specifies that an item will be downloaded by the node"
 301 |     echo -e "                   from the server or share to the local node for processing."
 302 |     echo 
 303 |     echo -e "--upload           This option specifies that the output file will be copied back to"
 304 |     echo -e "                   the server, the --outputdir option is mandatory."
 305 |     echo 
 306 |     echo -e "--no-scp | -b      Do not use scp for downloading items. Use cp instead. Assumes that a"
 307 |     echo -e "                   network file system (NFS/SMB) is mounted under a local mount point."
 308 |     echo 
 309 |     echo -e "--outputdir | -o   Directory on server where processed files are put. If the result of "
 310 |     echo -e "                   encoding a wav file is an mp3 file, the mp3 file is put in the "
 311 |     echo -e "                   directory specified with this option."
 312 |     echo 
 313 |     echo -e "--homedir | -H     Directory in which PPSS is installed on the node."
 314 |     echo -e "                   Default is '$PPSS_HOME_DIR'."
 315 |     echo
 316 |     echo -e "--script | -S      Script to run on the node. PPSS must copy this script to the node."
 317 |     echo
 318 |     echo -e "--randomize | -R   Randomise which items to process by the client in distributed mode."
 319 |     echo -e "                   This makes sure that with many nodes, it is prevented that some"
 320 |     echo -e "                   clients spend all their time trying to get a lock on an item."
 321 |     echo
 322 |     echo -e "Example: encoding some wav files to mp3 using lame:"
 323 |     echo 
 324 |     echo -e "$0 -c 'lame ' -d /path/to/wavfiles -j " 
 325 |     echo 
 326 |     echo -e "Running PPSS based on a configuration file."
 327 |     echo
 328 |     echo -e "$0 -C config.cfg"
 329 |     echo 
 330 |     echo -e "Generating a configuration file. Wavs are converted to mp3. SCP is used for data transfer."
 331 |     echo
 332 |     echo -e "$0 config -C ppss-config.cfg -d /some/dir -o output --download --upload -K known_hosts \\"
 333 |     echo -e "-k ppss-key.dsa -n nodes.txt -m 10.0.0.100 \\"
 334 |     echo -e "-c 'lame --quiet \"\$ITEM\" -o \"\$OUTPUT_DIR/\$OUTPUT_FILE\".mp3' "
 335 |     echo 
 336 |     echo -e "Running PPSS on a client as part of a cluster."
 337 |     echo 
 338 |     echo -e "$0 node -d /somedir -c 'cp \"\$ITEM\" /some/destination' -m 10.0.0.50 -u ppss -k ppss-key.key"
 339 |     echo    
 340 | }
 341 | 
 342 | kill_process () {
 343 | 
 344 |     echo "$KILL_KEY" >> "$FIFO"
 345 | }
 346 | 
 347 | exec_cmd () { 
 348 | 
 349 |     STATUS=""
 350 |     CMD="$1"
 351 |     NOMP="$2" # Disable multiplexing.
 352 | 
 353 |     if [[ "$ARCH" == "FreeBSD" ]]
 354 |     then
 355 |         CMD="bash $CMD"
 356 |     fi
 357 | 
 358 |     if [[ ! -z "$SSH_SERVER" ]] 
 359 |     then
 360 |         if [[ -z "$NOMP" ]] 
 361 |         then
 362 |             ssh $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER $CMD
 363 |             STATUS=$?
 364 |         elif [[ "$NOMP" == "1" ]] 
 365 |         then
 366 |             ssh $SSH_OPTS_NOMP $SSH_KEY $USER@$SSH_SERVER $CMD
 367 |             STATUS=$?
 368 |         fi
 369 |     else
 370 |         eval "$CMD"
 371 |         STATUS=$?
 372 |         log DEBUG "LOCAL EXEC - status is $STATUS"
 373 |     fi
 374 |     return $STATUS
 375 | }
 376 | 
 377 | does_file_exist () {
 378 | 
 379 |     #
 380 |     # this function makes remote or local checking of existence of items transparent.
 381 |     #
 382 |     FILE="$1"
 383 |     RES=$(exec_cmd "ls -1 $FILE" 2>&1)
 384 |     if [[ "$?" == "0" ]] 
 385 |     then
 386 |         return 0
 387 |     else 
 388 |         return 1
 389 |     fi
 390 | }
 391 | 
 392 | check_for_interrupt () {
 393 | 
 394 |     #
 395 |     # PPSS can be interupted with a stop or pause command.
 396 |     #
 397 |     does_file_exist "$STOP_SIGNAL"
 398 |     if [[ "$?" == "0" ]] 
 399 |     then
 400 |         set_status "STOPPED" "$FAILED_ITEMS_COUNTER"
 401 |         log INFO "STOPPING job. Stop signal found."
 402 |         STOP="1"
 403 |         return 1
 404 |     fi
 405 | 
 406 |     does_file_exist "$PAUSE_SIGNAL"
 407 |     if [[ "$?" == "0" ]] 
 408 |     then
 409 |         set_status "PAUSED" "$FAILED_ITEMS_COUNTER"
 410 |         log INFO "PAUSE: sleeping for $PAUSE_DELAY SECONDS."
 411 |         sleep $PAUSE_DELAY
 412 |         check_for_interrupt
 413 |     else
 414 |         set_status "RUNNING" "$FAILED_ITEMS_COUNTER"
 415 |     fi
 416 | }
 417 | 
 418 | cleanup () {
 419 |     
 420 |     log DEBUG "$FUNCNAME - Cleaning up all temp files and processes. $1"
 421 | 
 422 |     for x in $MODES
 423 |     do
 424 |         if [[ "$x" == "$MODE" ]] 
 425 |         then
 426 |             if [[ "$x" != "node" ]] 
 427 |             then
 428 |                 rm -rf "$PPSS_DIR"
 429 |             fi
 430 |         fi
 431 |     done
 432 |     
 433 |     if [[ -e "$FIFO" ]] 
 434 |     then 
 435 |         rm "$FIFO"
 436 |     fi
 437 | 
 438 |     if [[ -e "$FIFO_LISTENER" ]] 
 439 |     then 
 440 |         rm "$FIFO_LISTENER"
 441 |     fi
 442 | 
 443 |     if [[ -e "$SSH_SOCKET" ]] 
 444 |     then
 445 |         rm -rf "$SSH_SOCKET"
 446 |     fi
 447 |     
 448 |     if [[ ! -z "$SSH_MASTER_PID" ]] 
 449 |     then
 450 |         kill "$SSH_MASTER_PID"
 451 |     fi
 452 | }
 453 | 
 454 | add_var_to_config () {
 455 |     
 456 |     if [[ "$MODE" == "config" ]] 
 457 |     then
 458 | 
 459 |         VAR="$1"
 460 |         VALUE="$2"
 461 | 
 462 |         echo -e "$VAR=$VALUE" >> $CONFIG
 463 |     fi
 464 | }
 465 | 
 466 | is_var_empty () {
 467 | 
 468 |     if [[ -z "$1" ]] 
 469 |     then
 470 |         showusage_
 471 |         cleanup
 472 |         exit 1
 473 |     fi
 474 | }
 475 | 
 476 | detect_source_dir_nfs_exported () {
 477 |     
 478 |     #log DEBUG "Executing $FUNCNAME"
 479 | 
 480 |     NFS=0
 481 |     NFS_MOUNTED_FS=$(mount | grep ":" | cut -d ":" -f 2 | awk '{ print $3 }')
 482 | 
 483 |     for mount in $NFS_MOUNTED_FS
 484 |     do
 485 |     
 486 |         #
 487 |         # If this for loop matches anything, the SRC_DIR is NFS exported.
 488 |         # inotify does not play well with NFS. So it must be disabled.
 489 |         #
 490 |         DIRECTORY="$SRC_DIR"
 491 |         while true
 492 |         do
 493 |             #log DEBUG "NFS TEST: $mount vs. $DIRECTORY"
 494 |             if [[ $DIRECTORY != [/.]* ]] 
 495 |             then
 496 |                 if [[ "$mount" == "$DIRECTORY" ]] 
 497 |                 then
 498 |                     NFS=1
 499 |                     break
 500 |                 fi
 501 |             else
 502 |                 break
 503 |             fi
 504 |             DIRECTORY=$(dirname "$DIRECTORY")
 505 |         done
 506 |     done
 507 |     if [[ "$NFS" == "1" ]] 
 508 |     then
 509 |         #log DEBUG "Source directory is NFS exported. Disabling inotify."
 510 |         return 1
 511 |     else
 512 |         #log DEBUG "Source directory is NOT NFS exported. Enabling inotify."
 513 |         return 0
 514 |     fi 
 515 | }
 516 | 
 517 | detect_inotify () {
 518 | 
 519 |     if [[ -e /usr/bin/inotifywait && "$INOTIFY" != "0" ]] && detect_source_dir_nfs_exported
 520 |     then
 521 |         INOTIFY=1
 522 |     else
 523 |         INOTIFY=0
 524 |     fi
 525 | }
 526 | 
 527 | process_arguments () {
 528 | 
 529 |     #
 530 |     # Process any command-line options that are specified."
 531 |     #
 532 | 
 533 |     if [[ "$#" == "0" ]] 
 534 |     then
 535 |         showusage_short
 536 |         exit 1
 537 |     fi
 538 | 
 539 |     while [[ $# -gt 0 ]] 
 540 |     do
 541 |         case $1 in
 542 | 
 543 |             --config|-C )
 544 |                             CONFIG="$2"
 545 |                             is_var_empty "$CONFIG" 
 546 | 
 547 |                             if [[ "$MODE" == "config" ]] 
 548 |                             then
 549 |                                 if [[ -e "$CONFIG" ]] 
 550 |                                 then
 551 |                                     echo "Do want to overwrite existing config file? [y/n]"
 552 |                                     read yn
 553 |                                     if [[ "$yn" == "y" ]] || [[ "$yn" == "yes" ]] 
 554 |                                     then
 555 |                                         rm "$CONFIG"
 556 |                                     else
 557 |                                         echo "Aborting..."
 558 |                                         cleanup
 559 |                                         exit 1
 560 |                                     fi 
 561 |                                 fi
 562 |                             fi
 563 | 
 564 |                             if [[ "$MODE" != "config" ]] 
 565 |                             then
 566 |                                 source $CONFIG
 567 |                             fi
 568 | 
 569 |                             if [[ ! -z "$SSH_KEY" ]] 
 570 |                             then
 571 |                                 SSH_KEY="-i $SSH_KEY"
 572 |                             fi
 573 | 
 574 |                             if [[ ! -e "./known_hosts" ]] 
 575 |                             then
 576 |                                 if [[ -e $SSH_KNOWN_HOSTS ]] 
 577 |                                 then
 578 |                                     if [[ "$SSH_KNOWN_HOSTS" != "known_hosts" ]] 
 579 |                                     then
 580 |                                         cat $SSH_KNOWN_HOSTS > ./known_hosts
 581 |                                     fi
 582 |                                 else
 583 |                                     echo "File $SSH_KNOWN_HOSTS does not exist."
 584 |                                     exit 1
 585 |                                 fi
 586 |                             fi
 587 |                             shift 2 ;;
 588 | 
 589 |           --working-dir|-w )
 590 |                             PPSS_DIR="$2"
 591 |                             add_var_to_config PPSS_DIR "$PPSS_DIR"
 592 |                             shift 2 ;; 
 593 |                  --node|-n ) 
 594 |                             NODES_FILE="$2"
 595 |                             add_var_to_config NODES_FILE "$NODES_FILE"
 596 |                             shift 2 ;; 
 597 |            --sourcefile|-f )
 598 |                             INPUT_FILE="$2"
 599 |                             is_var_empty "$INPUT_FILE" 
 600 |                             add_var_to_config INPUT_FILE "$INPUT_FILE"
 601 |                             shift 2 ;;
 602 |             --sourcedir|-d ) 
 603 |                             SRC_DIR="$2"
 604 |                             is_var_empty "$SRC_DIR" 
 605 |                             add_var_to_config SRC_DIR "$SRC_DIR"
 606 |                             shift 2 ;; 
 607 |                  --delay|-D)
 608 |                             MAX_DELAY="$2"
 609 |                             add_var_to_config MAX_DELAY "$MAX_DELAY"
 610 |                             shift 2 ;;
 611 |         --disable-inotify)
 612 |                             INOTIFY=0
 613 |                             add_var_to_config INOTIFY "$INOTIFY"
 614 |                             shift 1 ;;
 615 |         --enable-input-lock)
 616 |                             ENABLE_INPUT_LOCK=1
 617 |                             add_var_to_config ENABLE_INPUT_LOCK "$ENABLE_INPUT_LOCK"
 618 |                             shift 1 ;;
 619 |                  --daemon)
 620 |                             DAEMON="1"
 621 |                             QUIET="1"
 622 |                             detect_inotify
 623 |                             add_var_to_config DAEMON "$DAEMON"
 624 |                             add_var_to_config QUIET "$QUIET"
 625 |                             add_var_to_config INOTIFY "$INOTIFY"
 626 |                             shift 1 ;;
 627 |                  --interval) 
 628 |                             is_var_empty "$2"
 629 |                             DAEMON_POLLING_INTERVAL="$2"
 630 |                             add_var_to_config DAEMON_POLLING_INTERVAL "$DAEMON_POLLING_INTERVAL"
 631 |                             shift 2 ;;
 632 |                  --file-age)
 633 |                             is_var_empty "$2"
 634 |                             add_var_to_config DAEMON_FILE_AGE "$DAEMON_FILE_AGE"
 635 |                             shift 2 ;;
 636 |                 --email|-e)
 637 |                             is_var_empty "$2"
 638 |                             EMAIL="$2"
 639 |                             add_var_to_config EMAIL "$EMAIL"
 640 |                             shift 2 ;;
 641 |               --command|-c ) 
 642 |                             COMMAND="$2"
 643 |                             is_var_empty "$COMMAND"
 644 |                             if [[ "$MODE" == "config" ]] 
 645 |                             then
 646 |                                 COMMAND=\'$COMMAND\'
 647 |                                 add_var_to_config COMMAND "$COMMAND"
 648 |                             fi
 649 |                             shift 2 ;;
 650 | 
 651 |                         -h )
 652 |                             showusage_normal
 653 |                             exit 1 ;;
 654 |                      --help)
 655 |                             showusage_long
 656 |                             exit 1 ;;
 657 |               --homedir|-H )
 658 |                             is_var_empty "$2"
 659 |                             PPSS_HOME_DIR="$2"
 660 |                             add_var_to_config PPSS_DIR $PPSS_HOME_DIR
 661 |                             shift 2 ;; 
 662 |            --disable-ht|-j )
 663 |                             HYPERTHREADING=no
 664 |                             add_var_to_config HYPERTHREADING $HYPERTHREADING
 665 |                             shift 1 ;;
 666 |                   --log|-l )
 667 |                             LOGFILE="$2"
 668 |                             add_var_to_config LOGFILE "$LOGFILE"
 669 |                             shift 2 ;;
 670 |           --no-traversal|-r )
 671 |                             TRAVERSAL="0"
 672 |                             add_var_to_config LOGFILE "$TRAVERSAL"
 673 |                             shift 1 ;;
 674 |            --workingdir|-w ) 
 675 |                             WORKINGDIR="$2"
 676 |                             add_var_to_config WORKINGDIR "$WORKINGDIR"
 677 |                             shift 2 ;;
 678 |            --md5|-M ) 
 679 |                             USE_MD5="1"
 680 |                             add_var_to_config USE_MD5 "$USE_MD5"
 681 |                             shift 1 ;;
 682 |                   --key|-k )
 683 |                             SSH_KEY="$2"
 684 |                             is_var_empty "$SSH_KEY"
 685 |                             add_var_to_config SSH_KEY "$SSH_KEY"
 686 |                             if [[ ! -z "$SSH_KEY" ]] 
 687 |                             then
 688 |                                 SSH_KEY="-i $SSH_KEY"
 689 |                             fi
 690 |                             shift 2 ;;
 691 |         --known-hosts | -K ) 
 692 |                             SSH_KNOWN_HOSTS="$2"
 693 |                             add_var_to_config SSH_KNOWN_HOSTS "$SSH_KNOWN_HOSTS"
 694 |                             shift 2 ;;
 695 |                                 
 696 |               --no-scp |-b )
 697 |                             SECURE_COPY=0
 698 |                             add_var_to_config SECURE_COPY "$SECURE_COPY"
 699 |                             shift 1 ;;
 700 |            --randomize |-R )
 701 |                             RANDOMIZE=1
 702 |                             add_var_to_config RANDOMIZE "$RANDOMIZE"
 703 |                             shift 1 ;;
 704 |             --outputdir|-o )
 705 |                             REMOTE_OUTPUT_DIR="$2"
 706 |                             add_var_to_config REMOTE_OUTPUT_DIR "$REMOTE_OUTPUT_DIR"
 707 |                             shift 2 ;;
 708 |             --processes|-p )
 709 |                             is_var_empty "$2" 
 710 |                             MAX_NO_OF_RUNNING_JOBS="$2"
 711 |                             add_var_to_config MAX_NO_OF_RUNNING_JOBS "$MAX_NO_OF_RUNNING_JOBS" 
 712 |                             shift 2 ;;
 713 |             --master|-m ) 
 714 |                             SSH_SERVER="$2"
 715 |                             add_var_to_config SSH_SERVER "$SSH_SERVER"
 716 |                             shift 2 ;;
 717 |             --script|-S )
 718 |                             SCRIPT="$2"
 719 |                             add_var_to_config SCRIPT "$SCRIPT"
 720 |                             shift 2 ;;
 721 |             --debug)
 722 |                             PPSS_DEBUG="1"    
 723 |                             add_var_to_config PPSS_DEBUG "$PPSS_DEBUG"
 724 |                             shift 1 ;;
 725 |             --download)
 726 |                             DOWNLOAD_TO_NODE="1"    
 727 |                             add_var_to_config DOWNLOAD_TO_NODE "$DOWNLOAD_TO_NODE"
 728 |                             shift 1 ;;
 729 |             --upload)
 730 |                             if [[ -z "$REMOTE_OUTPUT_DIR" ]] 
 731 |                             then
 732 |                                 echo "ERROR: no server-side output directory specified with -o"
 733 |                                 exit 1
 734 |                             fi
 735 |                             UPLOAD_TO_SERVER="1"    
 736 |                             add_var_to_config UPLOAD_TO_SERVER "$UPLOAD_TO_SERVER"
 737 |                             shift 1 ;;
 738 |             --quiet|-q )    
 739 |                             QUIET="1"
 740 |                             add_var_to_config QUIET "$QUIET"
 741 |                             shift 1 ;;
 742 |             --user|-u )
 743 |                             USER="$2"
 744 |                             add_var_to_config USER "$USER"
 745 |                             shift 2 ;;
 746 |             --version|-v )
 747 |                             echo ""
 748 |                             echo "$SCRIPT_NAME version $SCRIPT_VERSION"
 749 |                             echo ""
 750 |                             exit 0 ;;
 751 |             * )
 752 |                             
 753 |                             showusage_short
 754 |                             echo
 755 |                             echo "Unknown option $1 "
 756 |                             echo
 757 |                             exit 1 ;;
 758 |         esac
 759 |     done
 760 | 
 761 |     if [[ -z "$SRC_DIR" && -z "$INPUT_FILE" ]] 
 762 |     then
 763 |         showusage_short
 764 |         echo
 765 |         echo "No source file or directory specified with -f or -d."
 766 |         exit 1
 767 |     fi
 768 | 
 769 |     if [[ ! -e "$SRC_DIR" && -z "$MODE" && -z "$INPUT_FILE" ]]
 770 |     then
 771 |         showusage_short
 772 |         echo
 773 |         echo "Source directory $SRC_DIR does not exist."
 774 |         exit 1
 775 |     fi
 776 |     
 777 |     if [[ "$SRC_DIR" == "." ]] 
 778 |     then
 779 |         echo 
 780 |         echo "|P|P|S|S| is not designed to process items from within the directory"
 781 |         echo "it is being run. PPSS will start to process its own files from"
 782 |         echo "its working directory $PPSS_DIR which is probably not wat you"
 783 |         echo "want. Are you sure you want to continue?"
 784 |         echo 
 785 |         read YN
 786 |         if [[ "$YN" != "y" && "$YN" != "Y" ]]
 787 |         then
 788 |             exit 1
 789 |         fi
 790 |     fi
 791 | 
 792 |     if [[ "$DAEMON" == "1" && -z "$SRC_DIR" ]]
 793 |     then
 794 |         showusage_short
 795 |         echo
 796 |         echo "Daemon monitors a specified directory (with the -d option) for files to process."
 797 |         echo "Read the on-line manual for more information."
 798 |         exit 1
 799 |     fi 
 800 | }
 801 | 
 802 | display_header () {
 803 | 
 804 |     log DSPLY ""
 805 |     log DSPLY "========================================================="
 806 |     log DSPLY "                       |P|P|S|S|                         "
 807 |     log DSPLY "$SCRIPT_NAME vers. $SCRIPT_VERSION"
 808 |     log DSPLY "========================================================="
 809 |     log DSPLY "Hostname:\t\t$HOSTNAME"
 810 |     log DSPLY "---------------------------------------------------------"
 811 | }
 812 | 
 813 | create_working_directory () {
 814 | 
 815 |     if [[ ! -e "$PPSS_DIR" ]] 
 816 |     then
 817 |         mkdir -p "$PPSS_DIR"
 818 |     fi
 819 | }
 820 | 
 821 | expand_str () {
 822 | 
 823 |     STR=$1
 824 |     LENGTH=$TYPE_LENGTH
 825 |     SPACE=" "
 826 | 
 827 |     while [[ "${#STR}" -lt "$LENGTH" ]] 
 828 |     do
 829 |         STR=$STR$SPACE
 830 |     done
 831 | 
 832 |     echo "$STR"
 833 | }
 834 | 
 835 | are_we_sourced () {
 836 | 
 837 |     RES=$(basename $SOURCED)
 838 |     
 839 |     if [[ "$RES" == "ppss" ]] 
 840 |     then
 841 |         return 1
 842 |     else
 843 |         return 0
 844 |     fi 
 845 | }
 846 | 
 847 | get_time_in_seconds () {
 848 | 
 849 |     if [[ "$ARCH" == "SunOS" ]] 
 850 |     then
 851 |         #
 852 |         # Dirty hack because this ancient operating system does not support +%s...
 853 |         #
 854 |         # SWB: Clever, but horrible.  Use perl since I already created a dependency
 855 |         # there by using it to fix sed and [[:alnum:]] not working.
 856 |         #        THE_TIME=$(truss /usr/bin/date 2>&1 | grep ^time | awk '{ print $3 }')
 857 |         THE_TIME=$(perl -e 'print time')
 858 |     else
 859 |         THE_TIME="$(date +%s)"
 860 |     fi
 861 | 
 862 |     echo "$THE_TIME"
 863 | }
 864 | 
 865 | set_md5 () {
 866 | 
 867 |         case $ARCH in
 868 |             "Darwin")  MD5=md5 ;;
 869 |             "FreeBSD") MD5=md5 ;;
 870 |             "SunOS")   MD5="digest -a md5" ;; 
 871 |             "Linux")   MD5=md5sum ;;
 872 |         esac
 873 | 
 874 |         echo "test" | $MD5 > /dev/null 2>&1
 875 |         if [[ ! "$?" ]] 
 876 |         then
 877 |             LOG ERROR "ERROR - PPSS requires $MD5. It may not be within the path or installed."
 878 |             return 1
 879 |         else
 880 |             return 0 
 881 |         fi
 882 | }
 883 | 
 884 | set_stat () {
 885 | 
 886 |     if [[ "$DAEMON" == "1" && "$INOTIFY" == "0" ]] 
 887 |     then
 888 |         case $ARCH in
 889 |             "Darwin")  STAT="stat -f%m" ;;
 890 |             "FreeBSD") STAT="stat -f%m" ;;
 891 |             "SunOS")   STAT="gstat -c%Y" ;;
 892 |             "Linux")   STAT="stat -c%Y" ;;
 893 |         esac
 894 | 
 895 |         $STAT . >> /dev/null 2>&1
 896 |         if [[ ! "$?" ]] 
 897 |         then
 898 |             LOG ERROR "ERROR - PPSS daemon mode requires stat. It may not be within the path or installed."
 899 |             return 1
 900 |         else
 901 |             return 0 
 902 |         fi
 903 |     else
 904 |         return 0
 905 |     fi
 906 | }
 907 | 
 908 | log () {
 909 | 
 910 |     #
 911 |     # Type 'DSPLY ERROR and WARN' is logged to the screen
 912 |     # Any other log-type is only logged to the logfile.
 913 |     #
 914 |     TYPE="$1"
 915 |     MESG="$2"
 916 |     TYPE_LENGTH=5 
 917 | 
 918 |     #
 919 |     # Performance hack. Don't go through all the code if not required.
 920 |     #
 921 | 
 922 |     if [[ "$TYPE" == "DEBUG" && "$PPSS_DEBUG" == "0" ]] 
 923 |     then
 924 |         return 
 925 |     fi
 926 | 
 927 |     TYPE_EXP=$(expand_str "$TYPE")
 928 | 
 929 |     DATE=$(date +%b\ %d\ %H:%M:%S)
 930 |     PREFIX="$DATE: ${TYPE_EXP:0:$TYPE_LENGTH}"
 931 |     PREFIX_SMALL="$DATE: "
 932 | 
 933 |     if [[ "$TYPE" != "ERROR" ]] 
 934 |     then
 935 |         ECHO_MSG="$PREFIX_SMALL $MESG"
 936 |     else
 937 |         ECHO_MSG="$PREFIX_SMALL [ERROR] $MESG"
 938 |     fi
 939 |     LOG_MSG="$PREFIX $MESG"
 940 | 
 941 |     if [[ ! -z "$PPSS_DEBUG" && "$PPSS_DEBUG" != "0" ]]
 942 |     then
 943 |         echo -e "$LOG_MSG" >> "$LOGFILE"
 944 | 
 945 |     elif [[ "$TYPE" == "INFO" || "$TYPE" == "ERROR" || "$TYPE" == "WARN" || "$TYPE" == "DSPLY" ]]
 946 |     then
 947 |         echo -e "$LOG_MSG" >> "$LOGFILE"
 948 |     fi
 949 | 
 950 |     if [[ "$TYPE" == "DSPLY" || "$TYPE" == "ERROR" || "$TYPE" == "WARN" && "$QUIET" == "0" ]]
 951 |     then
 952 |         echo -e "$ECHO_MSG"
 953 | 
 954 |     elif [[ "$TYPE" == "ERROR" && "$QUIET" == "1" ]]
 955 |     then
 956 |         echo -e "$ECHO_MSG"
 957 |     fi
 958 |     if [[ "$TYPE" == "PRCNT" ]] 
 959 |     then
 960 |        echo -en "\r$ECHO_MSG"
 961 |        #echo "$ECHO_MSG" # for debugging.
 962 |     fi
 963 | }
 964 | 
 965 | init_vars () {
 966 | 
 967 |     #
 968 |     # Get start time to measure how long PPSS has been running.
 969 |     #
 970 |     START_PPSS=$(get_time_in_seconds)
 971 | 
 972 |     #
 973 |     # Check if MD5(SUM) is present on the system.
 974 |     #
 975 |     set_md5
 976 |     #
 977 |     # Chec if stat is present and works on the system if daemon mode is enabled.
 978 |     #
 979 |     set_stat
 980 | 
 981 |     #
 982 |     # Is PPSS run as a daemon? Then use input locking, which is not required otherwise.
 983 |     #
 984 |     if [[ "$DAEMON" == "1" ]] 
 985 |     then
 986 |         INPUT_LOCK="$SRC_DIR/INPUT_LOCK"
 987 |     fi
 988 | 
 989 |     #
 990 |     # For some strange reason, this value differ on different operating systems due to
 991 |     # different behaviour betwen the ps utilily acros operating systems.
 992 |     #
 993 |     if [[ "$ARCH" == "Darwin" ]] 
 994 |     then
 995 |         MIN_JOBS=4
 996 |     elif [[ "$ARCH" == "Linux" ]] 
 997 |     then
 998 |         MIN_JOBS=3
 999 |     fi
1000 | 
1001 |     FIFO="$PPSS_DIR"/ppss-fifo-$RANDOM-$RANDOM
1002 |     FIFO_LISTENER="$PPSS_DIR"/ppss-fifo-listener-$RANDOM-$RANDOM
1003 | 
1004 |     if [[ ! -e "$FIFO" ]] 
1005 |     then    
1006 |         mkfifo -m 600 $FIFO
1007 |     fi
1008 | 
1009 |     if [[ ! -e "$FIFO_LISTENER" ]] 
1010 |     then    
1011 |         mkfifo -m 600 $FIFO_LISTENER
1012 |     fi
1013 | 
1014 |     exec 42<> $FIFO
1015 |     exec 43<> $FIFO_LISTENER
1016 | 
1017 |     set_status "RUNNING" 
1018 | 
1019 |     if [[ -e "$CPUINFO" ]] 
1020 |     then
1021 |         CPU=$(cat $CPUINFO | grep 'model name' | cut -d ":" -f 2 | sed -e s/^\ //g | sort | uniq)
1022 |         log DSPLY "CPU: $CPU"
1023 |     elif [[ "$ARCH" == "Darwin" ]] 
1024 |     then
1025 |         MODEL=$(system_profiler SPHardwareDataType | grep "Processor Name" | cut -d ":" -f 2)
1026 |         SPEED=$(system_profiler SPHardwareDataType | grep "Processor Speed" | cut -d ":" -f 2)
1027 |         log DSPLY "CPU: $MODEL $SPEED"
1028 |     elif [[ "$ARCH" == "SunOS" ]] 
1029 |     then
1030 |         CPU=$(/usr/sbin/psrinfo -v | grep MHz | cut -d " " -f 4,8 | awk '{ printf ("Processor architecture: %s @ %s MHz.\n", $1,$2) }' | head -n 1)
1031 |         log DSPLY "$CPU"
1032 |     else
1033 |         log DSPLY "CPU: Cannot determine. Provide a patch for your arch!"
1034 |         log DSPLY "Arch is $ARCH"
1035 |     fi 
1036 | 
1037 |     if [[ -z "$MAX_NO_OF_RUNNING_JOBS" ]] 
1038 |     then 
1039 |         get_no_of_cpus $HYPERTHREADING
1040 |     fi
1041 | 
1042 |     if [[ ! -z "$SSH_SERVER" ]] 
1043 |     then
1044 |         does_file_exist "$PPSS_HOME_DIR/$JOB_LOG_DIR"
1045 |         if [[ ! "$?" == "0" ]]  
1046 |         then
1047 |             log DEBUG "Remote Job log directory $PPSS_HOME_DIR/$JOB_lOG_DIR does not exist. Creating."
1048 |             exec_cmd "mkdir $PPSS_HOME_DIR/$JOB_LOG_DIR"
1049 |         fi
1050 |     fi
1051 | 
1052 |     if [[ ! -e "$JOB_LOG_DIR" ]] 
1053 |     then
1054 |         mkdir -p "$JOB_LOG_DIR"
1055 |     fi
1056 | 
1057 |     if [[ ! -z "$SSH_SERVER" ]] 
1058 |     then
1059 |         ITEM_LOCK_DIR="$PPSS_HOME_DIR/$ITEM_LOCK_DIR"
1060 |     fi
1061 |     
1062 |     does_file_exist "$ITEM_LOCK_DIR"
1063 |     if [[ ! "$?" == "0" ]] 
1064 |     then
1065 |         if [[ ! -z "$SSH_SERVER" ]] 
1066 |         then
1067 |             log DEBUG "Creating remote item lock dir."
1068 |         else
1069 |             log DEBUG "Creating local item lock dir."
1070 |         fi
1071 |         exec_cmd "mkdir $ITEM_LOCK_DIR"
1072 |         if [[ ! "$?" ]] 
1073 |         then
1074 |             log DEBUG "Failed to create item lock dir."
1075 |         fi
1076 |     fi
1077 | 
1078 |     if [[ ! -z "$SSH_SERVER" ]] 
1079 |     then
1080 |         does_file_exist "$REMOTE_OUTPUT_DIR"
1081 |         if [[ ! "$?" == "0" ]] 
1082 |         then
1083 |             log DEBUG "Remote output dir $REMOTE_OUTPUT_DIR does not exist."
1084 |             exec_cmd "mkdir $REMOTE_OUTPUT_DIR"
1085 |         fi
1086 |     fi
1087 | 
1088 |     if [[ ! -e "$PPSS_LOCAL_TMPDIR" ]] 
1089 |     then
1090 |         mkdir "$PPSS_LOCAL_TMPDIR"
1091 |     fi
1092 | 
1093 |     if [[ ! -e "$PPSS_LOCAL_OUTPUT" ]] 
1094 |     then
1095 |         mkdir "$PPSS_LOCAL_OUTPUT"
1096 |     fi
1097 | 
1098 |     if [[ ! -e "$PPSS_NODE_STATUS" ]] 
1099 |     then
1100 |         mkdir -p "$PPSS_NODE_STATUS"
1101 |     fi
1102 | 
1103 | }
1104 | 
1105 | upload_status () {
1106 | 
1107 |     if [[ -e "$NODE_STATUS_FILE" ]] 
1108 |     then
1109 |         scp -q -o GlobalKnownHostsFile=./known_hosts -i ppss-key.dsa $NODE_STATUS_FILE $USER@$SSH_SERVER:$PPSS_HOME_DIR/$PPSS_NODE_STATUS/ 
1110 |         if [[ "$?" == "0" ]] 
1111 |         then
1112 |             log DEBUG "Uploaded status to server ok."
1113 |         else
1114 |             log DEBUG "Uploaded status to server failed."
1115 |         fi
1116 |     else
1117 |         log DEBUG "Status file not found thus not uploaded."
1118 |     fi
1119 | }
1120 | 
1121 | set_status () {
1122 | 
1123 |     if [[ ! -z "$SSH_SERVER" ]] 
1124 |     then
1125 |         STATUS="$1"
1126 |         if [[ -e "$LIST_OF_PROCESSED_ITEMS" ]] 
1127 |         then
1128 |             NO_PROCESSED=$(wc -l "$LIST_OF_PROCESSED_ITEMS" | awk '{ print $1 }' )
1129 |         else
1130 |             NO_PROCESSED="0"
1131 |         fi
1132 |         NODE=$(cat $PPSS_DIR/$NODE_ID)
1133 |         FAILED="$2"
1134 | 
1135 |         if [[ -z "$FAILED" ]] 
1136 |         then
1137 |             FAILED=0
1138 |         fi
1139 |        
1140 |         echo "$NODE $HOSTNAME $STATUS $NO_PROCESSED" "$FAILED" > "$NODE_STATUS_FILE"
1141 |         upload_status
1142 |     fi
1143 | }
1144 | 
1145 | check_status () {
1146 | 
1147 |     ERROR="$1"
1148 |     FUNCTION="$2"
1149 |     MESSAGE="$3"
1150 | 
1151 |     if [[ ! "$ERROR" == "0" ]] 
1152 |     then
1153 |         log DSPLY "$FUNCTION - $MESSAGE"
1154 |         set_status ERROR 
1155 |         cleanup 
1156 |         exit "$ERROR"
1157 |     fi
1158 | }
1159 | 
1160 | erase_ppss () {
1161 | 
1162 |     SSH_SOCKET="ppss_ssh_socket-$NODE"
1163 | 
1164 |     SSH_OPTS_NODE="-o BatchMode=yes -o ControlPath=$SSH_SOCKET \
1165 |                            -o GlobalKnownHostsFile=./known_hosts \
1166 |                            -o ControlMaster=auto \
1167 |                            -o Cipher=blowfish \
1168 |                            -o ConnectTimeout=5 "
1169 |     
1170 |     echo "Are you realy sure you want to erase PPSS from all nodes!? (YES/NO)"
1171 |     read YN
1172 | 
1173 |     if [[ "$YN" == "yes" ]] || [[ "$YN" == "YES" ]] 
1174 |     then
1175 |         for NODE in $(cat $NODES_FILE)
1176 |         do
1177 |             log DSPLY "Erasing PPSS homedir $PPSS_HOME_DIR from node $NODE."
1178 |             ssh -q $SSH_KEY $SSH_OPTS_NODE $USER@$NODE "rm -rf $PPSS_HOME_DIR"
1179 |         done
1180 |     else
1181 |         log DSPLY "Aborting.."
1182 |     fi
1183 | }
1184 | 
1185 | stack_push_tmp () {
1186 | 
1187 |     TMP1="$1"
1188 | 
1189 |     if [[ -z "$TMP_STACK" ]] 
1190 |     then
1191 |         TMP_STACK="$TMP1"
1192 |     else
1193 |         TMP_STACK="$TMP_STACK"$'\n'"$TMP1"
1194 |     fi
1195 | }   
1196 | 
1197 | stack_push () {
1198 | 
1199 |     line="$1"
1200 | 
1201 |     if [[ -z "$STACK" ]] 
1202 |     then
1203 |         STACK="$line"
1204 |     else
1205 |         STACK="$line"$'\n'"$STACK"
1206 |     fi
1207 | }   
1208 | 
1209 | unprocessed_stack_push () {
1210 | 
1211 |     line="$1"
1212 | 
1213 |     if [[ -z "$PROCESSED_ITEMS" ]] 
1214 |     then
1215 |         UNPROCESSED_ITEMS="$line"
1216 |     else
1217 |         UNPROCESSED_ITEMS="$line"$'\n'"$UNPROCESSED_ITEMS"
1218 |     fi
1219 | }
1220 | 
1221 | processed_stack_push () {
1222 | 
1223 |     line="$1"
1224 | 
1225 |     if [[ -z "$PROCESSED_ITEMS" ]] 
1226 |     then
1227 |         PROCESSED_ITEMS="$line"
1228 |     else
1229 |         PROCESSED_ITEMS="$line"$'\n'"$PROCESSED_ITEMS"
1230 |     fi
1231 | }
1232 | 
1233 | stack_pop () {
1234 | 
1235 |     TMP_STACK=""
1236 |     i=0
1237 |     tmp=""
1238 |     for x in $STACK
1239 |     do
1240 |         if [[ "$i" == "0" ]] 
1241 |         then
1242 |             tmp="$x"
1243 |         else
1244 |             stack_push_tmp "$x"
1245 |         fi
1246 |         ((i++))
1247 |     done
1248 |     STACK="$TMP_STACK"
1249 |     REGISTER="$tmp"
1250 |     if [[ -z "$REGISTER" ]] 
1251 |     then
1252 |         return 1
1253 |     else
1254 |         return 0
1255 |     fi
1256 | }     
1257 | 
1258 | is_screen_installed () {
1259 | 
1260 |     if [[ "$DISABLE_SCREEN_TEST" == "1" ]]    
1261 |     then
1262 |         return 0
1263 |     fi
1264 | 
1265 |     NODE="$1"
1266 |     ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "screen -m -D -S test ls" > /dev/null 2>&1 
1267 |     if [[ ! "$?" == "0" ]] 
1268 |     then
1269 |         log ERROR "The 'Screen' command may not be installed on node $NODE."
1270 |         log ERROR "Or some other SSH related error occurred." 
1271 |         return 1
1272 |     else
1273 |         log DEBUG "'Screen' is installed on node $NODE."
1274 |     fi
1275 | }
1276 | 
1277 | deploy () {
1278 | 
1279 |     NODE="$1"
1280 | 
1281 |     SSH_SOCKET="ppss_ssh_socket-$NODE"
1282 | 
1283 |     SSH_OPTS_NODE="-o BatchMode=yes -o ControlPath=$SSH_SOCKET \
1284 |                            -o GlobalKnownHostsFile=./known_hosts \
1285 |                            -o ControlMaster=auto \
1286 |                            -o Cipher=blowfish \
1287 |                            -o ConnectTimeout=5 "
1288 | 
1289 |     SSH_OPTS_SLAVE="-o BatchMode=yes -o ControlPath=$SSH_SOCKET \
1290 |                            -o GlobalKnownHostsFile=./known_hosts \
1291 |                            -o ControlMaster=no \
1292 |                            -o Cipher=blowfish \
1293 |                            -o ConnectTimeout=5 "
1294 | 
1295 |     ERROR=0
1296 |     set_error () {
1297 | 
1298 |         if [[ "$ERROR" == "1" ]] 
1299 |         then 
1300 |             ERROR=1 
1301 |         elif [[ ! "$1" == "0" ]] 
1302 |         then
1303 |             ERROR=1
1304 |         fi
1305 |     }
1306 |     if [[ ! -e "$SSH_SOCKET" ]] 
1307 |     then
1308 |         ssh -q -N $SSH_OPTS_NODE $SSH_KEY $USER@$NODE &
1309 |         SSH_PID=$!
1310 |     fi
1311 | 
1312 |     is_screen_installed "$NODE"
1313 | 
1314 |     KEY=$(echo $SSH_KEY | cut -d " " -f 2) 
1315 | 
1316 |     ssh -q  $SSH_OPTS_SLAVE $SSH_KEY $USER@$NODE "cd ~ && mkdir -p $PPSS_HOME_DIR && mkdir -p $PPSS_HOME_DIR/$JOB_LOG_DIR && mkdir -p $PPSS_HOME_DIR/ITEM_LOCK_DIR >> /dev/null 2>&1" 
1317 |     set_error $?
1318 |     ssh -q $SSH_OPTS_SLAVE $SSH_KEY $USER@$NODE "cd ~ && cd $PPSS_HOME_DIR && cd $PPSS_DIR && echo $NODE > $NODE_ID"
1319 |     set_error $?
1320 |     scp -q $SSH_OPTS_SLAVE $SSH_KEY $0 $USER@$NODE:~/$PPSS_HOME_DIR
1321 |     set_error $?
1322 |     scp -q $SSH_OPTS_SLAVE $SSH_KEY $KEY $USER@$NODE:~/$PPSS_HOME_DIR
1323 |     set_error $?
1324 |     scp -q $SSH_OPTS_SLAVE $SSH_KEY $CONFIG $USER@$NODE:~/$PPSS_HOME_DIR
1325 |     set_error $?
1326 |     scp -q $SSH_OPTS_SLAVE $SSH_KEY known_hosts $USER@$NODE:~/$PPSS_HOME_DIR
1327 |     set_error $?
1328 | 
1329 |     if [[ ! -z "$SCRIPT" ]] 
1330 |     then
1331 |         scp -q $SSH_OPTS_SLAVE $SSH_KEY $SCRIPT $USER@$NODE:~/$PPSS_HOME_DIR
1332 |         set_error $?
1333 |     fi
1334 | 
1335 |     if [[ ! -z "$INPUT_FILE" ]] 
1336 |     then
1337 |         scp -q $SSH_OPTS_SLAVE $SSH_KEY $INPUT_FILE $USER@$NODE:~/$PPSS_HOME_DIR
1338 |         set_error $?
1339 |     fi
1340 | 
1341 |     if [[ "$ERROR" == "0" ]] 
1342 |     then
1343 |         log DSPLY  "PPSS installed on node $NODE."
1344 |     else
1345 |         log DSPLY  "PPSS failed to install on $NODE."
1346 |     fi
1347 | 
1348 |     kill $SSH_PID
1349 | }
1350 | 
1351 | deploy_ppss () {
1352 | 
1353 |     if [[ -z "$NODES_FILE" ]] || [[ ! -e "$NODES_FILE" ]] 
1354 |     then
1355 |         log ERROR "No file containing list of nodes missing / not specified."
1356 |         set_status ERROR
1357 |         cleanup 
1358 |         exit 1
1359 |     fi
1360 | 
1361 |     exec_cmd "mkdir -p $PPSS_HOME_DIR/$PPSS_NODE_STATUS"
1362 |     
1363 |     KEY=$(echo $SSH_KEY | cut -d " " -f 2) 
1364 |     if [[ -z "$KEY" ]] || [[ ! -e "$KEY" ]] 
1365 |     then
1366 |         log ERROR "Private SSH key $KEY not found."
1367 |         set_status "ERROR"
1368 |         cleanup 
1369 |         exit 1
1370 |     fi
1371 | 
1372 |     if [[ ! -e "$SCRIPT" && ! -z "$SCRIPT" ]] 
1373 |     then
1374 |         log ERROR "Script $SCRIPT not found."
1375 |         set_status "ERROR"
1376 |         cleanup
1377 |         exit 1
1378 |     fi
1379 | 
1380 |     INSTALLED_ON_SSH_SERVER=0
1381 |     for NODE in $(cat $NODES_FILE) 
1382 |     do
1383 |         deploy "$NODE" &
1384 |         if [[ "$ARCH" == "SunOS" ]] 
1385 |         then
1386 |             sleep 1
1387 |         else
1388 |             sleep 0.1
1389 |         fi
1390 |         if [[ "$NODE" == "$SSH_SERVER" ]] 
1391 |         then   
1392 |             INSTALLED_ON_SSH_SERVER=1
1393 |         fi
1394 |     done
1395 | 
1396 |     if [[ "$INSTALLED_ON_SSH_SERVER" == "0" ]] 
1397 |     then
1398 |         log DEBUG "SSH SERVER $SSH_SERVER is not a node."
1399 |     else
1400 |         log DEBUG "SSH SERVER $SSH_SERVER is also a node."
1401 |     fi
1402 | }
1403 | 
1404 | start_ppss_on_node () {
1405 | 
1406 |     NODE="$1"
1407 |     log DSPLY "Starting PPSS on node $NODE."
1408 |     ssh $SSH_KEY $USER@$NODE -o ConnectTimeout=5 -o GlobalKnownHostsFile=./known_hosts "cd $PPSS_HOME_DIR ; screen -d -m -S PPSS ~/$PPSS_HOME_DIR/$0 node --config ~/$PPSS_HOME_DIR/$CONFIG" 
1409 |     if [[ ! "$?" == "0" ]] 
1410 |     then
1411 |         log ERROR  "|P|P|S|S| failed to start on node $NODE."
1412 |     fi
1413 | }
1414 | 
1415 | init_ssh_server_socket () {
1416 | 
1417 |         if [[ ! -e "$SSH_SOCKET" ]] 
1418 |         then
1419 |             DIR=$(dirname $SSH_SOCKET)
1420 |             mkdir -p "$DIR"
1421 |         fi
1422 | }
1423 | 
1424 | test_server () {
1425 | 
1426 |     # Testing if the remote server works as expected.
1427 |     if [[ ! -z "$SSH_SERVER" ]] 
1428 |     then 
1429 |         init_ssh_server_socket
1430 |         
1431 |         exec_cmd "date >> /dev/null"
1432 |         check_status "$?" "$FUNCNAME" "Server $SSH_SERVER could not be reached"
1433 | 
1434 |         ssh -N -M $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER &
1435 |         SSH_MASTER_PID="$!"
1436 |         log DEBUG "SSH Master pid is $SSH_MASTER_PID"
1437 |         log INFO "Connected to server: $SSH_SERVER"
1438 | 
1439 |         does_file_exist "$PPSS_HOME_DIR/$PPSS_DIR"
1440 |         if [[ ! "$?" == "0" && ! -z "$SSH_SERVER" ]] 
1441 |         then
1442 |             log DEBUG "Remote PPSS home directory $PPSS_HOME_DIR/$PPSS_DIR does not exist. Creating."
1443 |             exec_cmd "mkdir -p $PPSS_HOME_DIR/$PPSS_DIR"
1444 |         fi
1445 |     else
1446 |         log DEBUG "No remote server specified, assuming stand-alone mode."
1447 |     fi
1448 | }
1449 | 
1450 | get_no_of_cpus () {
1451 | 
1452 |     # Use hyperthreading or not?
1453 |     HPT=$1
1454 |     NUMBER=""
1455 | 
1456 |     if [[ -z "$HPT" ]] 
1457 |     then
1458 |         HPT=yes
1459 |     fi
1460 | 
1461 |     got_cpu_info () {
1462 | 
1463 |     ERROR="$1"
1464 |     check_status "$ERROR" "$FUNCNAME" "cannot determine number of cpu cores. Specify with -p." 
1465 | 
1466 |     }
1467 | 
1468 |     if [[ "$HPT" == "yes" ]] 
1469 |     then
1470 |         if [[ "$ARCH" == "Linux" ]] 
1471 |         then
1472 |             NUMBER=$(grep -c ^processor $CPUINFO)
1473 |             got_cpu_info "$?"
1474 | 
1475 |         elif [[ "$ARCH" == "Darwin" ]] 
1476 |         then
1477 |             NUMBER=$(sysctl -a hw | grep -w logicalcpu | awk '{ print $2 }')
1478 |             got_cpu_info "$?"
1479 | 
1480 |         elif [[ "$ARCH" == "FreeBSD" ]] 
1481 |         then
1482 |             NUMBER=$(sysctl hw.ncpu | awk '{ print $2 }')
1483 |             got_cpu_info "$?"
1484 | 
1485 |         elif [[ "$ARCH" == "SunOS" ]] 
1486 |         then
1487 |             NUMBER=$(psrinfo | grep -c on-line)
1488 |             got_cpu_info "$?"
1489 |         else
1490 |             if [[ -e "$CPUINFO" ]] 
1491 |             then
1492 |                 NUMBER=$(grep -c ^processor $CPUINFO)
1493 |                 got_cpu_info "$?"
1494 |             fi 
1495 |         fi
1496 | 
1497 |         if [[ ! -z "$NUMBER" ]] 
1498 |         then
1499 |             log DSPLY "Found $NUMBER logic processors."
1500 |         fi
1501 | 
1502 |     elif [[ "$HPT" == "no" ]] 
1503 |     then
1504 |         log DSPLY "Hyperthreading is disabled."
1505 | 
1506 |         if [[ "$ARCH" == "Linux" ]] 
1507 |         then
1508 |             PHYSICAL=$(grep 'physical id' $CPUINFO)
1509 |             if [[ "$?" ]] 
1510 |             then
1511 |                 PHYSICAL=$(grep 'physical id' $CPUINFO | sort | uniq | wc -l)
1512 |                 if [[ "$PHYSICAL" == "1" ]] 
1513 |                 then
1514 |                     log DSPLY "Found $PHYSICAL physical CPU."
1515 |                 else
1516 |                     log DSPLY "Found $PHYSICAL physical CPUs."
1517 |                 fi
1518 | 
1519 |                 TMP=$(grep 'core id' $CPUINFO) 
1520 |                 if [[ "$?" ]] 
1521 |                 then
1522 |                     log DEBUG "Starting job only for each physical core on all physical CPU(s)."
1523 |                     NUMBER=$(grep 'core id' $CPUINFO | sort | uniq | wc -l) 
1524 |                     log DSPLY "Found $NUMBER physical cores."
1525 |                 else
1526 |                     log DSPLY "Single core processor(s) detected."
1527 |                     log DSPLY "Starting job for each physical CPU."
1528 |                     NUMBER=$PHYSICAL
1529 |                 fi
1530 |             else
1531 |                 log INFO "No 'physical id' section found in $CPUINFO, typical for older cpus."            
1532 |                 NUMBER=$(grep -c ^processor $CPUINFO)
1533 |                 got_cpu_info "$?"
1534 |             fi
1535 |         elif [[ "$ARCH" == "Darwin" ]] 
1536 |         then
1537 |             NUMBER=$(sysctl -a hw | grep -w physicalcpu | awk '{ print $2 }')
1538 |             got_cpu_info "$?"
1539 |         elif [[ "$ARCH" == "FreeBSD" ]] 
1540 |         then
1541 |             NUMBER=$(sysctl hw.ncpu | awk '{ print $2 }')
1542 |             got_cpu_info "$?"
1543 |         else
1544 |             NUMBER=$(cat $CPUINFO | grep "cpu cores" | cut -d ":" -f 2 | uniq | sed -e s/\ //g)
1545 |             got_cpu_info "$?"
1546 |         fi
1547 | 
1548 |     fi
1549 | 
1550 |     if [[ ! -z "$NUMBER" ]] 
1551 |     then
1552 |         MAX_NO_OF_RUNNING_JOBS=$NUMBER
1553 |     else
1554 |         log ERROR "Number of CPUs not obtained."
1555 |         log ERROR "Please specify manually with -p."
1556 |         set_status "ERROR"
1557 |         exit 1
1558 |     fi
1559 | }
1560 | 
1561 | random_delay () {
1562 | 
1563 |     ARGS="$1"
1564 | 
1565 |     if [[ -z "$ARGS" ]] 
1566 |     then
1567 |         log ERROR "$FUNCNAME Function random delay, no argument specified."
1568 |         set_status "ERROR"
1569 |         exit 1
1570 |     fi
1571 | 
1572 |     NUMBER=$RANDOM
1573 | 
1574 |     let "NUMBER %= $ARGS"
1575 |     sleep "$NUMBER"
1576 | }
1577 | 
1578 | escape_item () {
1579 | 
1580 |     TMP="$1"
1581 | 
1582 |     ITEM_ESCAPED=`echo "$TMP" | \
1583 |             sed s/\\ /\\\\\\\\\\\\\\ /g | \
1584 |             sed s/\\'/\\\\\\\\\\\\\\'/g | \
1585 |             sed s/\\|/\\\\\\\\\\\\\\|/g | \
1586 |             sed s/\&/\\\\\\\\\\\\\\&/g | \
1587 |             sed s/\;/\\\\\\\\\\\\\\;/g | \
1588 |             sed s/\>/\\\\\\\\\\>/g | \
1589 |             sed s/\</\\\\\\\\\\</g | \
1590 |             sed s/\(/\\\\\\\\\\(/g | \
1591 |             sed s/\)/\\\\\\\\\\)/g ` 
1592 | }
1593 | 
1594 | download_item () {
1595 | 
1596 |     if [[ ! "$DOWNLOAD_TO_NODE" == "1" ]] || [[ "$VIRTUAL" == "1" ]] 
1597 |     then
1598 |         return 1
1599 |     fi
1600 | 
1601 |     ITEM="$1"
1602 |     VIRTUAL="0"
1603 |     ERR_STATE="0"
1604 |     HASH=""
1605 | 
1606 |     if [[ "$TRAVERSAL" == "1" ]] 
1607 |     then
1608 |         escape_item "$ITEM"
1609 |         does_file_exist "$ITEM_ESCAPED"
1610 |         ERR_STATE="$?"
1611 |         DOWNLOAD_ITEM="$ITEM"
1612 |         HASH=$(echo "$DOWNLOAD_ITEM" | $MD5 | awk '{ print $1 }')
1613 |         LOCAL_DIR="$HASH"
1614 |     else
1615 |         escape_item "$ITEM"
1616 |         does_file_exist "$SRC_DIR/$ITEM_ESCAPED"
1617 |         ERR_STATE="$?"
1618 |         DOWNLOAD_ITEM="$SRC_DIR/$ITEM"
1619 |     fi
1620 | 
1621 |     if [[ "$ERR_STATE" == "0" ]] 
1622 |     then
1623 |         log DEBUG "$FUNCNAME Remote item $ITEM exists"
1624 |         VIRTUAL=0
1625 |         
1626 |     else
1627 |         log DEBUG "$FUNCNAME Remote item $ITEM does NOT exist"
1628 |         VIRTUAL=1
1629 |     fi
1630 | 
1631 |     if [[ "$DOWNLOAD_TO_NODE" == "1" && "$VIRTUAL" == "0" ]] 
1632 |     then
1633 |         log DEBUG "Transfering item $ITEM from source to local disk."
1634 |         if [[ "$SECURE_COPY" == "1" && ! -z "$SSH_SERVER" ]] 
1635 |         then
1636 |             if [[ "$TRAVERSAL" == "1" ]] 
1637 |             then
1638 |                 escape_item "$DOWNLOAD_ITEM" 
1639 |                 mkdir -p "$PPSS_LOCAL_TMPDIR/$LOCAL_DIR"
1640 |                 log DEBUG "$SSH_SERVER:$ITEM_ESCAPED $PPSS_LOCAL_TMPDIR/$LOCAL_DIR"
1641 |                 scp -q $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER:"$ITEM_ESCAPED" ./$PPSS_LOCAL_TMPDIR/"$LOCAL_DIR"
1642 |             else
1643 |                 escape_item "$DOWNLOAD_ITEM" 
1644 |                 log DEBUG "$SSH_SERVER:$ITEM_ESCAPED $PPSS_LOCAL_TMPDIR"
1645 |                 scp -q $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER:"$ITEM_ESCAPED" $PPSS_LOCAL_TMPDIR 
1646 |             fi
1647 |             log DEBUG "Exit code of remote download transfer is $?"
1648 |         else
1649 |             if [[ "$TRAVERSAL" == "1" ]]
1650 |             then
1651 |                 mkdir -p "$PPSS_LOCAL_TMPDIR/$LOCAL_DIR"
1652 |                 cp "$DOWNLOAD_ITEM" "$PPSS_LOCAL_TMPDIR/$LOCAL_DIR" 
1653 |                 log DEBUG "Exit code of local transfer is $?"
1654 |             else
1655 |                 cp "$DOWNLOAD_ITEM" "$PPSS_LOCAL_TMPDIR/" 
1656 |                 log DEBUG "Exit code of local transfer is $?"
1657 |             fi
1658 |         fi
1659 |     else
1660 |         log DEBUG "No transfer of item $ITEM to local workpath."
1661 |     fi
1662 | }
1663 | 
1664 | upload_item () {
1665 | 
1666 |     if [[ ! "$UPLOAD_TO_SERVER" == "1" ]] 
1667 |     then
1668 |         log DEBUG "Upload to server is disabled."
1669 |         return 1
1670 |     fi
1671 |     log DEBUG "Upload from node to server - $1 to $2"
1672 | 
1673 |     LOCAL_SRC_DIR="$1" # the local source directory where the file resides
1674 |     REMOTE_DEST_DIR="$2" # to recreate the directory structure
1675 |     DESTINATION=""
1676 | 
1677 |     if [[ "$TRAVERSAL" == "1" ]] 
1678 |     then 
1679 |         log DEBUG "Recursive copy is enabled."
1680 |         TMP_DESTINATION="$REMOTE_OUTPUT_DIR/$REMOTE_DEST_DIR" 
1681 |     else
1682 |         log DEBUG "Recursive copy is disabled."
1683 |         TMP_DESTINATION="$REMOTE_OUTPUT_DIR"
1684 |     fi
1685 | 
1686 |     exec_cmd "mkdir -p \"$TMP_DESTINATION\""
1687 |     escape_item "$TMP_DESTINATION"
1688 |     DESTINATION="$ITEM_ESCAPED"
1689 | 
1690 |     log DEBUG "Destination == $DESTINATION"
1691 | 
1692 |     log DEBUG "Uploading item $LOCAL_SRC_DIR."
1693 |     if [[ "$SECURE_COPY" == "1" ]] 
1694 |     then
1695 |         log DEBUG "Secure copy is enabled."
1696 |         log DEBUG "Copy contents of $LOCAL_SRC_DIR to server at $DESTINATION"
1697 |         scp $SSH_OPTS $SSH_KEY $LOCAL_SRC_DIR/* $USER@$SSH_SERVER:"$DESTINATION" 
1698 |         ERROR="$?"
1699 |         if [[ ! "$ERROR" == "0" ]] 
1700 |         then
1701 |             log ERROR "Uploading of $LOCAL_SRC_DIR via SCP to $DIR_ESCAPED failed."
1702 |         else
1703 |             log DEBUG "Upload of item $LOCAL_SRC_DIR to $DIR_ESCAPED success" 
1704 |             rm -rf ./"$LOCAL_SRC_DIR"
1705 |         fi
1706 |     else    
1707 |         log DEBUG "Secure copy is disabled - using regular cp."
1708 |         cp "$LOCAL_SRC_DIR"/* "$TMP_DESTINATION"
1709 |         if [[ ! "$?" == "0" ]] 
1710 |         then
1711 |             log DEBUG "ERROR - uploading of $LOCAL_SRC_DIR via CP to $DESTINATION failed."
1712 |         fi
1713 |     fi
1714 | }
1715 | 
1716 | lock_item () {
1717 | 
1718 |     if [[ "$INOTIFY" == "1" && "$DAEMON" == "1" ]] 
1719 |     then
1720 |         #
1721 |         # In daemon mode, there is no risk that processes try to process
1722 |         # the same item. Therefore, locking is not required.
1723 |         #
1724 |         return 0
1725 |     else
1726 |         ITEM="$1"
1727 | 
1728 |         LOCK_FILE_NAME=$(echo "$ITEM" | $MD5 | awk '{ print $1 }')
1729 |         ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
1730 |         log DEBUG "Locking item $ITEM_LOCK_FILE"
1731 |         exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
1732 |         ERROR="$?"
1733 |         return "$ERROR"
1734 |     fi
1735 | }
1736 | 
1737 | get_input_lock () {
1738 | 
1739 |     while true
1740 |     do 
1741 |         exec_cmd "mkdir $INPUT_LOCK >> /dev/null 2>&1 " 
1742 |         if [[ "$?" ]] 
1743 |         then
1744 |             log DEBUG "Input lock is obtained..."
1745 |             break
1746 |         else
1747 |             log DEBUG "Input lock is present...sleeping.."
1748 |             sleep 5
1749 |         fi
1750 |     done
1751 | }
1752 | 
1753 | release_input_lock () {
1754 |     
1755 |     exec_cmd "rm -rf $INPUT_LOCK"
1756 |     if [[ "$?" ]] 
1757 |     then
1758 |         log DEBUG "Input lock was released..."
1759 |         return 0
1760 |     else
1761 |         log ERROR "Input lock was already gone...this should never happen..."
1762 |         return 1
1763 |     fi
1764 | }
1765 | 
1766 | list_all_input_items () {
1767 | 
1768 |     oldIFS=$IFS # save the field separator
1769 |     IFS=$'\n' # new field separator, the end of line  
1770 | 
1771 |     while read line 
1772 |     do
1773 |         echo "$line"
1774 |     done < "$LISTOFITEMS"
1775 |     IFS="$oldIFS"
1776 | }
1777 | 
1778 | remove_processed_items_from_input_file () {
1779 | 
1780 |     # 
1781 |     # This function removes all items that have already been processed.
1782 |     # Processed items have a lock dir in the PPPSS_ITEM_LOCK_DIR.
1783 |     #
1784 | 
1785 |     if [[ -e "$LIST_OF_PROCESSED_ITEMS" ]] 
1786 |     then
1787 |         PROCESSED_ITEMS=$(cat $LIST_OF_PROCESSED_ITEMS)
1788 |     fi
1789 | 
1790 |     log DEBUG "Running $FUNCNAME"
1791 | 
1792 |     if [[ -z "$PROCESSED_ITEMS" ]] 
1793 |     then
1794 |         log DEBUG "Variable processed_items is empty."
1795 |         return 1
1796 |     fi
1797 | 
1798 |     if [[ "$MODE" == "status" ]] 
1799 |     then 
1800 |         log DEBUG "Mode is status."
1801 |         return 1
1802 |     fi
1803 | 
1804 |     if [[ ! -e "$LISTOFITEMS" ]] 
1805 |     then
1806 |         echo "$LISTOFITEMS does not exist!"
1807 |         return 1
1808 |     else
1809 |         SIZE=$(wc -l "$LISTOFITEMS")
1810 |         if [[ "$SIZE" == "0" ]] 
1811 |         then
1812 |             echo "$LISTOFITEMS exists but is empty."
1813 |             return 1
1814 |         fi
1815 |     fi
1816 | 
1817 |     INPUTFILES=$(list_all_input_items)
1818 | 
1819 |     OUT_FILE=$PPSS_DIR/out-file-$RANDOM$RANDOM$RANDOM$RAMDOM
1820 |     LIST_SORTED=$PPSS_DIR/list-sorted-$RANDOM$RANDOM$RANDOM$RANDOM
1821 |     PROCESSED_SORTED=$PPSS_DIR/processed-sorted-$RANDOM$RANDOM$RANDOM$RANDOM
1822 |     cat "$LISTOFITEMS" | sort > "$LIST_SORTED"
1823 |     cat "$LIST_OF_PROCESSED_ITEMS" | sort > "$PROCESSED_SORTED"
1824 |     comm -3 $LIST_SORTED $PROCESSED_SORTED > $OUT_FILE
1825 |     mv "$OUT_FILE" "$LISTOFITEMS"
1826 | }
1827 | 
1828 | get_all_items () {
1829 | 
1830 |     if [[ "$DAEMON" == "1" && "$INOTIFY" == "0" ]]  && [[ "$ENABLE_INPUT_LOCK" == "1" ]] 
1831 |     then
1832 |         get_input_lock
1833 |     fi
1834 | 
1835 |     GLOBAL_COUNTER=1
1836 |     
1837 |     if [[ -e "$LISTOFITEMS" ]] 
1838 |     then
1839 |         rm "$LISTOFITEMS"
1840 |     fi
1841 | 
1842 |     count=0
1843 | 
1844 |     if [[ -z "$INPUT_FILE" ]] 
1845 |     then
1846 |         if [[ ! -z "$SSH_SERVER" ]] # Are we running stand-alone or as a node?"
1847 |         then
1848 |             if [[ "$TRAVERSAL" == "1" ]] 
1849 |             then
1850 |                 $(exec_cmd "find $SRC_DIR/ ! -type d" > "$LISTOFITEMS")
1851 |                 check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
1852 |             else
1853 |                 log DEBUG "Recursion is disabled."
1854 |                 if [[ "$ARCH" != "SunOS" ]]
1855 |                 then
1856 |                     $(exec_cmd "find $SRC_DIR/ -maxdepth 1 ! -type d" > "$LISTOFITEMS")
1857 |                 else
1858 |                     # SWB: -depth takes no arguments.  I think he meant to use
1859 |                     # -maxdepth to limit search to just the source directory
1860 |                     # (But that won't work on Solaris.)  Try the -prune workaround
1861 |                     # Use this to work around a problem using -name with an
1862 |                     # embedded / when SRC_DIR is more than one level deep.
1863 |                     _dir_basename=`basename "$SRC_DIR"`
1864 |                     $(exec_cmd "find \"$SRC_DIR\" ( ! -name \"$_dir_basename\" -o -type f ) -prune ! -type d -print" > "$LISTOFITEMS")
1865 |                 fi
1866 |                 check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
1867 |             fi
1868 | 
1869 |         else 
1870 |             if [[ -e "$SRC_DIR" ]] 
1871 |             then
1872 |                 if [[ "$TRAVERSAL" == "1" ]] 
1873 |                 then
1874 |                     log DEBUG "Recursion is enabled."
1875 |                     $(find "$SRC_DIR"/ ! -type d >> "$LISTOFITEMS") 
1876 |                     check_status "$?" "$FUNCNAME" "Could not list files within local source directory."
1877 |                 else
1878 |                     log DEBUG "Recursion is disabled."
1879 |                     if [[ "$ARCH" != "SunOS" ]]
1880 |                     then
1881 |                         $(find "$SRC_DIR"/ -maxdepth 1 ! -type d >> "$LISTOFITEMS")
1882 |                     else
1883 |                         #
1884 | 		                # SWB: see above depth vs. maxdepth comment and Solaris
1885 |                         # brain-deadness workaround.
1886 | 		                # Use this to work around a problem using -name with an
1887 |                         # embedded / when SRC_DIR is more than one level deep.
1888 |                         _dir_basename=`basename "$SRC_DIR"`
1889 |                         $(find "$SRC_DIR" \( ! -name "$_dir_basename" -o -type f \) -prune ! -type d -print > "$LISTOFITEMS")
1890 |                     fi
1891 |                     check_status "$?" "$FUNCNAME" "Could not list files within local source directory."
1892 |                 fi
1893 |                 if [[ ! -e "$LISTOFITEMS" ]] 
1894 |                 then
1895 |                     log ERROR "Local input file is not created, something is wrong. Bug?"
1896 |                     set_status "ERROR"
1897 |                     cleanup
1898 |                     exit 1
1899 |                 fi
1900 |                 
1901 |             else
1902 |                 ITEMS=""
1903 |             fi
1904 |         fi
1905 |     else  # Using an input file as the source of our items or STDIN.
1906 |         if [[ ! -z "$SSH_SERVER" ]] # Are we running stand-alone or as a slave?"
1907 |         then
1908 |             log DEBUG "Running as node, input file has been pushed (hopefully)."
1909 |         fi
1910 |         if [[ ! -e "$INPUT_FILE" && ! "$INPUT_FILE" == "-" ]] 
1911 |         then
1912 |             log ERROR "Input file $INPUT_FILE does not exist."
1913 |             set_status "ERROR"
1914 |             cleanup
1915 |             exit 1
1916 |         fi
1917 |     
1918 |         if [[ ! "$INPUT_FILE" == "-" ]] 
1919 |         then
1920 |             cp "$INPUT_FILE" "$LISTOFITEMS"
1921 |             check_status "$?" "$FUNCNAME" "Copy of input file failed!"
1922 |         else
1923 |             log DEBUG "Reading from stdin.."
1924 |             while read LINE
1925 |             do
1926 |                 echo "$LINE" >> "$LISTOFITEMS"
1927 |             done
1928 |         fi
1929 | 
1930 |         if [[ ! -e "$LISTOFITEMS" ]] 
1931 |         then
1932 |             log ERROR "Input is empty."
1933 |             infanticide
1934 |             terminate_listener 
1935 |             cleanup
1936 |         fi
1937 |     fi
1938 | 
1939 |     if [[ "$RANDOMIZE" == "1" && "$MODE" != "status" ]] 
1940 |     then
1941 |         log DEBUG "Randomizing input file."
1942 |         IFS_BACK="$IFS"
1943 |         IFS=$'\n'
1944 |         TMP_FILE="$PPSS_DIR/TMP-$RANDOM$RANDOM.txt"
1945 |         for i in $(cat $LISTOFITEMS); do echo "$RANDOM $i"; done | sort | sed -E 's/^[0-9]+ //' > "$TMP_FILE"
1946 |         mv "$TMP_FILE" "$LISTOFITEMS"
1947 |         IFS="$IFS_BACK"
1948 |     else
1949 |         log DEBUG "Randomisation of input file disabled."
1950 |     fi
1951 |  
1952 | 
1953 |     remove_processed_items_from_input_file
1954 | 
1955 |     if [[ "$DAEMON" == "1" ]] 
1956 |     then
1957 |         release_input_lock
1958 |     fi
1959 | 
1960 |     SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
1961 | 
1962 |     if [[ "$SIZE_OF_INPUT" -le "0" && "$DAEMON" == "0" ]] 
1963 |     then
1964 |         log ERROR "Source file/dir seems to be empty."
1965 |         set_status "STOPPED"
1966 |         cleanup 
1967 |         exit 1
1968 |     fi
1969 | }
1970 | 
1971 | are_all_items_locked () {
1972 | 
1973 |     SIZE="$1"
1974 |     NUMBER=$(exec_cmd "ls -1 $ITEM_LOCK_DIR | wc -l")
1975 |     log DEBUG "$NUMBER of $SIZE items are locked."
1976 |     if [[ "$NUMBER" -ge "$SIZE" ]] 
1977 |     then
1978 |         return 0
1979 |     else
1980 |         return 1
1981 |     fi
1982 | }
1983 | 
1984 | get_item () {
1985 | 
1986 |     check_for_interrupt 
1987 | 
1988 |     if [[ "$STOP" == "1" ]] 
1989 |     then
1990 |         log DEBUG "Found stop signal."
1991 |         return 1
1992 |     fi
1993 | 
1994 |     #
1995 |     # Return error if list size is empty.
1996 |     #
1997 |     if [[ -z "$SIZE_OF_INPUT" ]] 
1998 |     then
1999 |         log DEBUG "Got no size of input..."
2000 |         return 1
2001 |     fi
2002 | 
2003 |     #
2004 |     # Return error if the list is empty.
2005 |     #
2006 |     if [[ "$SIZE_OF_INPUT" -le "0" ]] 
2007 |     then
2008 |         return 1
2009 |     fi
2010 | 
2011 |     #
2012 |     # Check if all items have been processed.
2013 |     #
2014 |     if [[ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]] 
2015 |     then
2016 |         log DEBUG "Counter $GLOBAL_COUNTER is greater than sizeof input $SIZE_OF_INPUT."
2017 |         return 1
2018 |     fi
2019 | 
2020 |     #
2021 |     # Quit if all items have been locked.
2022 |     #
2023 |     if are_all_items_locked "$SIZE_OF_INPUT"
2024 |     then
2025 |         log DEBUG "All items have been locked."
2026 |         return 1
2027 |     else
2028 |         log DEBUG "There are still unlocked items."
2029 |     fi
2030 |     
2031 | 
2032 |     ITEM="$(sed -n $GLOBAL_COUNTER\p $LISTOFITEMS)"
2033 | 
2034 |     if [[ -z "$ITEM" ]] 
2035 |     then
2036 |         log DEBUG "Item was emtpy..."
2037 |         ((GLOBAL_COUNTER++))
2038 |         get_item
2039 |     else
2040 |         ((GLOBAL_COUNTER++))
2041 | 
2042 |         if [[ ! -z "$SSH_SERVER" ]]  || [[ "$LOCAL_LOCKING" == "1" ]] 
2043 |         then
2044 |             lock_item "$ITEM"
2045 |             LOCK="$?"
2046 |             if [[ ! "$LOCK" == "0" ]] 
2047 |             then
2048 |                 log DEBUG "Item $ITEM is locked."
2049 |                 get_item
2050 |             else
2051 |                 log DEBUG "Got lock on $ITEM"
2052 |                 return 0
2053 |             fi
2054 |         else
2055 |             return 0
2056 |         fi
2057 |     fi
2058 | }
2059 | 
2060 | start_new_worker () {
2061 | 
2062 |     #
2063 |     # This function kicks the listener to start a worker process.
2064 |     # 
2065 |     if ! are_we_sourced
2066 |     then
2067 |             echo "$START_KEY" >> "$FIFO"
2068 |             return $?
2069 |     fi
2070 | }
2071 | 
2072 | stop-ppss () {
2073 | 
2074 |     STOP_PPSS=$(get_time_in_seconds)
2075 |     elapsed "$START_PPSS" "$STOP_PPSS"
2076 |     log DSPLY "$PROCESSING_TIME"
2077 | }
2078 | 
2079 | elapsed () {
2080 | 
2081 |     BEFORE="$1"
2082 |     AFTER="$2"
2083 | 
2084 |     ELAPSED="$(expr $AFTER - $BEFORE)"
2085 | 
2086 |     REMAINDER="$(expr $ELAPSED % 3600)"
2087 |     HOURS="$(expr $(expr $ELAPSED - $REMAINDER) / 3600)"
2088 | 
2089 |     SECS="$(expr $REMAINDER % 60)"
2090 |     MINS="$(expr $(expr $REMAINDER - $SECS) / 60)"
2091 | 
2092 |     PROCESSING_TIME=$(printf "Total processing time (hh:mm:ss): %02d:%02d:%02d" $HOURS $MINS $SECS)
2093 | }
2094 | 
2095 | mail_on_error () {
2096 | 
2097 |     ITEM="$1"
2098 |     LOGFILE="$2"
2099 | 
2100 |     if [[ "$MAIL_ON_ERROR" == "1" ]] 
2101 |     then
2102 |         cat "$LOGFILE" | mail -s "$HOSTNAME - PPSS: procesing failed for item." "$EMAIL" 
2103 |         if [[ "$?" == "0" ]] 
2104 |         then
2105 |             log DEBUG "Error mail sent."
2106 |         else
2107 |             log ERROR "Sending of error email failed."
2108 |         fi
2109 |     fi
2110 | }
2111 | 
2112 | 
2113 | add_item_to_failed_list () {
2114 | 
2115 |     ITEM="$1"
2116 |     get_input_lock
2117 |     echo "$ITEM" >> "$FAILED_ITEMS" 
2118 |     release_input_lock
2119 | }
2120 | 
2121 | commando () {
2122 | 
2123 |     #
2124 |     # This function will start a chain reaction of events.
2125 |     #
2126 |     # The commando executes a command on an item and, when finished,
2127 |     # executes the start_new_worker. This function selects a new
2128 |     # item and sends it to the fifo. The listener process receives
2129 |     # the item and excutes this commando function on the item.
2130 |     # So in essence, the commando function keeps calling itself 
2131 |     # indirectly until no items are left. This will form a single
2132 |     # working queue. By executing multiple start_new_worker 
2133 |     # functions based on the CPU cores available, parallel processing
2134 |     # is achieved, with a queue for each core.
2135 |     #
2136 |     ERR_STATE=0
2137 |     VIRTUAL=0
2138 | 
2139 |     #
2140 |     # This code tests if the item exist (is physical or virtuel)
2141 |     # Example: a file is physical, a URL is virtual.
2142 |     #
2143 |     ITEM="$1"
2144 |     ORIG_ITEM="$ITEM"
2145 | 
2146 |     #log DEBUG "$FUNCNAME is processing item $ITEM"
2147 | 
2148 |     if [[ "$TRAVERSAL" == "1" ]] 
2149 |     then
2150 |         escape_item "$ITEM"
2151 |         does_file_exist "$ITEM_ESCAPED" # The item contains the full path.
2152 |         ERR_STATE="$?"
2153 |     else
2154 |         escape_item "$ITEM"
2155 |         does_file_exist "$SRC_DIR/$ITEM_ESCAPED" # The item is only the file name itself.
2156 |         ERR_STATE="$?"
2157 |     fi
2158 | 
2159 |     #
2160 |     # If recursion is used, a file name of an item may not be unique.
2161 |     # The same filename can be used for files in differen directories.
2162 |     # Therefore, the output directory must reflect the original directory
2163 |     # structure. If recursion is not used, this is not necessary.
2164 |     #
2165 |     # Further more, if the item is not a real world object but a string of sort
2166 |     # the item is considered virtual. No recursion. 
2167 |     #
2168 |     if [[ "$ERR_STATE" == "0" ]] 
2169 |     then
2170 |         VIRTUAL="0"
2171 |         if [[ "$TRAVERSAL" == "1" ]] 
2172 |         then
2173 |             ITEM_DIR_NAME=$(dirname "$ITEM" | sed s:$SRC_DIR::g)
2174 |             ITEM_BASE_NAME=$(basename "$ITEM")
2175 |             HASH=$(echo "$ITEM" | $MD5 | awk '{ print $1 }')
2176 |             if [[ "$UPLOAD_TO_SERVER" == "1" ]] 
2177 |             then
2178 |                 OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$HASH"
2179 |             else
2180 |                 if [[ -z "$REMOTE_OUTPUT_DIR" ]] 
2181 |                 then
2182 |                     OUTPUT_DIR="$PPSS_LOCAL_OUTPUT"
2183 |                 else
2184 |                     OUTPUT_DIR="$REMOTE_OUTPUT_DIR/$ITEM_DIR_NAME"
2185 |                 fi
2186 |             fi
2187 |         else
2188 |             ITEM_DIR_NAME="$SRC_DIR"
2189 |             ITEM_BASE_NAME="$ITEM"
2190 |             escape_item "$ITEM_BASE_NAME"
2191 |             if [[ "$UPLOAD_TO_SERVER" == "1" ]] 
2192 |             then
2193 |                 OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_ESCAPED"
2194 |             else
2195 |                 OUTPUT_DIR="$REMOTE_OUTPUT_DIR"
2196 |             fi
2197 |         fi
2198 |     else
2199 |         VIRTUAL="1"
2200 |         ITEM_DIR_NAME=""
2201 |         ITEM_BASE_NAME="$ITEM"
2202 |         escape_item "$ITEM_BASE_NAME"
2203 |         if [[ "$UPLOAD_TO_SERVER" == "1" ]] 
2204 |         then
2205 |             OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_ESCAPED"
2206 |         else
2207 |             if [[ -z "$REMOTE_OUTPUT_DIR" ]] 
2208 |             then
2209 |                 OUTPUT_DIR="$PPSS_LOCAL_OUTPUT"
2210 |             else
2211 |                 OUTPUT_DIR="$REMOTE_OUTPUT_DIR"
2212 |             fi
2213 |         fi
2214 |     fi
2215 | 
2216 |     OUTPUT_FILE="$ITEM_BASE_NAME"
2217 | 
2218 |     #
2219 |     # Decide if an item must be transfered from server to the node.
2220 |     # or be processed in-place (NFS / SMB mount?)
2221 |     #
2222 |     if [[ "$DOWNLOAD_TO_NODE" == "0" ]] 
2223 |     then
2224 |         if [[ "$VIRTUAL" == "1" ]] 
2225 |         then
2226 |             log DEBUG "Item is virtual, thus not downloading."
2227 |         else
2228 |             log DEBUG "Using item straight from the server, no copy."
2229 |             if [[ "$TRAVERSAL" == "0" ]] 
2230 |             then
2231 |                 ITEM="$SRC_DIR/$ITEM"
2232 |             else
2233 |                 ITEM="$ITEM"
2234 |             fi
2235 |         fi
2236 |     else
2237 |         download_item "$ITEM"
2238 |         if [[ "$TRAVERSAL" == "1" ]] 
2239 |         then
2240 |             ITEM="$PPSS_LOCAL_TMPDIR/$HASH/$ITEM_BASE_NAME"
2241 |         else
2242 |             ITEM="$PPSS_LOCAL_TMPDIR/$ITEM_BASE_NAME"
2243 |         fi
2244 |     fi
2245 |     
2246 |     #
2247 |     # Create the log file containing the output of the command.
2248 |     #
2249 |     if [[ "$USE_MD5" == "1" ]] 
2250 |     then
2251 |         LOG_FILE_NAME=$(echo "$ITEM" | $MD5 | awk '{ print $1 }')
2252 |     else
2253 |         if [[ "$ARCH" != "SunOS" ]]
2254 |         then
2255 |             LOG_FILE_NAME=$(echo "$ITEM" | sed s/[^[:alnum:]]/_/g)
2256 |         else
2257 |             LOG_FILE_NAME=$(echo "$ITEM" | perl -p -e 'chomp;s/\W/_/g')
2258 |         fi
2259 |     fi
2260 |     ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
2261 | 
2262 |     if [[ -e "$ITEM_LOG_FILE" && "$DISABLE_SKIPPING" == "0" ]] 
2263 |     then
2264 |         log DEBUG "Item is already processed, skipping..."  
2265 |         start_new_worker
2266 |         return 0
2267 |     fi
2268 | 
2269 |     #
2270 |     # Create the local output directory.
2271 |     #
2272 |     if [[ ! -z "$OUTPUT_DIR" ]] 
2273 |     then
2274 |         log DEBUG "Creating local output dir $OUTPUT_DIR"
2275 |         mkdir -p "$OUTPUT_DIR"
2276 |     fi    
2277 |     
2278 |     
2279 |     ERROR=""
2280 |     #
2281 |     # Some formatting of item log files. 
2282 |     #
2283 |     DATE=$(date +%b\ %d\ %H:%M:%S)
2284 |     echo "===== PPSS Item Log File =====" > "$ITEM_LOG_FILE"
2285 |     echo -e "Host:\t\t$HOSTNAME" >> "$ITEM_LOG_FILE"
2286 |     echo -e "Process:\t$PID" >> "$ITEM_LOG_FILE"
2287 |     echo -e "Item:\t\t$ITEM" >> "$ITEM_LOG_FILE"
2288 |     echo -e "Start date:\t$DATE" >> "$ITEM_LOG_FILE"
2289 |     echo -e "" >> "$ITEM_LOG_FILE"
2290 |     
2291 | #-->#
2292 | #--># The actual execution of the command as specified by
2293 | #--># the -c option.
2294 | #-->#
2295 | 
2296 |     # Check for "$ITEM" or "${ITEM" in the command line.
2297 |     # ${ITEM} allows the usage of string operations.
2298 | 
2299 |     BEFORE=$(get_time_in_seconds)
2300 |     # SWB: Solaris default grep (/usr/bin/grep) doesn't support -E
2301 |     if [[ "$ARCH" == "SunOS" ]]
2302 |     then
2303 |         echo "$COMMAND" | /usr/xpg4/bin/grep -E -i '\$\{?ITEM' >> /dev/null 2>&1
2304 |         RETVAL="$?"
2305 |     else
2306 |         echo "$COMMAND" | grep -E -i '\$\{?ITEM' >> /dev/null 2>&1
2307 |         RETVAL="$?"
2308 |     fi
2309 |     if [[ "$RETVAL" == "0" ]] 
2310 |     then 
2311 |         eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1
2312 |         ERROR="$?"
2313 |         MYPID="$!"
2314 |     else
2315 |         eval '$COMMAND"$ITEM" >> "$ITEM_LOG_FILE" 2>&1'
2316 |         ERROR="$?"
2317 |         MYPID="$!"
2318 |     fi
2319 |     AFTER=$(get_time_in_seconds)
2320 | 
2321 |     echo -e "" >> "$ITEM_LOG_FILE"
2322 | 
2323 |     # Some error logging. Success or fail.
2324 |     if [[ ! "$ERROR" == "0" ]] 
2325 |     then
2326 |        mail_on_error "$ITEM" "$ITEM_LOG_FILE"
2327 |        log DEBUG "Processing Item $ITEM failed."
2328 |        echo "$FAIL_KEY" >> "$FIFO" 
2329 |        echo -e "Status:\t\tFAILURE" >> "$ITEM_LOG_FILE"
2330 |        add_item_to_failed_list "$ORIG_ITEM"
2331 |     else
2332 |        echo -e "Status:\t\tSUCCESS" >> "$ITEM_LOG_FILE"
2333 |     fi
2334 | 
2335 |     #
2336 |     # If part of a cluster, remove the downloaded item after 
2337 |     # it has been processed and uploaded as not to fill up disk space.
2338 |     #
2339 |     if [[ "$DOWNLOAD_TO_NODE" == "1" ]] 
2340 |     then
2341 |         if [[ -e "$ITEM" ]] 
2342 |         then
2343 |             rm -rf "$ITEM"
2344 |         else        
2345 |             log DEBUG "There is no local file to remove.. strange..."
2346 |         fi
2347 | 
2348 |     fi
2349 | 
2350 |     #
2351 |     # Upload the output file back to the server.
2352 |     #
2353 | 
2354 |     if [[ "$UPLOAD_TO_SERVER" == "1" ]] 
2355 |     then
2356 |         log DEBUG "Upload $OUTPUT_DIR and ITEM DIR NAME IS $ITEM_DIR_NAME"
2357 |         if [[ "$ITEM_DIR_NAME" == "." ]] 
2358 |         then
2359 |             ITEM_DIR_NAME=""
2360 |         fi
2361 |         upload_item "$OUTPUT_DIR" "$ITEM_DIR_NAME" # local output directory / original item dir path
2362 |     else
2363 |         log DEBUG "Uploading disabled."
2364 |     fi
2365 | 
2366 |     #
2367 |     # Upload the log file to the server.
2368 |     #
2369 | 
2370 |     elapsed "$BEFORE" "$AFTER" 
2371 |     echo "$PROCESSING_TIME" >> "$ITEM_LOG_FILE"
2372 |     
2373 |     echo -e "" >> "$ITEM_LOG_FILE"
2374 | 
2375 |     if [[ ! -z "$SSH_SERVER" ]] 
2376 |     then
2377 |         scp -q $SSH_OPTS $SSH_KEY "$ITEM_LOG_FILE" $USER@$SSH_SERVER:$PPSS_HOME_DIR/$JOB_LOG_DIR
2378 |         if [[ ! "$?" ]] 
2379 |         then
2380 |             log DEBUG "Uploading of item log file failed."
2381 |         fi
2382 |     fi
2383 | 
2384 |     start_new_worker
2385 | }
2386 | 
2387 | infanticide () {
2388 |     
2389 |     log DEBUG "Running $FUNCNAME"
2390 | 
2391 |     #
2392 |     # This code is run if ctrl+c is pressed. Very important to prevent
2393 |     # any child processes running after the parent has died. Keeps the system clean.
2394 |     #
2395 |     # This command kills all processes that are related to the master
2396 |     # process as defined by $PID. All processes that have ever been
2397 |     # spawned, although disowned or backgrounded will be killed...
2398 |     #
2399 |     # SWB: Fix ps command on Solaris, uses -e instead of a for all procs
2400 |     # and "comm" instead of "command" in the format specification.
2401 |     if [[ "$ARCH" == "SunOS" ]]
2402 |     then 
2403 |         PROCLIST=$(ps -e -o pid,pgid,ppid,comm | grep [0-9] | grep $PID | grep -v -i grep)
2404 |     else
2405 |         PROCLIST=$(ps a -o pid,pgid,ppid,command | grep [0-9] | grep $PID | grep -v -i grep)
2406 |     fi
2407 |     oldIFS=$IFS # save the field separator
2408 |     IFS=$'\n' # new field separator, the end of line  
2409 |     for x in $(echo "$PROCLIST")
2410 |     do
2411 |         MYPPID=$(echo $x | awk '{ print $3 }')
2412 |         MYPID=$(echo $x | awk '{ print $1 }')
2413 |         if [[ ! "$MYPPID" == "$PID" && ! "$MYPPID" == "1" ]] 
2414 |         then
2415 |             if [[ ! "$MYPID" == "$PID" ]] 
2416 |             then
2417 |                 log DEBUG "Killing process $MYPID"
2418 |                 kill $MYPID >> /dev/null 2>&1
2419 |             else
2420 |                 log DEBUG "Not killing master process..$MYPID.." 
2421 |             fi
2422 |         else
2423 |                 log DEBUG "Not killing listener process. $MYPID.."
2424 |         fi
2425 |     done
2426 |     IFS=$oldIFS
2427 | 
2428 | }
2429 | 
2430 | run_command () {
2431 | 
2432 |     INPUT="$1"
2433 |     
2434 |     log DEBUG "Current active workers is $ACTIVE_WORKERS"
2435 | 
2436 |     if [[ "$ACTIVE_WORKERS" -lt "$MAX_NO_OF_RUNNING_JOBS" ]] 
2437 |     then
2438 |         if [[ -z "$INPUT" ]] 
2439 |         then
2440 |             stack_pop
2441 |             INPUT="$REGISTER"
2442 |         fi
2443 | 
2444 |         log INFO "Now processing $INPUT"
2445 | 
2446 |         if [[ ! -z "$INPUT" && ! -d "$INPUT" ]] 
2447 |         then
2448 |             commando "$INPUT" &
2449 |             MYPID="$!"
2450 |             disown
2451 |             PIDS="$PIDS $MYPID"
2452 |             ((ACTIVE_WORKERS++))
2453 |             log DEBUG "Increasing active workers to $ACTIVE_WORKERS"
2454 |             echo "$INPUT" >> "$LIST_OF_PROCESSED_ITEMS"
2455 |             return 0
2456 |         else
2457 |             log DEBUG "Item is a directory or is empty."
2458 |             return 0
2459 |         fi
2460 |     else
2461 |         log DEBUG "Maximum number of workers are bussy, no more additional workers..."
2462 |     fi
2463 | }
2464 | 
2465 | display_jobs_remaining () {
2466 | 
2467 |     if [[ "$ACTIVE_WORKERS" == "1" && "$QUIET" == "0" ]] 
2468 |     then
2469 |         log PRCNT "One job is remaining.                                      "
2470 |     elif [[ "$QUIET" == "0" ]] 
2471 |     then
2472 |         if [[ "$ACTIVE_WORKERS" == "1" ]] 
2473 |         then
2474 |             echo -en "\n"
2475 |         fi
2476 |         log PRCNT "$((ACTIVE_WORKERS)) jobs are remaining.                                  "
2477 |     fi
2478 | }
2479 | 
2480 | show_eta () {
2481 |     
2482 |     CURRENT_PROCESSED=$((GLOBAL_COUNTER-MAX_NO_OF_RUNNING_JOBS))
2483 |     TOTAL="$SIZE_OF_INPUT"
2484 |     START_TIME=$START_PPSS
2485 |     NOW=$(get_time_in_seconds)
2486 |     MODULO=$((GLOBAL_COUNTER % 5 ))
2487 | 
2488 |     if [[ "$QUIET" == "1" ]] 
2489 |     then
2490 |         return 0
2491 |     fi
2492 | 
2493 |     if [[ "$CURRENT_PROCESSED" -le "0" ]] 
2494 |     then 
2495 |         return 0
2496 |     else
2497 |         if [[ "$MODULO" == "0" ]] 
2498 |         then
2499 |             RUNNING_TIME=$((NOW-START_TIME))
2500 |             if [[ ! "$RUNNING_TIME" -le "0" && ! "$CURRENT_PROCESSED" == "0" ]] && [[ "$CURRENT_PROCESSED" -gt "$MAX_NO_OF_RUNNING_JOBS" ]] 
2501 |             then
2502 |                 TIME_PER_ITEM=$(( RUNNING_TIME / ( CURRENT_PROCESSED - MAX_NO_OF_RUNNING_JOBS ) )) 
2503 |                 log DEBUG "Time per item is $TIME_PER_ITEM seconds."
2504 |                 TOTAL_TIME=$(( ($TIME_PER_ITEM * SIZE_OF_INPUT) + $TIME_PER_ITEM ))
2505 |                 TOTAL_TIME_IN_SECONDS=$((START_TIME+TOTAL_TIME))
2506 |                 if [[ "$ARCH" == "Darwin" ]] 
2507 |                 then
2508 |                     DATE=$(date -r $TOTAL_TIME_IN_SECONDS)
2509 |                 elif [[ "$ARCH" == "SunOS" ]]
2510 |                 then
2511 |                     # SWB: Hack around Solaris' feeble date command
2512 |                     DATE=$(perl -e "print scalar localtime($TOTAL_TIME_IN_SECONDS)")
2513 |                 else
2514 |                     DATE=$(date -d @$TOTAL_TIME_IN_SECONDS)
2515 |                 fi
2516 |                 echo
2517 |                 log DSPLY "ETA: $DATE"
2518 |                 echo -en "\033[2A"
2519 |             fi
2520 |         fi
2521 |     fi
2522 | }
2523 | 
2524 | display_progress () {
2525 | 
2526 |     if [[ "$DAEMON" == "0" ]] 
2527 |     then
2528 |         SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
2529 |         GC=0
2530 |         if [[ ! "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]] 
2531 |         then
2532 |             GC="$GLOBAL_COUNTER"
2533 |         else
2534 |             GC="$SIZE_OF_INPUT"
2535 |         fi
2536 |         PERCENT=$((100 * $GC / $SIZE_OF_INPUT ))
2537 |         if [[ ! "$ACTIVE_WORKERS" == "0" ]] # && [[ "$FINISHED" == "0" ]] 
2538 |         then
2539 |             if [[ "$QUIET" == "0" ]] 
2540 |             then
2541 |                 log PRCNT "$PERCENT% complete. Processed $GC of $SIZE_OF_INPUT. Failed $FAILED_ITEMS_COUNTER/$SIZE_OF_INPUT." 
2542 |                 show_eta
2543 |             elif [[ "$DAEMON" == "0" ]] 
2544 |             then
2545 |                 echo -en "\r$PERCENT% --"
2546 |             fi
2547 | 
2548 |             if [[ "$PERCENT" == "100" ]] 
2549 |             then
2550 |                 if [[ "$QUIET" == "1" ]] 
2551 |                 then
2552 |                     echo
2553 |                 fi
2554 |                 FINISHED=1
2555 |             fi
2556 |         fi
2557 |     fi
2558 | }
2559 | 
2560 | terminate_listener () {
2561 | 
2562 |     GLOBAL_FAILED_COUNTER="$1"
2563 | 
2564 |     log DEBUG "Running $FUNCNAME"
2565 | 
2566 |     if [[ ! -z "$SSH_MASTER_PID" ]] 
2567 |     then
2568 |         kill "$SSH_MASTER_PID" 
2569 |     else
2570 |         log DEBUG "SSH master PID is empty."
2571 |     fi 
2572 | 
2573 |     set_status "STOPPED" "$GLOBAL_FAILED_COUNTER"
2574 |     log DEBUG "Listener stopped."
2575 | 
2576 |     if [[ ! "$PERCENT" == "100" ]] 
2577 |     then
2578 |         echo
2579 | 	    stop-ppss
2580 |         log DSPLY "$FAILED_ITEMS_COUNTER failed items."
2581 |         log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
2582 |     else
2583 |         echo
2584 | 	    stop-ppss
2585 |         log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
2586 |     fi
2587 |     if [[ "$QUIET" == "1" ]] 
2588 |     then
2589 |         echo 
2590 |     fi
2591 | 
2592 |     if [[ ! -z "$EMAIL" ]]  
2593 |     then
2594 |         echo  "|P|P|S|S| job finished." | mail -s "$HOSTNAME - PPSS has finished." "$EMAIL"
2595 |         if [[ ! "$?" == "0" ]] 
2596 |         then 
2597 |             log ERROR "Sending os status mail failed."
2598 |         fi
2599 |     fi
2600 | 
2601 |     echo "$GLOBAL_FAILED_COUNTER" >> "$FIFO_LISTENER"
2602 | 
2603 | }
2604 | 
2605 | inotify_listener () {
2606 |     inotifywait "$SRC_DIR" -m -r -e close -q --format '%w%f' | \
2607 |     while read -r line
2608 |     do
2609 |         if [[ ! -d "$line" ]] 
2610 |         then
2611 |             echo "$line" > "$FIFO"
2612 |         fi
2613 |     done
2614 | }
2615 | 
2616 | is_item_unprocessed () {
2617 | 
2618 |     VAR="$1"
2619 |     STATUS=0
2620 | 
2621 |     if [[ -z "$VAR" ]] 
2622 |     then
2623 |         log DEBUG "$FUNCNAME: something is wrong, no argument received."
2624 |         return 1
2625 |     fi
2626 | 
2627 |     for x in $PROCESSED_ITEMS
2628 |     do
2629 |         if [[ "$x" == "$VAR" ]] 
2630 |         then
2631 |             STATUS=1
2632 |         fi
2633 |     done
2634 | 
2635 |     log DEBUG "Is item $VAR unprocessed: $STATUS"
2636 |     
2637 |     return $STATUS
2638 | }
2639 | 
2640 | is_item_file_and_unmodified () {
2641 | 
2642 |     ITEM="$1"
2643 | 
2644 |     if [[ -e "$ITEM" ]] 
2645 |     then
2646 |         NOW=$(date +%s)
2647 |         FILEDATE=$($STAT "$ITEM")
2648 |         ELAPSED="$(expr $NOW - $FILEDATE)"
2649 |         if [[ "$ELAPSED" -gt "$DAEMON_FILE_AGE" ]] 
2650 |         then
2651 |             log DEBUG "$FUNCNAME File $ITEM is aged $ELAPSED"
2652 |             return 0
2653 |         else
2654 |             log DEBUG "$FUNCNAME File $ITEM too young $ELAPSED"
2655 |             return 1
2656 |         fi
2657 |     else
2658 |         log DEBUG "$FUNCNAME: file does not exist."
2659 |         return 0
2660 |     fi
2661 | }
2662 | 
2663 | process_item_as_daemon () {
2664 | 
2665 |     ITEM="$1"
2666 |     if is_item_unprocessed "$ITEM"
2667 |     then
2668 |         if is_item_file_and_unmodified "$ITEM"
2669 |         then
2670 |             echo "$ITEM" >> "$FIFO"
2671 |             processed_stack_push "$ITEM"
2672 |         else
2673 |             stack_push "$ITEM"
2674 |         fi
2675 |     fi
2676 | }
2677 | 
2678 | daemon_listener () {
2679 | 
2680 |     while true
2681 |     do
2682 |         get_all_items
2683 |         while get_item
2684 |         do
2685 |             process_item_as_daemon "$ITEM"
2686 |         done
2687 |         while stack_pop
2688 |         do 
2689 |             process_item_as_daemon "$REGISTER"  
2690 |         done
2691 |         sleep "$DAEMON_POLLING_INTERVAL"
2692 |     done
2693 | }
2694 | 
2695 | start_daemon_listener () {
2696 | 
2697 |     daemon_listener &
2698 |     MYPID="$!"
2699 |     disown
2700 |     PIDS="$PIDS $MYPID"
2701 | }
2702 | 
2703 | start_inotify_listener () {
2704 | 
2705 |     ACTIVE_WORKERS=0
2706 |     inotify_listener &
2707 |     MYPID="$!"
2708 |     disown
2709 |     PIDS="$PIDS $MYPID"
2710 | }
2711 | 
2712 | start_as_daemon () {
2713 | 
2714 |     if [[ "$DAEMON" == "1" ]] 
2715 |     then
2716 |         log DEBUG "Daemon mode enabled."
2717 |         if [[ "$INOTIFY" == "1" ]] 
2718 |         then
2719 |             log INFO "Linux inotify enabled."
2720 |             start_inotify_listener
2721 |         else
2722 |             start_daemon_listener
2723 |             log INFO "Linux inotify disabled."
2724 |         fi
2725 |     else
2726 |         log DEBUG "Daemon mode disabled."
2727 |     fi
2728 | }
2729 | 
2730 | decrease_active_workers () {
2731 | 
2732 |     if [[ "$ACTIVE_WORKERS" -gt "0" ]] 
2733 |     then
2734 |         ((ACTIVE_WORKERS--))
2735 |     fi
2736 | }
2737 | 
2738 | listen_for_job () {
2739 | 
2740 |     FINISHED=0
2741 |     ACTIVE_WORKERS="$MAX_NO_OF_RUNNING_JOBS"
2742 |     PIDS=""
2743 |     log DEBUG "Listener started."
2744 | 
2745 |     start_as_daemon
2746 | 
2747 |     while read event <& 42
2748 |     do
2749 |         log INFO "Current active workers is $ACTIVE_WORKERS"
2750 | 
2751 |         if [[ "$event" == "$START_KEY" ]] 
2752 |         then
2753 |             decrease_active_workers
2754 | 
2755 |             log DEBUG "Got a 'start-key' event"
2756 | 
2757 |             if [[ "$DAEMON" == "0" ]] 
2758 |             then
2759 |                 if get_item
2760 |                 then
2761 |                     log DEBUG "Got an item, running command..."
2762 |                     run_command "$ITEM"
2763 |                 else
2764 |                     log DEBUG "No more new items..."
2765 |                     if [[ "$ACTIVE_WORKERS" == "0" ]] 
2766 |                     then
2767 |                         display_progress 
2768 |                         break
2769 |                     else
2770 |                         display_jobs_remaining
2771 |                     fi
2772 |                 fi
2773 |             else
2774 |                 log DEBUG "Daemon mode: a worker finished..."
2775 |                 run_command 
2776 |             fi
2777 |         elif [[ "$event" == "$FAIL_KEY" ]] 
2778 |         then
2779 |             ((FAILED_ITEMS_COUNTER++))
2780 |             log DEBUG "An item failed to process. $FAILED_ITEMS_COUNTER"
2781 |         elif [[ "$event" == "$KILL_KEY" ]] 
2782 |         then
2783 |             infanticide
2784 |             break
2785 |         else
2786 |             log DEBUG "Event is an item."
2787 |             stack_push "$event"
2788 |             run_command
2789 |         fi
2790 | 
2791 |         display_progress 
2792 | 
2793 |         set_status "RUNNING" "$FAILED_ITEMS_COUNTER"
2794 | 
2795 |     done
2796 | 
2797 |     terminate_listener "$FAILED_ITEMS_COUNTER"
2798 | }
2799 | 
2800 | start_all_workers () {
2801 | 
2802 |     if [[ "$MAX_NO_OF_RUNNING_JOBS" == "1" ]] 
2803 |     then
2804 |         log DSPLY "Starting one (1) single worker."
2805 |     else
2806 |         log DSPLY "Starting $MAX_NO_OF_RUNNING_JOBS parallel workers."
2807 |     fi
2808 | 
2809 |     if [[ "$DAEMON" == "0" ]] 
2810 |     then
2811 |         log DSPLY "---------------------------------------------------------"
2812 |     elif [[ "$INOTIFY" == "1" ]] 
2813 |     then
2814 |         return 0
2815 |     fi
2816 | 
2817 |     i=0
2818 |     while [[ "$i" -lt "$MAX_NO_OF_RUNNING_JOBS" ]] 
2819 |     do
2820 |         start_new_worker
2821 |         log DEBUG "Starting worker $i"
2822 |         ((i++))
2823 | 
2824 |         if [[ ! "$MAX_DELAY" == "0" ]] 
2825 |         then
2826 |             random_delay "$MAX_DELAY"
2827 |         fi
2828 |     done
2829 | }
2830 | 
2831 | get_status_of_nodes () {
2832 | 
2833 |     RESULT_FILE="$1"
2834 |     FAILED=0
2835 | 
2836 |     ssh -q $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER cat "$PPSS_HOME_DIR/$PPSS_NODE_STATUS/*" > "$RESULT_FILE" 2>&1
2837 |     if [[ ! "$?" == "0" ]] 
2838 |     then
2839 |         log DSPLY "|P|P|S|S| has not been started yet on nodes."
2840 |         return 1
2841 |     fi 
2842 | 
2843 |     IFS=$'\n'
2844 | 
2845 |     for x in $(cat $RESULT_FILE)
2846 |     do
2847 |         IP=$(echo $x | awk '{ print $1 }')
2848 |         HOST=$(echo $x | awk '{ print $2 }')
2849 |         STATUS=$(echo $x | awk '{ print $3 }')
2850 |         RES=$(echo $x | awk '{ print $4 }')
2851 |         FAIL=$(echo $x | awk '{ print $5 }')
2852 |         if [[ -z "$RES" ]] 
2853 |         then
2854 |             RES="0"
2855 |         fi
2856 |         PROCESSED=$((PROCESSED+RES))
2857 |         FAILED=$((FAILED+FAIL))
2858 |         LINE=$(echo "$IP $HOST $RES $FAIL $STATUS" | awk '{ printf ("%-16s %-16s % 8s %6s %7s\n",$1,$2,$3,$4,$5) }')
2859 |         log DSPLY "$LINE"
2860 |     done    
2861 | 
2862 |     log DSPLY "---------------------------------------------------------"
2863 |     LINE=$(echo $PROCESSED $FAILED | awk '{ printf ("Total processed/failed: %18s %6s \n",$1,$2) }')
2864 |     log DSPLY "$LINE"
2865 | 
2866 |     rm "$RESULT_FILE"
2867 | 
2868 | }
2869 | 
2870 | show_status () {
2871 | 
2872 |     . $CONFIG
2873 | 
2874 |     if [[ ! -z "$SSH_KEY" ]] 
2875 |     then
2876 |         SSH_KEY="-i $SSH_KEY"
2877 |     fi
2878 |     get_all_items
2879 | 
2880 |     ITEMS=$(wc -l $LISTOFITEMS | awk '{ print $1 }')
2881 |     
2882 |     if [[ ! -z "$ITEMS" && ! "$ITEMS" == "0" ]] 
2883 |     then
2884 |         PROCESSED=$(exec_cmd "ls -1 $PPSS_HOME_DIR/$ITEM_LOCK_DIR 2>/dev/null | wc -l" 1) 2>&1 >> /dev/null
2885 |         check_status "$?" "Could not get number of processed items."
2886 |         TMP_STATUS=$((100 * $PROCESSED / $ITEMS))
2887 |         log DSPLY "Status:\t\t$TMP_STATUS percent complete."
2888 |     else
2889 |         log DSPLY "Status: UNKNOWN - is PPSS deployed on nodes?"
2890 |     fi
2891 | 
2892 |     if [[ ! -z $NODES_FILE ]] 
2893 |     then
2894 |         TMP_NO=$(cat $NODES_FILE | wc -l)
2895 |         log DSPLY "Nodes:\t $TMP_NO"
2896 |     fi
2897 |     log DSPLY "Items:\t\t$ITEMS"
2898 | 
2899 |     log DSPLY "---------------------------------------------------------"
2900 |     HEADER=$(echo IP-address Hostname Processed Failed Status | awk '{ printf ("%-16s %-15s % 2s %2s %2s\n",$1,$2,$3,$4,$5) }')  
2901 |     log DSPLY "$HEADER"
2902 |     log DSPLY "---------------------------------------------------------"
2903 |     PROCESSED=0
2904 | 
2905 |     get_status_of_nodes "RESULT_FILE"
2906 | 
2907 | }
2908 | 
2909 | main () {
2910 |     
2911 |     case $MODE in
2912 |               node )
2913 |                     create_working_directory
2914 |                     test_server
2915 |                     init_vars
2916 |                     get_all_items
2917 |                     listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & 2>&1 >> /dev/null
2918 |                     LISTENER_PID=$!
2919 |                     start_all_workers
2920 |                     ;;
2921 |              start )
2922 |                     # This option only starts all nodes.
2923 |                     LOGFILE=/dev/null
2924 |                     display_header
2925 |                     if [[ ! -e "$NODES_FILE" ]] 
2926 |                     then
2927 |                         log ERROR "File $NODES with list of nodes does not exist."
2928 |                         set_status "STOPPED"
2929 |                         cleanup
2930 |                         exit 1
2931 |                     else
2932 |                         for NODE in $(cat $NODES_FILE)
2933 |                         do
2934 |                             start_ppss_on_node "$NODE" &
2935 |                         done
2936 |                     fi
2937 |                     cleanup
2938 |                     ;;
2939 |         config )
2940 |                     LOGFILE=/dev/null
2941 |                     display_header
2942 |                     log DSPLY "Generating configuration file $CONFIG"
2943 |                     add_var_to_config PPSS_LOCAL_TMPDIR "$PPSS_LOCAL_TMPDIR"
2944 |                     add_var_to_config PPSS_LOCAL_OUTPUT "$PPSS_LOCAL_OUTPUT"
2945 |                     cleanup
2946 |                     ;;
2947 | 
2948 |         stop )
2949 |                     LOGFILE=/dev/null
2950 |                     display_header
2951 |                     log DSPLY "Stopping PPSS on all nodes."
2952 |                     test_server
2953 |                     exec_cmd "touch $STOP_SIGNAL"
2954 |                     cleanup
2955 |                     ;;
2956 |         pause )
2957 |                     LOGFILE=/dev/null
2958 |                     display_header
2959 |                     log DSPLY "Pausing PPSS on all nodes."
2960 |                     exec_cmd "touch $PAUSE_SIGNAL"
2961 |                     cleanup
2962 |                     ;;
2963 |         continue )
2964 |                     LOGFILE=/dev/null
2965 |                     display_header
2966 |                     if does_file_exist "$STOP_SIGNAL"
2967 |                     then
2968 |                         log DSPLY "Continuing processing, please use $0 start to start PPSS on al nodes."
2969 |                         exec_cmd "rm -f $STOP_SIGNAL"
2970 |                     fi
2971 |                     if does_file_exist "$PAUSE_SIGNAL"
2972 |                     then
2973 |                         log DSPLY "Continuing PPSS on all nodes."
2974 |                         exec_cmd "rm -f $PAUSE_SIGNAL"
2975 |                     fi
2976 |                     cleanup
2977 |                     ;;
2978 |         deploy )
2979 |                     LOGFILE=ppss-deploy.txt
2980 |                     if [[ -e "$LOGFILE" ]] 
2981 |                     then
2982 |                         rm "$LOGFILE"
2983 |                     fi
2984 |             
2985 |                     init_ssh_server_socket
2986 |                     display_header
2987 |                     log DSPLY "Deploying PPSS on nodes. See ppss-deploy.txt for details."
2988 |                     deploy_ppss
2989 |                     wait
2990 |                     cleanup
2991 |                     ;;
2992 |         status )
2993 |                     LOGFILE=/dev/null
2994 |                     display_header
2995 |                     test_server
2996 |                     show_status
2997 |                     cleanup
2998 |                     ;;
2999 |         erase )
3000 |                     LOGFILE=/dev/null
3001 |                     display_header
3002 |                     log DSPLY "Erasing PPSS from all nodes."
3003 |                     erase_ppss
3004 |                     cleanup
3005 |                     ;;
3006 |         kill )
3007 |                     LOGFILE=/dev/null
3008 |                     for x in $(ps ux | grep ppss | grep -v grep | grep bash | awk '{ print $2 }')
3009 |                     do          
3010 |                          kill "$x"
3011 |                     done
3012 |                     cleanup
3013 |                     ;;
3014 | 
3015 |         * )
3016 |                     create_working_directory
3017 |                     display_header
3018 |                     init_vars
3019 |                     get_all_items
3020 |                     listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & 2>&1 >> /dev/null
3021 |                     LISTENER_PID=$!
3022 |                     start_all_workers
3023 |                     ;;
3024 |     esac
3025 | }
3026 | 
3027 | #
3028 | # PPSS can be sourced. This is mainly for testing purposes (unit tests).
3029 | #
3030 | if ! are_we_sourced
3031 | then
3032 | 
3033 |     #
3034 |     # First step: process all command-line arguments.
3035 |     # 
3036 |     process_arguments "$@"
3037 | 
3038 |     #
3039 |     # This command starts the that sets the whole framework in motion.
3040 |     # But only if the file is not sourced.
3041 |     #
3042 |     main
3043 |     #
3044 |     # Exit after all processes have finished.
3045 |     #
3046 |     #wait 
3047 |     if [[ -e "$FIFO_LISTENER" ]] 
3048 |     then
3049 |         while read event <& 43
3050 |         do
3051 |             cleanup
3052 |             exit "$event"
3053 |         done
3054 |     fi
3055 | fi
3056 | 


--------------------------------------------------------------------------------