├── IDEAS.md
├── LICENSE
├── Makefile
├── README.md
├── phases.sh
└── test_script.sh


/IDEAS.md:
--------------------------------------------------------------------------------
 1 | # Features
 2 |  * [ ] ineractive mode: ask before proceeding to the next phase
 3 |  * [ ] syntax checker
 4 |  * [ ] subphases can be implemented using multiple #
 5 |  * [x] be able to say 'run the rest' or 'skip the rest'
 6 |  * [x] to indicate a phase to skip use caret ```^```
 7 |  * [x] specify phase either by name or by sequence
 8 |  * [ ] conditional runs or skips of phases
 9 |  * [ ] either-or phases
10 |  * [x] list phases in a script
11 |  * [x] reporting of phase execution
12 |  * [ ] facility for logging
13 |      * log results of all commands from current run separately into .err and .out files
14 |      * a log with all script runs with timer, working directory variable values
15 |  * [ ] ability to restart from the point it failed on previous run
16 |  * [ ] properly handle ^C and other signals
17 |  * [ ] record the results of earlier runs (create .phases directory?)
18 |  * [ ] check if variables set in the skipped phase are used in the later executed phases
19 |  * [ ] what do I do with the shebang?
20 |  * [ ] warning for comments with no space after #
21 |  * [ ] git-aware (as an option)
22 |  * [ ] non-bash interpreters (LaTeX, psql)
23 |      * other comment character
24 |      * library of supported interpreters
25 |      * specific interpreter for each phase
26 |           * can be specified through shebang option
27 |  * [ ] optional end of phase in the script ```#^phase [name]```
28 |  * [ ] verbosity and logging levels (can be set separately)
29 |      * only errors
30 |      * +phase names
31 |      * +all stderr
32 |      * +filtered stdout
33 |      * +unfiltered stdout
34 |  * [x] most options can be specified for either command line or in #phase
35 |  * [ ] execution only specific lines (until/after/...)
36 |  * [ ] printout what is being executed with optional confirmation
37 |  * [ ] do not allow phases to be run in home directory
38 |  * [ ] truncate logs to a certain size
39 | 
40 | # Ideas for Command-line Options
41 |  * [ ] --unlogged arguments to the preprocessor or specific phase in the script
42 |  * [ ] --log-pipe='command' pipe log output through a specified command before saving
43 |  * [ ] --list-logs list runs information with log sizes
44 |  * [ ] --continue [command [run timestamp or how many run previously]] continue from the phase previous execution stopped
45 |  * [ ] --unlogged command
46 |  * [ ] --clean-logs [hour|day|week|months|all]
47 |  * [ ] --dry-run do not execute or log, only show what will happen
48 |  * [ ] --verbose [commands,output,quite] can be specified for each phase also
49 |  * [ ] --workdir [directory]
50 |  * [ ] --phasesdir [directory]
51 |  * [ ] --[no]check check the script for preprocessor correctness
52 |  * [ ] --restore-env restore environment from saved variables
53 |  * [ ] --prereq [phases] phases that must be executed before the specific phase
54 | 
55 | # Motivation
56 | 
57 |  * if implemented with standard bash facilities the original script becomes too long and it is hard to grasp what it does
58 |  * the goal of phases is somewhat similar to Makefile, maven or ant but  targets are hard to define if they cannot be presented as files
59 | 
60 | # Similar projects
61 | 
62 |  * https://github.com/dymatic/bpp/blob/master/bpp/sample.bpp but it is doing something completely different
63 | 
64 | # Content of the ```.phases``` directory
65 | 
66 |  * .config
67 |  * ```basename=<script>-<host>-<timestamp>```
68 |  * basename.log saves command line arguments, working directory
69 |  * basename.stdout (split in phases with the sequence number?)
70 |  * basename.err
71 |  * ```basename-<seq>-<phase>.env``` for environment variables
72 | 
73 | ## Examples
74 | ### loading data into the database and creating a dump
75 | 
76 | ```
77 |   phases schema_init,generate,^load,create_dump dbgen.sh options
78 |   phases --list [command [dates]]
79 | ```
80 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 Alexandre Sorokine
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | #
2 | # Install the script
3 | #
4 | 
5 | install:
6 | 	cp phases.sh /usr/local/bin/phases
7 | 	chmod +x /usr/local/bin/phases
8 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # phases
 2 | ## Minimally Invasive bash Preprocessor
 3 | 
 4 | I have to maintain several shell scripts that perform long sequences of steps like loading data into a database or running a simulation model with pre- and post-processing of the data.  There are often failures and scripts has to be rerun starting at some point.  With this preprocessor one can define sections of the script that can be selected for execution using command line parameters.  
 5 | 
 6 | ## Install
 7 | 
 8 | On Mac GNU sed and coreutils required.  They should be prefixed with 'g' (e.g., gsed, ghead).  Install them as `brew install coreutils`, do NOT use --with-default-names option.
 9 | 
10 | ## Usage
11 | 
12 | Insert the directive ```#phase <name>``` into your script to split the script into sections.  Use ```test_script.sh``` as an example.  Give some meaningful name for each phase.  
13 | 
14 | Now you can select a phase or several from your script to be executed independently of others.  In the following example only the phase ```load``` will be executed:
15 | 
16 | ```
17 | phases load test_script.sh
18 | ```
19 | 
20 | To execute all phases except for ```load``` use the option ```--skip```:
21 | 
22 | ```
23 | phases --skip load test_script.sh
24 | ```
25 | 
26 | For more help run ```phases``` without arguments.
27 | 
28 | ### Phase List
29 | 
30 | Phase list contains the names of phases to be executed or skipped.  Multiple phases can be specified using comma:
31 | 
32 | ```
33 | phases load,final test_script.sh
34 | ```
35 | 
36 | To specify a range of phases use dash (```-```):
37 | 
38 | ```
39 | phases load-final test_script.sh
40 | ```
41 | 
42 | You can mix phases and ranges:
43 | 
44 | ```
45 | phases load,final-clean test_script.sh
46 | ```
47 | 
48 | To skip a phase use either ```--skip``` option or caret before the phase name in the list.  The following command will execute all phases except for ```load```:
49 | 
50 | ```
51 | phases ^load test_script.sh
52 | ```
53 | 
54 | ### Implied ```init``` Phase
55 | 
56 | Any script starts with an implied phase ```init``` that starts at the first line of the script and ends before the first specified phase.  This phases is intended for various script initialization tasks like setting variables, defining functions, opening session, checks, etc.  This phases will be always executed unless skipped explicitly either with ```--skip``` option or with caret (```^```) in the phase list.
57 | 
58 | ### Details
59 | 
60 |  * In skip mode negated phases are simply ignored.  
61 |  * The order of phases in the list does not matter and the sequence phases will always be preserved as in the original script.
62 | 
63 | ## Other Useful Options
64 | 
65 | To make sure that all specified phases are properly understood run:
66 | 
67 | ```
68 | phases -l test_script.sh
69 | ```
70 | 
71 | Use ```--output``` option to save resulting script into a file instead of executing it.
72 | 
73 | # Limitations
74 | 
75 | 1.  Inside your script do not use file paths relative to the script location
76 | 2.  The names of the phases should not contain punctuation or special characters
77 | 


--------------------------------------------------------------------------------
/phases.sh:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | #
  3 | # phases script
  4 | #
  5 | 
  6 | # exit on errors
  7 | #set -e
  8 | 
  9 | # TODO: be able to override incl/excl phases (phase that is mentioned later gets a precedent)
 10 | # TODO: phase list consistency checks: warn on duplicates and illegal symbols
 11 | # TODO: specify phases by sequence numbers
 12 | # TODO: verify that error handling in the phased scripts properly works
 13 | # TODO: execution of phases in subshell
 14 | 
 15 | # create temporary directory
 16 | PHASES_SCRIPT=$(basename ${BASH_SOURCE[0]})
 17 | 
 18 | # OS-dependent program names
 19 | SED=sed
 20 | GREP=grep
 21 | HEAD=head
 22 | case "$OSTYPE" in
 23 |   darwin*)
 24 |     SED=gsed
 25 |     HEAD=ghead
 26 |     ;;
 27 |   *)
 28 |     ;;
 29 | esac
 30 | 
 31 | # utility functions
 32 | REAL_TAB=$(echo -e "\t")
 33 | 
 34 | function contains () { # modified from http://stackoverflow.com/questions/14366390/bash-if-condition-check-if-element-is-present-in-array
 35 |   # the function echoes -1 if the element is not found or its index otherwise
 36 |   local array="$1[@]"
 37 |   local seeking=$2
 38 |   local in=-1
 39 |   local i=0
 40 |   for element in "${!array}"; do
 41 |     if [[ "$element" == "$seeking" ]]; then
 42 |       in=$i
 43 |       break
 44 |     fi
 45 |     ((i++))
 46 |   done
 47 |   echo $in
 48 | }
 49 | 
 50 | # help message
 51 | function print_help {
 52 |   echo -e "Use:\n\t$PHASES_SCRIPT [-h|--help] | [[-s|--skip] [--] <^init,phase1,[phase2]-[phase4],...> <script.sh> [script_arguments]] | [-l|--list <script.sh>]"
 53 |   echo
 54 |   echo -e "\t-l|--list <script>\tlist phases in the script"
 55 |   echo -e "\t-h|--help\tthis help"
 56 |   echo -e "\t-s|--skip\tskip the phases in the list, run everything else"
 57 |   echo -e "\t-v|--verbose\tincrease verbosity, use more v to be more verbose"
 58 |   echo -e "\t-n|--noclean\t do not remove temporary directory when finished"
 59 |   echo -e "\t-o|--output=<file>\tsave resulting script in a file, do not execute it"
 60 |   exit
 61 | }
 62 | 
 63 | if [ "$#" == 0 ]; then
 64 |   echo ERROR: no arguments
 65 |   print_help
 66 | fi
 67 | 
 68 | # variables from command-line options
 69 | v=0 # verbosity level
 70 | skip=0 # treat the list of phases as include list (0) or skip list (1)
 71 | clean=1 # perform cleanup
 72 | 
 73 | # command line option processing
 74 | while :; do
 75 |   case $1 in
 76 |     -h|--help)
 77 |       print_help
 78 |       ;;
 79 |     -v|--verbose)
 80 |       # verbosity level
 81 |       ((v++))
 82 |       ;;
 83 |     -l|--list)
 84 |       if [ "$#" -lt 2 ]; then
 85 |         echo wrong number of arguments
 86 |         print_help
 87 |       else
 88 |         [[ "$#" -gt 2 ]] && echo Arguments after $2 ignored
 89 |         printf 'phase#\tline\tname\n'
 90 |         ( printf '0\tinit\n'; ${GREP} -n '^#phase ' "$2" | ${SED} -e "s/:#phase /$REAL_TAB/" ) | nl -b a -v 0
 91 |         exit
 92 |       fi
 93 |       ;;
 94 |     -s|--skip)
 95 |       skip=1
 96 |       ;;
 97 |     -n|--noclean)
 98 |       clean=0
 99 |       ;;
100 |     -o|--output)
101 |       if [ -n "$2" ]; then
102 |         outp_scr="$2"
103 |         shift 2
104 |         continue
105 |       else
106 |         printf 'ERROR: "--output" requires a non-empty option argument.\n' >&2
107 |         exit 1
108 |       fi
109 |       ;;
110 |     --output=?*)
111 |       outp_scr=${1#*=}
112 |       ;;
113 |     --output=)         # Handle the case of an empty --output=
114 |       printf 'ERROR: "--output" requires a non-empty option argument.\n' >&2
115 |       exit 1
116 |       ;;
117 |     --)
118 |       shift
119 |       break
120 |       ;;
121 |     -?*)
122 |       printf 'ERROR: Unknown option: %s\n' "$1" >&2
123 |       print_help
124 |       ;;
125 |     *)
126 |       break;
127 |       ;;
128 |   esac
129 | 
130 |   shift
131 | done
132 | 
133 | ((v>0)) && echo Verbosity set to $v
134 | ((v>1)) && echo Using programs: SED=$SED GREP=$GREP HEAD=$HEAD
135 | 
136 | # make sure that the target script exists
137 | TGT_SCRIPT="$2"
138 | SCR_BASENAME=$(basename "$TGT_SCRIPT")
139 | if [[ ! -x "$TGT_SCRIPT" ]]; then
140 |   echo "File $TGT_SCRIPT is not executable or does not exists" >&2
141 |   exit 2
142 | fi
143 | 
144 | # create temporary directory
145 | PHASES_TMPDIR=$(mktemp -d -t "phased-$SCR_BASENAME.XXXXXX")
146 | if [[ -z "$PHASES_TMPDIR" ]]; then
147 |   echo "Unable to create temporary directory $PHASES_TMPDIR" >&2
148 |   exit 1
149 | fi
150 | ((v>0)) && echo Temporary working directory set to $PHASES_TMPDIR
151 | ((clean==0)) && echo Temporary files will be preserved in $PHASES_TMPDIR
152 | 
153 | # cleanup function
154 | function cleanup() {
155 |   if [[ $clean == 1 && -e "$PHASES_TMPDIR" ]]; then
156 |     rm -fr "$PHASES_TMPDIR"
157 |     ((v>0)) && echo Removing temporary working directory $PHASES_TMPDIR
158 |   fi
159 | }
160 | 
161 | trap cleanup EXIT
162 | 
163 | # name of the preprocessed script
164 | PHASED_SCRIPT="$PHASES_TMPDIR/$SCR_BASENAME"
165 | ((v>0)) && echo Output script name: ${PHASED_SCRIPT}
166 | 
167 | # load list of phases from the target script and add phase 'init'
168 | mapfile -t ALL_PHASES < <(${SED} -n "s/^#phase // p" "$TGT_SCRIPT")
169 | ALL_PHASES=('init' ${ALL_PHASES[@]})
170 | ((v>0)) && echo Phases loaded from the script $TGT_SCRIPT: "${ALL_PHASES[@]}"
171 | 
172 | # expand ranges of phases
173 | declare -a PHASE_LIST=()
174 | for phase in ${1//,/ }; do
175 |   if [[ $phase = *-* ]]; then
176 | 
177 |     # save negation
178 |     if [[ $phase == ^* ]]; then
179 |       neg='^'
180 |       phase="${phase#^}"
181 |     else
182 |       neg=''
183 |     fi
184 | 
185 |     # prohibited interval with two open ends
186 |     if [[ $phase == -*- ]]; then
187 |       printf "ERROR: malformed interval %s\n" "$phase" >&2
188 |       exit 8
189 |     fi
190 | 
191 |     # interval with open left end, assign to the 0th element
192 |     [[ ${phase:0:1} == '-' ]] && \
193 |       i0=0 || i0=$(contains ALL_PHASES "${phase%%-*}")
194 |     # interval with open right end, assign to the last elemenet
195 |     [[ ${phase: -1} == '-' ]] && \
196 |       i1=$((${#ALL_PHASES[@]}-1)) || i1=$(contains ALL_PHASES "${phase##*-}")
197 |     ((v>1)) && echo Range $phase indices $i0 $i1
198 | 
199 |     if (( i0 < 0 || i1 < 0 || i0 >= i1 )); then
200 |       printf "ERROR: wrong phase range %s (%s-%s)\n" "$neg$phase" $i0 $i1 >&2
201 |       exit 6
202 |     fi
203 | 
204 |     for ph in "${ALL_PHASES[@]:$i0:((i1-i0+1))}"; do
205 |       PHASE_LIST+=($neg"$ph")
206 |     done
207 |   else
208 |     # verify that there are no syntax errors in the phases list
209 |     if [[ $(contains ALL_PHASES "${phase#^}") == "-1" ]]; then
210 |       echo ERROR: Phase \'$phase\' not contained in $TGT_SCRIPT >&2
211 |       exit 3
212 |     else
213 |       PHASE_LIST+=("$phase")
214 |     fi
215 |   fi
216 | done
217 | ((v>0)) && echo Expanded phase list: "${PHASE_LIST[@]}"
218 | 
219 | # parse list of phases into arrays of skipped and used phases
220 | declare -a USE_PHASES=()
221 | declare -a SKIP_PHASES=()
222 | for phase in "${PHASE_LIST[@]}"; do
223 |   if [[ "$skip" == 0 ]]; then
224 |     if [[ "$phase" == ^* ]]; then
225 |       SKIP_PHASES+=("${phase#^}")
226 |     else
227 |       USE_PHASES+=("$phase")
228 |     fi
229 |   else
230 |     [[ "$phase" != ^* ]] && SKIP_PHASES+=("$phase")
231 |   fi
232 | done
233 | ((v>0)) && echo Use phases: "${USE_PHASES[@]}"
234 | ((v>0)) && echo Skip phases: "${SKIP_PHASES[@]}"
235 | 
236 | # make sure that the same phase is not listed both as use and skip
237 | for phase in "${SKIP_PHASES[@]}"; do
238 |   if [[ $(contains USE_PHASES "${phase}") -ge 0 ]]; then
239 |     echo ERROR: Phase \'$phase\' both included and skipped: $1 >&2
240 |     exit 4
241 |   fi
242 | done
243 | 
244 | # compile the list of phases that will be executed
245 | declare -a EXEC_PHASES=()
246 | for phase in "${ALL_PHASES[@]}"; do
247 |   if [[ $(contains SKIP_PHASES "$phase") == -1 ]]; then
248 |     if [[ "$skip" == 0 ]]; then
249 |       [[ "$phase" == 'init' || $(contains USE_PHASES "$phase") -ge 0 ]] && \
250 |         EXEC_PHASES+=("$phase")
251 |       else
252 |         EXEC_PHASES+=("$phase")
253 |     fi
254 |   fi
255 | done
256 | ((v>0)) && echo Phases to be executed: "${EXEC_PHASES[@]}"
257 | 
258 | # create a script in temporary directory
259 | echo > "$PHASED_SCRIPT"
260 | 
261 | ## extract preamble
262 | if [[ "${EXEC_PHASES[0]}" == 'init' ]]; then
263 |   ((v>0)) && echo Extracting phase init
264 |   ${SED} -n "1,/^#phase / p" < "$TGT_SCRIPT" | ${HEAD} -n -1 >> "$PHASED_SCRIPT"
265 |   echo -e "echo !!! end of implied phase init !!!\necho\n" >> "$PHASED_SCRIPT"
266 | else
267 |   printf "echo !!! skipped phase: init !!! \n" >> "$PHASED_SCRIPT"
268 | fi
269 | 
270 | ## extract remaining phases
271 | for phase in "${ALL_PHASES[@]}"; do
272 |   [[ "$phase" == 'init' ]] && continue # skip phase init that already has been processed
273 | 
274 |   if [[ $(contains EXEC_PHASES "$phase") -ge 0 ]]; then
275 |     ((v>0)) && echo Extracting phase $phase
276 |     echo -e "echo !!! start of phase $phase !!!" >> "$PHASED_SCRIPT"
277 |     if [[ "$phase" != "${ALL_PHASES[-1]}" ]]; then
278 |       ${SED} -n "/^#phase $phase/,/^#phase / p" < "$TGT_SCRIPT" | ${HEAD} -n -1 >> "$PHASED_SCRIPT"
279 |     else
280 |       # keep all lines in the very last section
281 |       ${SED} -n "/^#phase $phase/,\$p" < "$TGT_SCRIPT" >> "$PHASED_SCRIPT"
282 |     fi
283 |     echo -e "echo !!! end of phase $phase !!!\necho\n" >> "$PHASED_SCRIPT"
284 |   else
285 |     printf "echo !!! skipped phase: %s !!!\necho\n" "$phase" >> "$PHASED_SCRIPT"
286 |   fi
287 | done
288 | 
289 | # execute phased script
290 | if [[ -n "$outp_scr" ]]; then
291 |   echo Saving output script into: $outp_scr
292 |   cp "$PHASED_SCRIPT" "$outp_scr"
293 | else
294 |   echo
295 |   echo "!!! executing phased script !!!"
296 |   chmod +x "$PHASED_SCRIPT"
297 |   "$PHASED_SCRIPT" "${@:3}"
298 |   excode=$?
299 |   echo "!!! Phased script completed with code $excode !!!"
300 | 
301 |   exit $excode
302 | fi
303 | 


--------------------------------------------------------------------------------
/test_script.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | # this is a test schrip for phases
 4 | 
 5 | # everything until the next non-empty line is pre[amble]
 6 | 
 7 | echo arguments: "$@"
 8 | 
 9 | echo "Intialization phase, do something"
10 | 
11 | #phase load
12 | 
13 | echo "Load phase, do something"
14 | 
15 | #phase process
16 | 
17 | echo "Process phase, do something"
18 | 
19 | #phase final
20 | 
21 | echo "Final phase, do something"
22 | 
23 | #phase clean
24 | 
25 | echo "Cleanup phase, do something"
26 | 


--------------------------------------------------------------------------------