├── .gitignore ├── tools ├── bin │ ├── ss │ ├── time_bin │ └── sqs ├── helpers │ ├── show_banner.sh │ ├── exchange-continue-jobline.sh │ ├── submit.sh │ ├── copy-templates.sh │ ├── continue-jobline.sh │ ├── exchange-jobfile.sh │ ├── sync-jobfile.sh │ └── prepare-todolists.sh ├── vf_continue_all.sh ├── vf_redistribute_collections_single.sh ├── vf_redistribute_collections_multiple.sh ├── vf_prepare_todolists.sh ├── vf_start_jobline.sh ├── vf_continue_jobline.sh ├── vf_prepare_folders.sh ├── templates │ ├── template1.bash.sh │ ├── template1.sge.sh │ ├── template1.lsf.sh │ ├── template1.torque.sh │ ├── template1.pbs.sh │ ├── template1.slurm.sh │ └── one-step.sh └── vf_report.sh ├── output-files └── README.md ├── input-files └── README.md ├── workflow └── README.md ├── README.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | 3 | *.pyc 4 | .dropbox.attr 5 | *.attr 6 | -------------------------------------------------------------------------------- /tools/bin/ss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VirtualFlow/VFLP/HEAD/tools/bin/ss -------------------------------------------------------------------------------- /tools/bin/time_bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VirtualFlow/VFLP/HEAD/tools/bin/time_bin -------------------------------------------------------------------------------- /output-files/README.md: -------------------------------------------------------------------------------- 1 | This folder contains all the output ligand databases generated by VFLP. -------------------------------------------------------------------------------- /input-files/README.md: -------------------------------------------------------------------------------- 1 | This folder contains generally all the input files which are needed for the ligand preparation. 2 | Currently, this is only the ligand input databases itself, which can also be stored at another location, according to the setting in the control file. 3 | -------------------------------------------------------------------------------- /workflow/README.md: -------------------------------------------------------------------------------- 1 | This folder contains all the files which are needed and used by VFLP during the runtime for its internal organization and logging. 2 | This folder is prepared by the user with the command vf_prepare_folders.sh during the preparation of the workflow, as described in the documentation. -------------------------------------------------------------------------------- /tools/helpers/show_banner.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 Christoph Gorgulla 2 | # 3 | # This file is part of VirtualFlow. 4 | # 5 | # VirtualFlow is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 2 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # VirtualFlow is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with VirtualFlow. If not, see . 17 | 18 | echo " :: :: :: ::::. :::::: :: :: .::::. :: ::::: :: .::::. :: ::" 19 | echo " :: :: :: :: :: :: :: :: :: :: :: :: :: :: :: :: :: ::" 20 | echo " :::: :: :::. :: :: :: :::::: :: ::::: :: :: :: ::::::::" 21 | echo " :: :: :: :: :: :::: :: :: :::: :: ::::: '::::' :: ::" 22 | -------------------------------------------------------------------------------- /tools/bin/sqs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Variables 21 | if [ -f ../workflow/control/all.ctrl ]; then 22 | controlfile="../workflow/control/all.ctrl" 23 | else 24 | echo -e " * The controlfile ../workflow/control/all.ctrl does not exist, but it is required. Exiting... \n\n" 25 | exit 1 26 | fi 27 | 28 | # Variables 29 | batchsystem="$(grep -m 1 "^batchsystem=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 30 | 31 | # Checking batchsystem type 32 | if [ "${batchsystem}" == "SLURM" ]; then 33 | squeue -l | grep ${USER:0:8} 34 | elif [ "${batchsystem}" == "MTP" ]; then 35 | qstat | grep ${USER:0:8} 36 | elif [ "${batchsystem}" == "LSF" ]; then 37 | bjobs | grep ${USER:0:8} 38 | elif [ "${batchsystem}" == "SGE" ]; then 39 | qstat | grep ${USER:0:8} 40 | else 41 | echo -e " * Unsupported batchsystem (${batchsystem}) specified in the file ../workflow/control/all.ctrl. Exiting... \n\n" 42 | exit 1 43 | fi 44 | -------------------------------------------------------------------------------- /tools/helpers/exchange-continue-jobline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if the first argument is -h 21 | usage="Usage: . exchange-continue-jobline first_jobline_no last_jobline_no job_template [quiet]" 22 | if [ "${1}" == "-h" ]; then 23 | echo -e "\n${usage}\n\n" 24 | exit 0 25 | fi 26 | if [[ "$#" -ne "3" && "$#" -ne "4" ]]; then 27 | echo -e "\nWrong number of arguments. Exiting." 28 | echo -e "\n${usage}\n\n" 29 | exit 1 30 | fi 31 | 32 | # Standard error response 33 | error_response_nonstd() { 34 | echo "Error was trapped which is a nonstandard error." 35 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 36 | echo "Error on line $1" 37 | exit 1 38 | } 39 | trap 'error_response_nonstd $LINENO' ERR 40 | 41 | # Variables 42 | i=0 43 | first_jobline_no=${1} 44 | last_jobline_no=${2} 45 | job_template=${3} 46 | if [ -z "${4}" ]; then 47 | quiet_mode="off" 48 | else 49 | quiet_mode=${4} 50 | fi 51 | 52 | # Continuing the jobline 53 | for jobline_no in $(seq ${first_jobline_no} ${last_jobline_no}); do 54 | i=$(( i + 1 )) 55 | . exchange-jobfile.sh ${job_template} ${jobline_no} ${quiet_mode} 56 | . continue-jobline.sh ${jobline_no} "sync" 57 | done 58 | 59 | # Displaying some information if no quiet option 60 | if [[ ! "$*" = *"quiet"* ]]; then 61 | echo 62 | echo "Number of joblines which were continued: ${i}" 63 | echo 64 | fi 65 | -------------------------------------------------------------------------------- /tools/helpers/submit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if the first argument is -h 21 | usage="# Usage: . submit jobfile [quiet]" 22 | if [ "${1}" == "-h" ]; then 23 | echo -e "\n${usage}\n\n" 24 | return 0 25 | fi 26 | if [[ "$#" -ne "1" && "$#" -ne "2" ]]; then 27 | echo -e "\nWrong number of arguments. Exiting." 28 | echo -e "\n${usage}\n\n" 29 | return 1 30 | fi 31 | # Standard error response 32 | error_response_nonstd() { 33 | echo "Error was trapped which is a nonstandard error." 34 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 35 | echo "Error on line $1" 36 | exit 1 37 | } 38 | trap 'error_response_nonstd $LINENO' ERR 39 | 40 | # Variables 41 | # Getting the batchsystem type 42 | batchsystem="$(grep -m 1 "^batchsystem=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 43 | jobfile=${1} 44 | jobline=$(echo ${jobfile} | awk -F '[./]' '{print $(NF-1)}') 45 | 46 | # Submitting the job 47 | cd ../ 48 | if [ "${batchsystem}" == "SLURM" ]; then 49 | sbatch ${jobfile} 50 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 51 | msub ${jobfile} 52 | elif [ "${batchsystem}" == "SGE" ]; then 53 | qsub ${jobfile} 54 | elif [ "${batchsystem}" == "LSF" ]; then 55 | bsub < ${jobfile} 56 | elif [ "${batchsystem}" == "BASH" ]; then 57 | bash ${jobfile} 58 | else 59 | echo 60 | echo "Error: The batch system (${batchsystem}) which was specified in the control file (${VF_CONTROLFILE}) is not supported." 61 | echo 62 | exit 1 63 | fi 64 | 65 | # Changing the directory 66 | cd helpers 67 | 68 | # Printing some information 69 | if [ ! "$*" = *"quiet"* ]; then 70 | echo "The job for jobline ${jobline} has been submitted at $(date)." 71 | echo 72 | fi 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## VFLP - VirtualFlow for Ligand Preparation 2 | 3 | 4 | Welcome to the VirtualFlow project! 5 | 6 | VirtualFlow is a versatile, parallel workflow platform for carrying out virtual screening related tasks on Linux-based computer clusters of any type and size which are managed by a batchsystem (such as SLURM). 7 | 8 | Currently, there exist two versions of VirtualFlow, which are tailored to different types of tasks: 9 | 10 | - [VFLP: VirtualFlow for Ligand Preparation](https://github.com/VirtualFlow/VFLP) 11 | - [VFVS : VirtualFlow for Virtual Screenings](https://github.com/VirtualFlow/VFVS) 12 | 13 | They use the same core technology regarding the workflow management and parallelization, and they can be used individually or in concert with each other. Additional versions are expected to arrive in the future. Pre-built ready-to-dock ligand libraries for VFVS are available for free (in the download section). 14 | 15 | 16 | 17 | ### Overview of Resources 18 | 19 | The following gives an overview of the various resources related to VirtualFlow: 20 | 21 | - [Homepage of VirtualFlow](https://virtual-flow.org/) 22 | - [The documentation of VirtualFlow](https://docs.virtual-flow.org/documentation/-LdE8RH9UN4HKpckqkX3/) 23 | - [Tutorials for VirtualFlow](https://virtual-flow.org/tutorials) 24 | - [Feature Requests (powered by Canny)](http://feedback.virtual-flow.org/feature-requests) 25 | - [GitHub Issue Tracker for VFVS](https://github.com/VirtualFlow/VFVS/issues) 26 | - [GitHub Issue Tracker for VFLP](https://github.com/VirtualFlow/VFLP/issues) 27 | - In-code documentation of the source code files 28 | 29 | 30 | ### Contributing 31 | 32 | If you are interested in contributing to VirtualFlow, whether it is to report a bug or to extend VirtualFlow with your own code, please see the file [CONTRIBUTING.md](CONTRIBUTING.md) and the file [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). 33 | 34 | 35 | 36 | ### License 37 | 38 | The project ist distributed under the GNU GPL v2.0. Please see the file [LICENSE](LICENSE) for more details. 39 | 40 | 41 | 42 | ### Citation 43 | 44 | The use of VirtualFlow or the VirtualFlow ligand libraries in any reports or publications should be acknowledged by including a citation to: 45 | 46 | - Gorgulla, C., Boeszoermenyi, A., Wang, Z. et al. An open-source drug discovery platform enables ultra-large virtual screens. Nature (2020). https://doi.org/10.1038/s41586-020-2117-z 47 | 48 | 49 | 50 | ### VirtualFlow Forum 51 | 52 | If you need help or have any questions related to VirtualFlow, please use our forum: 53 | 54 | * https://community.virtual-flow.org/ 55 | -------------------------------------------------------------------------------- /tools/helpers/copy-templates.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if the first argument is -h 21 | usage="Usage: . copy-templates templates [quiet]" 22 | if [ "${1}" = "-h" ]; then 23 | echo "${usage}" 24 | return 25 | fi 26 | 27 | # Standard error response 28 | error_response_nonstd() { 29 | echo "Error was trapped which is a nonstandard error." 30 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 31 | echo "Error on line $1" 32 | exit 1 33 | } 34 | trap 'error_response_nonstd $LINENO' ERR 35 | 36 | # Variables 37 | if [ -f ../../workflow/control/all.ctrl ]; then 38 | controlfile="../../workflow/control/all.ctrl" 39 | else 40 | controlfile="../templates/all.ctrl" 41 | fi 42 | central_todo_list_splitting_size="$(grep -m 1 "^central_todo_list_splitting_size=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 43 | 44 | 45 | # Copying the template files 46 | if [[ "${1}" = "subjobfiles" || "${1}" = "all" ]]; then 47 | cp ../templates/one-step.sh ../../workflow/job-files/sub/ 48 | cp ../templates/one-queue.sh ../../workflow/job-files/sub/ 49 | chmod u+x ../../workflow/job-files/sub/one-step.sh 50 | fi 51 | if [[ "${1}" = "todofiles" || "${1}" = "all" ]]; then 52 | split -a 4 -d -l ${central_todo_list_splitting_size} ../templates/todo.all ../../workflow/ligand-collections/todo/todo.all. 53 | cp ../../workflow/ligand-collections/todo/todo.all.[0-9]* ../../workflow/ligand-collections/var/ 54 | cp ../templates/todo.all ../../workflow/ligand-collections/var/todo.original 55 | ln -s todo.all.0000 ../../workflow/ligand-collections/todo/todo.all 56 | fi 57 | if [[ "${1}" = "controlfiles" || "${1}" = "all" ]]; then 58 | cp ../templates/all.ctrl ../../workflow/control/ 59 | fi 60 | 61 | # Displaying some information 62 | if [[ ! "$*" = *"quiet"* ]]; then 63 | echo 64 | echo "The templates were copied." 65 | echo 66 | fi 67 | -------------------------------------------------------------------------------- /tools/vf_continue_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | #Checking the input arguments 21 | usage="Usage: vf_continue_all.sh " 22 | if [ "${1}" == "-h" ]; then 23 | echo -e "\n${usage}\n\n" 24 | exit 0 25 | fi 26 | if [ "$#" -ne "2" ]; then 27 | echo -e "\nWrong number of arguments. Exiting." 28 | echo -e "\n${usage}\n\n" 29 | exit 1 30 | fi 31 | 32 | # Displaying the banner 33 | echo 34 | echo 35 | . helpers/show_banner.sh 36 | echo 37 | echo 38 | 39 | # Standard error response 40 | error_response_nonstd() { 41 | echo "Error was trapped which is a nonstandard error." 42 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 43 | echo "Error on line $1" 44 | exit 1 45 | } 46 | trap 'error_response_nonstd $LINENO' ERR 47 | 48 | # Clean up 49 | clean_up() { 50 | rm -r ${tempdir}/ 2>/dev/null || true 51 | } 52 | trap 'clean_up' EXIT 53 | 54 | # Variables 55 | job_template=$1 56 | delay_time=$2 57 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 58 | export VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 59 | vf_tempdir="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 60 | no_of_jobfiles=$(ls ../workflow/job-files/main/ | wc -l) 61 | 62 | # Verbosity 63 | export VF_VERBOSITY_COMMANDS="$(grep -m 1 "^verbosity_commands=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 64 | if [ "${VF_VERBOSITY_COMMANDS}" = "debug" ]; then 65 | set -x 66 | fi 67 | 68 | # Body 69 | tempdir=${vf_tempdir}/$USER/VFLP/${VF_JOBLETTER}/vf_continue_all_$(date | tr " :" "_") 70 | mkdir -p ${tempdir} 71 | cat /dev/null > ${tempdir}/sqs.out 72 | bin/sqs > ${tempdir}/sqs.out || true 73 | 74 | # Loop for each jobfile 75 | counter=1 76 | for file in $(ls -v ../workflow/job-files/main/); do 77 | VF_JOBLINE_NO=${file/.job} 78 | if ! grep -q "${VF_JOBLETTER}\-${VF_JOBLINE_NO}\." ${tempdir}/sqs.out; then 79 | ./vf_continue_jobline.sh ${VF_JOBLINE_NO} ${VF_JOBLINE_NO} ${job_template} 1 80 | if [ ! "${counter}" -eq "${no_of_jobfiles}" ]; then 81 | sleep $delay_time 82 | fi 83 | fi 84 | counter=$((counter + 1)) 85 | done 86 | 87 | rm ${tempdir}/sqs.out 88 | -------------------------------------------------------------------------------- /tools/vf_redistribute_collections_single.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | #Checking the input arguments 21 | usage="Usage: vf_redistribute_collections_single.sh 22 | 23 | One collection will be placed per collection file/queue. 24 | All existing files in the output folder will be deleted." 25 | if [ "${1}" == "-h" ]; then 26 | echo -e "\n${usage}\n\n" 27 | exit 0 28 | fi 29 | if [ "$#" -ne "5" ]; then 30 | echo -e "\nWrong number of arguments. Exiting." 31 | echo -e "\n${usage}\n\n" 32 | exit 1 33 | fi 34 | 35 | # Displaying the banner 36 | echo 37 | echo 38 | . helpers/show_banner.sh 39 | echo 40 | echo 41 | 42 | # Standard error response 43 | error_response_nonstd() { 44 | echo "Error was trapped which is a nonstandard error." 45 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 46 | echo "Error on line $1" 47 | exit 1 48 | } 49 | trap 'error_response_nonstd $LINENO' ERR 50 | 51 | # Variables 52 | input_collection_file=$1 53 | queues_per_step_new=$2 54 | steps_per_job_new=$3 55 | first_job_no=$4 56 | output_folder=$5 57 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 58 | 59 | # Verbosity 60 | export VF_VERBOSITY_COMMANDS="$(grep -m 1 "^verbosity_commands=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 61 | if [ "${VF_VERBOSITY_COMMANDS}" = "debug" ]; then 62 | set -x 63 | fi 64 | 65 | # Preparing the directory 66 | echo -e " *** Preparing the output directory ***\n" 67 | rm ${output_folder}/* 2>/dev/null || true 68 | 69 | # Loop for each collection 70 | queue_no=1 71 | step_no=1 72 | job_no=${first_job_no} 73 | echo -e " *** Starting to distribute the collections ***\n" 74 | while IFS= read -r line || [[ -n "$line" ]]; do 75 | echo " * Assigning collection $line to queue $job_no-$step_no-$queue_no" 76 | echo $line > ${output_folder}/${job_no}-${step_no}-${queue_no} 77 | queue_no=$((queue_no + 1)) 78 | if [ "${queue_no}" -gt "${queues_per_step_new}" ]; then 79 | queue_no=1 80 | step_no=$((step_no + 1)) 81 | if [ "${step_no}" -gt "${steps_per_job_new}" ]; then 82 | step_no=1 83 | job_no=$((job_no + 1)) 84 | fi 85 | fi 86 | done < "${input_collection_file}" 87 | 88 | echo -e "\n * Resdistribution complete\n\n" 89 | 90 | -------------------------------------------------------------------------------- /tools/vf_redistribute_collections_multiple.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | #Checking the input arguments 21 | usage="Usage: vf_redistribute_collections_multiple 22 | 23 | collection will be placed per collection file/queue." 24 | 25 | 26 | if [ "${1}" == "-h" ]; then 27 | echo -e "\n${usage}\n\n" 28 | exit 0 29 | fi 30 | if [ "$#" -ne "7" ]; then 31 | echo -e "\nWrong number of arguments. Exiting." 32 | echo -e "\n${usage}\n\n" 33 | exit 1 34 | fi 35 | 36 | # Displaying the banner 37 | echo 38 | echo 39 | . helpers/show_banner.sh 40 | echo 41 | echo 42 | 43 | # Standard error response 44 | error_response_nonstd() { 45 | echo "Error was trapped which is a nonstandard error." 46 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 47 | echo "Error on line $1" 48 | exit 1 49 | } 50 | trap 'error_response_nonstd $LINENO' ERR 51 | 52 | # Variables 53 | input_collection_file=$1 54 | jobline_no_start=$2 55 | jobline_no_end=$3 56 | steps_per_job=$4 57 | queues_per_step=$5 58 | collections_per_queue=$6 59 | output_folder=$7 60 | 61 | # Preparing the directory 62 | echo -e " *** Preparing the output directory ***\n" 63 | #rm ${output_folder}/* 2>/dev/null || true 64 | 65 | # Loop for each collection 66 | queue_no=1 67 | step_no=1 68 | echo -e " *** Starting to distribute the collections ***\n" 69 | sed -i "s/^$//g" ${input_collection_file} 70 | for i in $(seq 1 $collections_per_queue); do 71 | for job_no in $(seq ${jobline_no_start} ${jobline_no_end}); do 72 | for step_no in $(seq 1 ${steps_per_job}); do 73 | for queue_no in $(seq 1 ${queues_per_step_new}); do 74 | collection="$(head -n 1 ${input_collection_file})" 75 | collection="$(echo ${collection} | tr -d '\040\011\012\015')" 76 | if [[ ${collection} == *"_"* ]]; then 77 | echo " * Assigning collection ${collection} to queue $job_no-$step_no-$queue_no" 78 | echo ${collection} >> ${output_folder}/${job_no}-${step_no}-${queue_no} 79 | sed -i "/${collection}/d" ${input_collection_file} || true 80 | fi 81 | done 82 | done 83 | done 84 | done 85 | 86 | echo -e "\n * Resdistribution complete\n\n" 87 | 88 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at cgorgulla@g.harvard.edu. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /tools/vf_prepare_todolists.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | #Checking the input arguments 21 | usage="Usage: vf_prepare_todolists.sh 22 | 23 | Description: Prepares the todo lists of the joblines in advance of the workflow. Uses the corresponding control files in ../workflow/control for each jobline. 24 | 25 | Arguments: 26 | : Positive integer 27 | : Positive integer 28 | " 29 | if [ "${1}" == "-h" ]; then 30 | echo -e "\n${usage}\n\n" 31 | exit 0 32 | fi 33 | if [ "$#" -ne "2" ]; then 34 | echo -e "\nWrong number of arguments. Exiting..." 35 | echo -e "\n${usage}\n\n" 36 | exit 1 37 | fi 38 | 39 | # Displaying the banner 40 | echo 41 | echo 42 | . helpers/show_banner.sh 43 | echo 44 | echo 45 | 46 | # Standard error response 47 | error_response_nonstd() { 48 | echo "Error was trapped which is a nonstandard error." 49 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 50 | echo "Error on line $1" 51 | exit 1 52 | } 53 | trap 'error_response_nonstd $LINENO' ERR 54 | 55 | determine_controlfile() { 56 | 57 | # Determining the VF_CONTROLFILE to use for this jobline 58 | cd .. 59 | VF_CONTROLFILE="" 60 | for file in $(ls ../workflow/control/*-* 2>/dev/null || true); do 61 | file_basename=$(basename $file) 62 | jobline_range=${file_basename/.*} 63 | jobline_no_start=${jobline_range/-*} 64 | jobline_no_end=${jobline_range/*-} 65 | if [[ "${jobline_no_start}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${jobline_no_end}" ]]; then 66 | export VF_CONTROLFILE="${file}" 67 | break 68 | fi 69 | done 70 | 71 | # Checking if a specific control file was found 72 | if [ -z "${VF_CONTROLFILE}" ]; then 73 | if [[ -f ../workflow/control/all.ctrl ]]; then 74 | 75 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 76 | 77 | else 78 | # Error response 79 | echo "Error: No relevant control file was found..." 80 | false 81 | fi 82 | fi 83 | cd helpers 84 | } 85 | 86 | 87 | # Variables 88 | start_jobline_no=$1 89 | end_jobline_no=$2 90 | 91 | # Body 92 | cd helpers 93 | for i in $(seq $start_jobline_no $end_jobline_no); do 94 | 95 | # Determining the controlfile 96 | export VF_JOBLINE_NO=$i 97 | determine_controlfile 98 | 99 | # Variables 100 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 101 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 102 | export VF_JOBLETTER="$(grep -m 1 "^job_letter=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 103 | steps_per_job="$(grep -m 1 "^steps_per_job=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 104 | queues_per_step="$(grep -m 1 "^queues_per_step=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 105 | 106 | # Preparing the todolists 107 | bash prepare-todolists.sh $i $steps_per_job $queues_per_step; 108 | 109 | done 110 | cd .. 111 | 112 | 113 | -------------------------------------------------------------------------------- /tools/vf_start_jobline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | #Checking the input arguments 21 | usage="Usage: vf_start_jobline.sh 22 | 23 | Description: Prepares the jobfiles of joblines and if specified submits them. 24 | 25 | Arguments: 26 | : Positive integer 27 | : Positive integer 28 | : Filename (with absolute or relative path) of the job templates in the template folder, depending on the batchsystem 29 | : Whether the newly created job should be directly submitted to the batch system. Possible options: submit, nosubmit 30 | : Time delay between submitted jobs (to disperse the jobs in time to prevent problems with the central task list) 31 | " 32 | 33 | if [ "${1}" == "-h" ]; then 34 | echo -e "\n${usage}\n\n" 35 | exit 0 36 | fi 37 | 38 | if [[ "$#" -ne "5" ]]; then 39 | 40 | # Printing some information 41 | echo 42 | echo "The wrong number of arguments was provided." 43 | echo "Number of expected arguments: 5" 44 | echo "Number of provided arguments: ${#}" 45 | echo "Use the -h option to display basic usage information of the command." 46 | echo 47 | echo 48 | exit 1 49 | fi 50 | 51 | # Displaying the banner 52 | echo 53 | echo 54 | . helpers/show_banner.sh 55 | echo 56 | echo 57 | 58 | # Standard error response 59 | error_response_nonstd() { 60 | echo "Error was trapped which is a nonstandard error." 61 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 62 | echo "Error on line $1" 63 | echo 64 | exit 1 65 | } 66 | trap 'error_response_nonstd $LINENO' ERR 67 | 68 | # Variables 69 | start_jobline_no=${1} 70 | end_jobline_no=${2} 71 | delay_time=${5} 72 | submit_mode=${4} 73 | job_template=${3} 74 | if [ -f ../workflow/control/all.ctrl ]; then 75 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 76 | else 77 | export VF_CONTROLFILE="templates/all.ctrl" 78 | fi 79 | 80 | # Verbosity 81 | VF_VERBOSITY_COMMANDS="$(grep -m 1 "^verbosity_commands=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 82 | export VF_VERBOSITY_COMMANDS 83 | if [ "${VF_VERBOSITY_COMMANDS}" = "debug" ]; then 84 | set -x 85 | fi 86 | 87 | # Getting the batchsystem type 88 | batchsystem="$(grep -m 1 "^batchsystem=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 89 | 90 | # Formatting screen output 91 | echo "" 92 | 93 | # Duplicating the main template job file and syncing the copied jobfiles with the control file 94 | for i in $(seq ${start_jobline_no} ${end_jobline_no}); do 95 | cp ${job_template} ../workflow/job-files/main/${i}.job 96 | sed -i "s/-1\.1/-${i}\.1/g" ../workflow/job-files/main/${i}.job 97 | cd helpers 98 | . sync-jobfile.sh ${i} 99 | cd .. 100 | done 101 | 102 | # Formatting screen output 103 | echo "" 104 | 105 | # Submitting the job files 106 | if [[ "${submit_mode}" = "submit" ]]; then 107 | cd helpers 108 | for i in $(seq ${start_jobline_no} ${end_jobline_no}); do 109 | . submit.sh ../workflow/job-files/main/${i}.job 110 | if [ ! "${i}" = "${end_jobline_no}" ]; then 111 | sleep ${delay_time} 112 | fi 113 | done 114 | cd .. 115 | fi -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## How to contribute to VirtualFlow 2 | 3 | #### Table of Contents 4 | 5 | - [Introduction](#introduction) 6 | - [Code of Conduct](#code-of-conduct) 7 | - [Important Resources](#important-resources) 8 | - [Types of Contributions](#types-of-contributions) 9 | - [Feature Requests](#features-requests) 10 | - [Bug Reports](#bug-reports) 11 | - [Improving/Extending the Documentation](#improvingextending-the-documentation) 12 | - [Improving/Extending the Tutorials](#improvingextending-the-tutorials) 13 | - [Resolution of one of the Issues on GitHub](#resolution-of-one-of-the-github-issues) 14 | - [Contibuting Code](#contributing-code) 15 | 21 | - [Contact](#contact) 22 | 23 | 24 | ### Introduction 25 | 26 | Thank you for considering to contribute to VirtualFlow. It's people like you that will help VirtualFlow to grow further and to become an even greater drug discovery platform. 27 | 28 | Following the contributing guidelines will help to run this open source project smoothly and efficiently. 29 | 30 | VirtuaFlow is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials, blog posts, improving the documentation, requesting features, submitting bug reports, and writing code for VirtualFlow itself. See [further below](#types-of-contributions) for more details about the varios types of contributions. 31 | 32 | 33 | ### Code of Conduct 34 | 35 | We aspire to create a welcoming environment for collaboration on this project. To that end, we follow the [Contributor Covenant Code of Conduct Code](CODE_OF_CONDUCT.md). Please read and follow it if you would like to become a contributor to our project. 36 | 37 | 38 | ### Resources 39 | 40 | The following gives an overview of the various resources related to VirtualFlow: 41 | 42 | - [Homepage of VirtualFlow](https://virtual-flow.org/) 43 | - [The documentation of VirtualFlow](https://docs.virtual-flow.org/documentation/-LdE8RH9UN4HKpckqkX3/) 44 | - [Tutorials for VirtualFlow](https://virtual-flow.org/tutorials) 45 | - [Feature Requests (powered by Canny)](http://feedback.virtual-flow.org/feature-requests) 46 | - [GitHub Issue Tracker for VFVS](https://github.com/VirtualFlow/VFVS/issues) 47 | - [GitHub Issue Tracker for VFLP](https://github.com/VirtualFlow/VFLP/issues) 48 | - In-code documentation of the source code files 49 | 50 | 51 | ### Types of Contributions 52 | 53 | #### Features Requests 54 | 55 | New features can be requested [here](http://feedback.virtual-flow.org/feature-requests), which is powered by Canny. 56 | 57 | 58 | #### Bug Reports 59 | 60 | Bug reports can be done via our GitHub issue tracker: 61 | - [GitHub Issue Tracker for VFVS](https://github.com/VirtualFlow/VFVS/issues) 62 | - [GitHub Issue Tracker for VFLP](https://github.com/VirtualFlow/VFLP/issues) 63 | 64 | 65 | #### Improving/Extending the Documentation 66 | 67 | If you would like to extend the documenation, please contact us for futher coordination. 68 | 69 | 70 | #### Improving/Extending the Tutorials 71 | 72 | If you would like to improve/extend the tutorials, please contact us for futher coordination. 73 | 74 | 75 | #### Resolution of One of the GitHub Issues 76 | 77 | If you would like to help to resolve the issues coming up in our issue trackers, please contactus for futher coordination. 78 | 79 | 80 | #### Contributing Code 81 | 82 | Code contributions are managed by pull requests. 83 | 84 | Working on your first Pull Request? You can learn how from this free series: 85 | - [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github). 86 | 87 | 88 | 100 | 101 | ### Contact 102 | 103 | If you have any questions, or would to get in touch with us, please contact Christoph Gorgulla (cgorgulla@g.harvard.edu). 104 | -------------------------------------------------------------------------------- /tools/vf_continue_jobline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | #Checking the input arguments 21 | usage="Usage: vf_continue_jobline.sh first_jobline_no last_jobline_no job_template delay_time_in_seconds [quiet]" 22 | if [ "${1}" == "-h" ]; then 23 | echo -e "\n${usage}\n\n" 24 | exit 0 25 | fi 26 | 27 | if [[ "$#" -ne "4" ]] && [[ "$#" -ne "5" ]]; then 28 | echo -e "\nWrong number of arguments. Exiting.\n" 29 | echo -e "${usage}\n\n" 30 | exit 1 31 | fi 32 | 33 | # Displaying the banner 34 | if [[ "$0" != "$BASH_SOURCE" ]]; then # test if the script was sourced or executed 35 | echo 36 | echo 37 | . helpers/show_banner.sh 38 | echo 39 | echo 40 | fi 41 | 42 | # Standard error response 43 | error_response_nonstd() { 44 | echo "Error was trapped which is a nonstandard error." 45 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 46 | echo "Error on line $1" 47 | exit 1 48 | } 49 | trap 'error_response_nonstd $LINENO' ERR 50 | 51 | # Clean up 52 | clean_up() { 53 | rm -r ${tempdir}/ 2>/dev/null || true 54 | } 55 | trap 'clean_up' EXIT 56 | 57 | # Variables 58 | first_jobline_no=${1} 59 | last_jobline_no=${2} 60 | job_template=${3} 61 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 62 | export VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 63 | vf_tempdir="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 64 | 65 | # Verbosity 66 | VF_VERBOSITY_COMMANDS="$(grep -m 1 "^verbosity_commands=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 67 | export VF_VERBOSITY_COMMANDS 68 | if [ "${VF_VERBOSITY_COMMANDS}" = "debug" ]; then 69 | set -x 70 | fi 71 | 72 | # Formatting screen output 73 | echo "" 74 | 75 | # Removing old files if existent 76 | tempdir=${vf_tempdir}/$USER/VFLP/${VF_JOBLETTER}/vf_continue_jobline_$(date | tr " :" "_") 77 | if [ -f "${tempdir}/jobs-to-continue" ]; then 78 | rm ${tempdir}/jobs-to-continue 79 | fi 80 | mkdir -p ${tempdir} 81 | 82 | # Storing all the jobs which are currently running 83 | touch ${tempdir}/jobs-all 84 | bin/sqs > ${tempdir}/jobs-all 2>/dev/null || true 85 | 86 | # Storing all joblines which have to be restarted 87 | echo "Checking which joblines are already in the batchsystem" 88 | for VF_JOBLINE_NO in $(seq ${first_jobline_no} ${last_jobline_no}); do 89 | if ! grep -q "${VF_JOBLETTER}\-${VF_JOBLINE_NO}\." ${tempdir}/jobs-all; then 90 | echo "Adding jobline ${VF_JOBLINE_NO} to the list of joblines to be continued." 91 | echo ${VF_JOBLINE_NO} >> "${tempdir}/jobs-to-continue" 92 | else 93 | echo "Omitting jobline ${VF_JOBLINE_NO} because it was found in the batchsystem." 94 | fi 95 | done 96 | 97 | # Variables 98 | k=0 99 | delay_time="${4}" 100 | 101 | # Resetting the collections and continuing the jobs if existent 102 | if [ -f ${tempdir}/jobs-to-continue ]; then 103 | k_max="$(cat ${tempdir}/jobs-to-continue | wc -l)" 104 | for VF_JOBLINE_NO in $(cat ${tempdir}/jobs-to-continue ); do 105 | k=$(( k + 1 )) 106 | cd helpers 107 | echo "Continuing jobline ${VF_JOBLINE_NO}" 108 | . exchange-continue-jobline.sh ${VF_JOBLINE_NO} ${VF_JOBLINE_NO} ${job_template} quiet 109 | cd .. 110 | if [ ! "${k}" = "${k_max}" ]; then 111 | sleep ${delay_time} 112 | fi 113 | done 114 | fi 115 | 116 | # Removing the temporary files 117 | if [ -f "${tempdir}/jobs-all" ]; then 118 | rm ${tempdir}/jobs-all 119 | fi 120 | if [ -f "${tempdir}/jobs-to-continue" ]; then 121 | rm ${tempdir}/jobs-to-continue 122 | fi 123 | 124 | # Displaying some information 125 | if [[ ! "$*" = *"quiet"* ]]; then 126 | echo "Number of joblines which were continued: ${k}" 127 | echo 128 | fi 129 | -------------------------------------------------------------------------------- /tools/helpers/continue-jobline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if the first argument is -h 21 | usage="Usage: . continue-jobline VF_JOBLINE_NO sync_mode" 22 | if [ "${1}" == "-h" ]; then 23 | echo -e "\n${usage}\n\n" 24 | exit 0 25 | fi 26 | if [[ "$#" -ne "2" ]]; then 27 | echo -e "\nWrong number of arguments. Exiting." 28 | echo -e "\n${usage}\n\n" 29 | exit 1 30 | fi 31 | 32 | # Standard error response 33 | error_response_nonstd() { 34 | echo "Error was trapped which is a nonstandard error." 35 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 36 | echo "Error on line $1" 37 | exit 1 38 | } 39 | trap 'error_response_nonstd $LINENO' ERR 40 | 41 | # Variables 42 | # Getting the batchsystem type 43 | batchsystem="$(grep -m 1 "^batchsystem=" ../${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 44 | sync_mode=${2} 45 | 46 | # Getting the jobline number and the old job number 47 | VF_JOBLINE_NO=${1} 48 | if [ "${batchsystem}" = "SLURM" ]; then 49 | line=$(cat ../../workflow/job-files/main/${VF_JOBLINE_NO}.job | grep -m 1 "job-name") 50 | VF_OLD_JOB_NO=${line/^#SBATCH --job-name=[a-zA-Z]-} 51 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 52 | line=$(cat ../../workflow/job-files/main/${VF_JOBLINE_NO}.job | grep -m 1 "\-N") 53 | VF_OLD_JOB_NO=${line/^\#PBS -N [a-zA-Z]-} 54 | elif [ "${batchsystem}" = "SGE" ]; then 55 | line=$(cat ../../workflow/job-files/main/${VF_JOBLINE_NO}.job | grep -m 1 "\-N") 56 | VF_OLD_JOB_NO=${line/^\#\$ -N [a-zA-Z]-} 57 | elif [ "${batchsystem}" = "LSF" ]; then 58 | line=$(cat ../../workflow/job-files/main/${VF_JOBLINE_NO}.job | grep -m 1 "^#BSUB \-J") 59 | VF_OLD_JOB_NO=${line/^\#BSUB -J [a-zA-Z]-} 60 | fi 61 | VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 62 | 63 | 64 | # Computing the new job number 65 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 66 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 67 | 68 | 69 | # Syncing the workflow settings if specified 70 | if [ "${sync_mode}" == "sync" ]; then 71 | . sync-jobfile.sh ${VF_JOBLINE_NO} 72 | fi 73 | 74 | # Changing the job number 1.1 (of template/new job file) to current job number 75 | if [ "${batchsystem}" = "SLURM" ]; then 76 | sed -i "s/^#SBATCH --job-name=\([a-zA-Z]\)-.*/#SBATCH --job-name=\1-${new_job_no}/g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 77 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 78 | sed -i "s/^#PBS -N \([a-zA-Z]\)-.*/#PBS -N \1-${new_job_no}/g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 79 | elif [ "${batchsystem}" = "LSF" ]; then 80 | sed -i "s/^#BSUB -J \([a-zA-Z]\)-.*/#BSUB -J \1-${new_job_no}/g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 81 | elif [ "${batchsystem}" = "SGE" ]; then 82 | sed -i "s/^#\\$ -N \([a-zA-Z]\)-.*/#\$ -N \1-${new_job_no}/g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 83 | fi 84 | 85 | # Changing the output filenames 86 | if [ "${batchsystem}" = "SLURM" ]; then 87 | sed -i "s|^#SBATCH --output=.*|#SBATCH --output=../workflow/output-files/jobs/job-${new_job_no}_%j.out|g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 88 | sed -i "s|^#SBATCH --error=.*|#SBATCH --output=../workflow/output-files/jobs/job-${new_job_no}_%j.out|g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 89 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 90 | sed -i "s|^#PBS -\([oe]\) .*|#PBS -\1 ../workflow/output-files/jobs/job-${new_job_no}_\${PBS_JOBID}.out|g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 91 | elif [ "${batchsystem}" = "SGE" ]; then 92 | sed -i "s|^#\\$ -\([oe]\) .*|#\$ -\1 ../workflow/output-files/jobs/job-${new_job_no}_\${PBS_JOBID}.out|g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 93 | elif [ "${batchsystem}" = "LSF" ]; then 94 | sed -i "s|^#BSUB -oo .*|#BSUB -oo ../workflow/output-files/jobs/job-${new_job_no}_%J.out|g" ../../workflow/job-files/main/${VF_JOBLINE_NO}.job 95 | fi 96 | 97 | 98 | # Submitting new job 99 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 100 | -------------------------------------------------------------------------------- /tools/helpers/exchange-jobfile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if the first argument is -h 21 | usage="Usage: . exchange-jobfile template_file VF_JOBLINE_NO [quiet]" 22 | if [ "${1}" = "-h" ]; then 23 | echo "${usage}" 24 | return 25 | fi 26 | if [[ "$#" -ne "2" && "$#" -ne "3" ]]; then 27 | echo -e "\nWrong number of arguments. Exiting.\n" 28 | echo -e "${usage}\n\n" 29 | exit 1 30 | fi 31 | 32 | # Standard error response 33 | error_response_nonstd() { 34 | echo "Error was trapped which is a nonstandard error." 35 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 36 | echo "Error on line $1" 37 | exit 1 38 | } 39 | trap 'error_response_nonstd $LINENO' ERR 40 | 41 | 42 | # Variables 43 | jobline_no=${2} 44 | 45 | # Determining the controlfile to use for this jobline 46 | controlfile="" 47 | for file in $(ls ../../workflow/control/*-* 2>/dev/null || true); do 48 | file_basename=$(basename $file) 49 | jobline_range=${file_basename/.*} 50 | jobline_no_start=${jobline_range/-*} 51 | jobline_no_end=${jobline_range/*-} 52 | if [[ "${jobline_no_start}" -le "${jobline_no}" && "${jobline_no}" -le "${jobline_no_end}" ]]; then 53 | controlfile="${file}" 54 | break 55 | fi 56 | done 57 | if [ -z "${controlfile}" ]; then 58 | controlfile="../../workflow/control/all.ctrl" 59 | fi 60 | 61 | # Getting the batchsystem type 62 | batchsystem="$(grep -m 1 "^batchsystem=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 63 | 64 | # Getting the jobline number and the current job number 65 | job_template=${1} 66 | new_job_file=${jobline_no}.job 67 | if [ "${batchsystem}" = "SLURM" ]; then 68 | line=$(cat ../../workflow/job-files/main/${new_job_file} | grep -m 1 "job\-name=") 69 | job_no=${line/"#SBATCH --job-name=[a-zA-Z]-"} 70 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 71 | line=$(cat ../../workflow/job-files/main/${new_job_file} | grep -m 1 "\-N") 72 | job_no=${line/"#PBS -N [a-zA-Z]-"} 73 | elif [ "${batchsystem}" = "SGE" ]; then 74 | line=$(cat ../../workflow/job-files/main/${new_job_file} | grep -m 1 "\-N") 75 | job_no=${line/"#\$ -N [a-zA-Z]-"} 76 | elif [ "${batchsystem}" = "LSF" ]; then 77 | line=$(cat ../../workflow/job-files/main/${new_job_file} | grep -m 1 "\-J") 78 | job_no=${line/"#BSUB -N [a-zA-Z]-"} 79 | fi 80 | # Copying the new job file 81 | cp ../${job_template} ../../workflow/job-files/main/${new_job_file} 82 | . copy-templates.sh subjobfiles 83 | 84 | # Changing the job number 1.1 (of template/new job file) to current job number 85 | if [ "${batchsystem}" = "SLURM" ]; then 86 | sed -i "s/^#SBATCH --job-name=\([a-zA-Z]\)-.*/#SBATCH --job-name=\1-${job_no}/g" ../../workflow/job-files/main/${new_job_file} 87 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 88 | sed -i "s/^#PBS -N \([a-zA-Z]\)-.*/#PBS -N \1-${job_no}/g" ../../workflow/job-files/main/${new_job_file} 89 | elif [ "${batchsystem}" = "SGE" ]; then 90 | sed -i "s/^#\\$ -N \([a-zA-Z]\)-.*/#\$ -N \1-${job_no}/g" ../../workflow/job-files/main/${new_job_file} 91 | elif [ "${batchsystem}" = "LSF" ]; then 92 | sed -i "s/^#BSUB -J \([a-zA-Z]\)-.*/#BSUB -J \1-${job_no}/g" ../../workflow/job-files/main/${new_job_file} 93 | fi 94 | 95 | # Changing the output filenames 96 | if [ "${batchsystem}" = "SLURM" ]; then 97 | sed -i "s|^#SBATCH --output=.*|#SBATCH --output=.*/workflow/output-files/jobs/job-${job_no}_%j.out|g" ../../workflow/job-files/main/${new_job_file} 98 | sed -i "s|^#SBATCH --error=.*|#SBATCH --output=.*/workflow/output-files/jobs/job-${job_no}_%j.out|g" ../../workflow/job-files/main/${new_job_file} 99 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 100 | sed -i "s|^#PBS -\([oe]\) .*|#PBS -\1 .*/workflow/output-files/jobs/job-${job_no}_\${PBS_JOBID}.out|g" ../../workflow/job-files/main/${new_job_file} 101 | elif [ "${batchsystem}" = "SGE" ]; then 102 | sed -i "s|^#\\$ -\([oe]\) .*|#\$ -\1 .*/workflow/output-files/jobs/job-${job_no}_\${JOB_ID}.out|g" ../../workflow/job-files/main/${new_job_file} 103 | elif [ "${batchsystem}" = "LSF" ]; then 104 | sed -i "s|^#BSUB -oo .*|#BSUB -oo .*/workflow/output-files/jobs/job-${job_no}_%J.out|g" ../../workflow/job-files/main/${new_job_file} 105 | fi 106 | 107 | # Displaying some information 108 | if [[ ! "$*" = *"quiet"* ]]; then 109 | echo "The jobfiles were exchanged." 110 | echo 111 | fi 112 | 113 | -------------------------------------------------------------------------------- /tools/vf_prepare_folders.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if the first argument is -h 21 | usage="Usage: . vf_prepare_folders.sh 22 | " 23 | 24 | if [ "${1}" == "-h" ]; then 25 | echo -e "\n${usage}\n\n" 26 | exit 0 27 | fi 28 | 29 | # Variables 30 | if [ -f ../workflow/control/all.ctrl ]; then 31 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 32 | else 33 | export VF_CONTROLFILE="templates/all.ctrl" 34 | fi 35 | 36 | # Verbosity 37 | export VF_VERBOSITY_COMMANDS="$(grep -m 1 "^verbosity_commands=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 38 | if [ "${VF_VERBOSITY_COMMANDS}" = "debug" ]; then 39 | set -x 40 | fi 41 | 42 | if [[ "$#" -ne "0" ]]; then 43 | 44 | # Printing some information 45 | echo 46 | echo "The wrong number of arguments was provided." 47 | echo "Number of expected arguments: 0" 48 | echo "Number of provided arguments: ${#}" 49 | echo "Use the -h option to display basic usage information of the command." 50 | echo 51 | echo 52 | exit 1 53 | fi 54 | 55 | # Preparing the output-files folder 56 | # Getting user confirmation 57 | echo 58 | while true; do 59 | read -p "Do you really wish prepare/reset the output-files folder? " answer 60 | case ${answer} in 61 | [Yy]* ) confirm="yes"; break;; 62 | [Nn]* ) confirm="no"; break;; 63 | * ) echo "Please answer yes or no.";; 64 | esac 65 | done 66 | 67 | # Preparing the folder 68 | if [ "${confirm}" = "yes" ]; then 69 | 70 | # Printing some information 71 | echo 72 | echo " * Preparing the output-files folder..." 73 | 74 | # Removing the folder if it exists 75 | if [ "$(ls -A ../output-files 2>/dev/null)" ]; then 76 | rm -r ../output-files/* 2>/dev/null 77 | fi 78 | 79 | # Creating the directory 80 | mkdir -p ../output-files/ 81 | fi 82 | 83 | # Preparing the workflow folder 84 | # Getting user confirmation 85 | echo 86 | while true; do 87 | read -p "Do you really wish to prepare/reset the workflow folder? " answer 88 | case ${answer} in 89 | [Yy]* ) confirm="yes"; break;; 90 | [Nn]* ) confirm="no"; break;; 91 | * ) echo "Please answer \"yes\" or \"no\".";; 92 | esac 93 | done 94 | 95 | # Preparing the folder 96 | if [ "${confirm}" = "yes" ]; then 97 | 98 | # Printing some information 99 | echo 100 | echo " * Preparing the workflow folder..." 101 | 102 | # Removing the folders if they exists and creating new ones 103 | if [ "$(ls -A ../workflow/ligand-collections/todo/ 2>/dev/null)" ]; then 104 | rm -r ../workflow/ligand-collections/todo/ 2>/dev/null 105 | fi 106 | mkdir -p ../workflow/ligand-collections/todo/ 107 | 108 | if [ "$(ls -A ../workflow/ligand-collections/current/ 2>/dev/null)" ]; then 109 | rm -r ../workflow/ligand-collections/current/ 2>/dev/null 110 | fi 111 | mkdir -p ../workflow/ligand-collections/current/ 112 | 113 | if [ "$(ls -A ../workflow/ligand-collections/done/ 2>/dev/null)" ]; then 114 | rm -r ../workflow/ligand-collections/done/ 2>/dev/null 115 | fi 116 | mkdir -p ../workflow/ligand-collections/done/ 117 | 118 | if [ "$(ls -A ../workflow/ligand-collections/ligand-lists/ 2>/dev/null)" ]; then 119 | rm -r ../workflow/ligand-collections/ligand-lists/ 2>/dev/null 120 | fi 121 | mkdir -p ../workflow/ligand-collections/ligand-lists/ 122 | 123 | if [ "$(ls -A ../workflow/ligand-collections/var/ 2>/dev/null)" ]; then 124 | rm -r ../workflow/ligand-collections/var/ 2>/dev/null 125 | fi 126 | mkdir -p ../workflow/ligand-collections/var/ 127 | 128 | if [ "$(ls -A ../workflow/output-files/jobs/ 2>/dev/null)" ]; then 129 | rm -r ../workflow/output-files/jobs/ 2>/dev/null 130 | fi 131 | mkdir -p ../workflow/output-files/jobs/ 132 | 133 | if [ "$(ls -A ../workflow/output-files/queues/ 2>/dev/null)" ]; then 134 | rm -r ../workflow/output-files/queues/ 2>/dev/null 135 | fi 136 | mkdir -p ../workflow/output-files/queues/ 137 | 138 | if [ "$(ls -A ../workflow/job-files/main/ 2>/dev/null)" ]; then 139 | rm -r ../workflow/job-files/main/ 2>/dev/null 140 | fi 141 | mkdir -p ../workflow/job-files/main/ 142 | 143 | if [ "$(ls -A ../workflow/job-files/sub/ 2>/dev/null)" ]; then 144 | rm -r ../workflow/job-files/sub/ 2>/dev/null 145 | fi 146 | mkdir -p ../workflow/job-files/sub/ 147 | 148 | if [ "$(ls -A ../workflow/control/ 2>/dev/null)" ]; then 149 | rm -r ../workflow/control/ 2>/dev/null 150 | fi 151 | mkdir -p ../workflow/control/ 152 | 153 | # Copyinng the templates 154 | cd helpers 155 | . copy-templates.sh all 156 | cd .. 157 | fi 158 | 159 | -------------------------------------------------------------------------------- /tools/helpers/sync-jobfile.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying help if first argument is -h 21 | if [ "${1}" = "-h" ]; then 22 | usage="Usage: . sync-jobfile.sh jobline_no" 23 | echo -e "\n${usage}\n\n" 24 | return 25 | fi 26 | if [[ "$#" -ne "1" && "$#" -ne "2" ]]; then 27 | echo -e "\nWrong number of arguments. Exiting." 28 | echo -e "\n${usage}\n\n" 29 | return 1 30 | fi 31 | 32 | # Standard error response 33 | error_response_nonstd() { 34 | echo "Error was trapped which is a nonstandard error." 35 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 36 | echo "Error on line $1" 37 | exit 1 38 | } 39 | trap 'error_response_nonstd $LINENO' ERR 40 | 41 | # Variables 42 | jobline_no=${1} 43 | 44 | # Determining the controlfile to use for this jobline 45 | controlfile="" 46 | for file in $(ls ../../workflow/control/*-* 2>/dev/null || true); do 47 | file_basename=$(basename $file) 48 | jobline_range=${file_basename/.*} 49 | jobline_no_start=${jobline_range/-*} 50 | jobline_no_end=${jobline_range/*-} 51 | if [[ "${jobline_no_start}" -le "${jobline_no}" && "${jobline_no}" -le "${jobline_no_end}" ]]; then 52 | export controlfile="${file}" 53 | break 54 | fi 55 | done 56 | if [ -z "${controlfile}" ]; then 57 | export controlfile="../../workflow/control/all.ctrl" 58 | fi 59 | 60 | # Getting the batchsystem type 61 | batchsystem="$(grep -m 1 "^batchsystem=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 62 | 63 | 64 | # Printing some information 65 | echo -e "Syncing the jobfile of jobline ${jobline_no} with the controlfile file ${controlfile}." 66 | 67 | # Syncing the number of nodes 68 | steps_per_job_new="$(grep -m 1 "^steps_per_job=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 69 | if [ "${batchsystem}" = "SLURM" ]; then 70 | job_line=$(grep -m 1 "nodes=" ../../workflow/job-files/main/${jobline_no}.job) 71 | steps_per_job_old=${job_line/"#SBATCH --nodes="} 72 | sed -i "s/nodes=${steps_per_job_old}/nodes=${steps_per_job_new}/g" ../../workflow/job-files/main/${jobline_no}.job 73 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 74 | job_line=$(grep -m 1 " -l nodes=" ../../workflow/job-files/main/${jobline_no}.job) 75 | steps_per_job_old=${job_line/"#PBS -l nodes="} 76 | steps_per_job_old=${steps_per_job_old/:*} 77 | sed -i "s/nodes=${steps_per_job_old}:/nodes=${steps_per_job_new}:/g" ../../workflow/job-files/main/${jobline_no}.job 78 | fi 79 | 80 | # Syncing the number of cpus per step 81 | cpus_per_step_new="$(grep -m 1 "^cpus_per_step=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 82 | if [ "${batchsystem}" = "SLURM" ]; then 83 | job_line="$(grep -m 1 "cpus-per-task=" ../../workflow/job-files/main/${jobline_no}.job)" 84 | cpus_per_step_old=${job_line/"#SBATCH --cpus-per-task="} 85 | sed -i "s/cpus-per-task=${cpus_per_step_old}/cpus-per-task=${cpus_per_step_new}/g" ../../workflow/job-files/main/${jobline_no}.job 86 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 87 | job_line="$(grep -m 1 " -l nodes=" ../../workflow/job-files/main/${jobline_no}.job)" 88 | cpus_per_step_old=${job_line/\#PBS -l nodes=*:ppn=} 89 | sed -i "s/ppn=${cpus_per_step_old}/ppn=${cpus_per_step_new}/g" ../../workflow/job-files/main/${jobline_no}.job 90 | elif [ "${batchsystem}" = "LSF" ]; then 91 | job_line="$(grep -m 1 "\-n" ../../workflow/job-files/main/${jobline_no}.job)" 92 | cpus_per_step_old=${job_line/\#BSUB -n } 93 | sed -i "s/-n ${cpus_per_step_old}/-n ${cpus_per_step_new}/g" ../../workflow/job-files/main/${jobline_no}.job 94 | sed -i "s/ptile=${cpus_per_step_old}/ptile=${cpus_per_step_new}/g" ../../workflow/job-files/main/${jobline_no}.job 95 | fi 96 | 97 | # Syncing the timelimit 98 | line=$(cat ${controlfile} | grep -m 1 "^timelimit=") 99 | timelimit_new="$(grep -m 1 "^timelimit=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 100 | if [ "${batchsystem}" == "SLURM" ]; then 101 | job_line=$(grep -m 1 "^#SBATCH \-\-time=" ../../workflow/job-files/main/${jobline_no}.job) 102 | timelimit_old=${job_line/"#SBATCH --time="} 103 | sed -i "s/${timelimit_old}/${timelimit_new}/g" ../../workflow/job-files/main/${jobline_no}.job 104 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 105 | job_line=$(grep -m 1 "^#PBS \-l walltime=" ../../workflow/job-files/main/${jobline_no}.job) 106 | timelimit_old=${job_line/"#PBS -l walltime="} 107 | sed -i "s/${timelimit_old}/${timelimit_new}/g" ../../workflow/job-files/main/${jobline_no}.job 108 | elif [ "${batchsystem}" == "SGE" ]; then 109 | job_line=$(grep -m 1 "^#\$ \-l h_rt=" ../../workflow/job-files/main/${jobline_no}.job) 110 | timelimit_old=${job_line/"#\$ -l h_rt="} 111 | sed -i "s/${timelimit_old}/${timelimit_new}/g" ../../workflow/job-files/main/${jobline_no}.job 112 | elif [ "${batchsystem}" == "LSF" ]; then 113 | job_line=$(grep -m 1 "^#BSUB \-W " ../../workflow/job-files/main/${jobline_no}.job) 114 | timelimit_old=${job_line/"#BSUB -W "} 115 | sed -i "s/${timelimit_old}/${timelimit_new}/g" ../../workflow/job-files/main/${jobline_no}.job 116 | fi 117 | 118 | # Syncing the partition 119 | line=$(cat ${controlfile} | grep -m 1 "^partition=") 120 | partition_new="$(grep -m 1 "^partition=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 121 | if [ "${batchsystem}" = "SLURM" ]; then 122 | sed -i "s/--partition=.*/--partition=${partition_new}/g" ../../workflow/job-files/main/${jobline_no}.job 123 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 124 | sed -i "s/^#PBS -q .*/#PBS -q ${partition_new}/g" ../../workflow/job-files/main/${jobline_no}.job 125 | elif [ "${batchsystem}" = "SGE" ]; then 126 | sed -i "s/^#\\$ -q .*/#\$ -q ${partition_new}/g" ../../workflow/job-files/main/${jobline_no}.job 127 | elif [ "${batchsystem}" = "LSF" ]; then 128 | sed -i "s/^#BSUB -q .*/#BSUB -q ${partition_new}/g" ../../workflow/job-files/main/${jobline_no}.job 129 | fi 130 | 131 | # Syncing the job letter 132 | line=$(cat ${controlfile} | grep -m 1 "^job_letter=") 133 | job_letter_new="$(grep -m 1 "^job_letter=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 134 | if [ "${batchsystem}" = "SLURM" ]; then 135 | sed -i "s/^#SBATCH --job-name=[a-zA-Z]/#SBATCH --job-name=${job_letter_new}/g" ../../workflow/job-files/main/${jobline_no}.job 136 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 137 | sed -i "s/^#PBS -N [a-zA-Z]/#PBS -N ${job_letter_new}/g" ../../workflow/job-files/main/${jobline_no}.job 138 | elif [ "${batchsystem}" = "SGE" ]; then 139 | sed -i "s/^#\\$ -N [a-zA-Z]/#\$ -N ${job_letter_new}/g" ../../workflow/job-files/main/${jobline_no}.job 140 | elif [ "${batchsystem}" = "LSF" ]; then 141 | sed -i "s/^#BSUB -J [a-zA-Z]/#BSUB -J ${job_letter_new}/g" ../../workflow/job-files/main/${jobline_no}.job 142 | fi 143 | 144 | -------------------------------------------------------------------------------- /tools/templates/template1.bash.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Job Information 21 | ################################################################################## 22 | 23 | echo 24 | echo " *** Job Information *** " 25 | echo "===============================================================" 26 | echo 27 | echo "Environment variables" 28 | echo "------------------------" 29 | env 30 | echo 31 | echo 32 | 33 | 34 | # Running the Job - Screening of the Ligands 35 | ###################################################################### 36 | echo 37 | echo " *** Job Output *** " 38 | echo "===========================================================" 39 | echo 40 | 41 | # Functions 42 | # Standard error response 43 | error_response_std() { 44 | 45 | # Printing some informatoin 46 | echo "Error was trapped" 1>&2 47 | echo "Error in bash script $0" 1>&2 48 | echo "Error on line $1" 1>&2 49 | echo "Environment variables" 1>&2 50 | echo "----------------------------------" 1>&2 51 | env 1>&2 52 | 53 | # Checking error response type 54 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]] || [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 55 | 56 | # Printing some information 57 | echo -e "\n * Trying to continue..." 58 | 59 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 60 | 61 | # Printing some information 62 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 63 | print_job_infos_end 64 | 65 | # Exiting 66 | exit 1 67 | fi 68 | } 69 | trap 'error_response_std $LINENO' ERR 70 | 71 | # Handling signals 72 | time_near_limit() { 73 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal. Trying to start a new job." 74 | } 75 | trap 'time_near_limit' 10 76 | 77 | termination_signal() { 78 | echo "The script ${BASH_SOURCE[0]} caught a termination signal. Stopping jobline." 79 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 80 | echo -e "\n Ignoring error. Trying to continue..." 81 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 82 | echo -e "\n Ignoring error. Trying to continue and start next job..." 83 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 84 | echo -e "\n Stopping the jobline." 85 | print_job_infos_end 86 | exit 1 87 | fi 88 | } 89 | trap 'termination_signal' 1 2 3 9 12 15 90 | 91 | # Printing final job information 92 | print_job_infos_end() { 93 | # Job information 94 | echo 95 | echo " *** Final Job Information *** " 96 | echo "======================================================================" 97 | echo 98 | echo "Starting time:" $VF_STARTINGTIME 99 | echo "Ending time: " $(date) 100 | echo 101 | } 102 | 103 | # Checking if the queue should be stopped 104 | check_queue_end1() { 105 | 106 | # Determining the VF_CONTROLFILE to use for this jobline 107 | VF_CONTROLFILE="" 108 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 109 | file_basename=$(basename $file) 110 | jobline_range=${file_basename/.*} 111 | VF_JOBLINE_NO_START=${jobline_range/-*} 112 | VF_JOBLINE_NO_END=${jobline_range/*-} 113 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 114 | export VF_CONTROLFILE="${file}" 115 | break 116 | fi 117 | done 118 | if [ -z "${VF_CONTROLFILE}" ]; then 119 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 120 | fi 121 | 122 | # Checking if the queue should be stopped 123 | line="$(cat ${VF_CONTROLFILE} | grep "stop_after_next_check_interval=")" 124 | stop_after_next_check_interval=${line/"stop_after_next_check_interval="} 125 | if [ "${stop_after_next_check_interval}" = "true" ]; then 126 | echo 127 | echo "This job line was stopped by the stop_after_next_check_interval flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 128 | echo 129 | print_job_infos_end 130 | exit 0 131 | fi 132 | 133 | # Checking if there are still ligand collections todo 134 | no_collections_incomplete="0" 135 | i=0 136 | # Using a loop to try several times if there are no ligand collections left - maybe the files where just shortly inaccessible 137 | while [ "${no_collections_incomplete}" == "0" ]; do 138 | no_collections_incomplete="$(cat ../workflow/ligand-collections/todo/todo.all* ../workflow/ligand-collections/todo/${VF_JOBLINE_NO}/*/* ../workflow/ligand-collections/current/${VF_JOBLINE_NO}/*/* 2>/dev/null | grep -c "[^[:blank:]]" || true)" 139 | i="$((i + 1))" 140 | if [ "${i}" == "5" ]; then 141 | break 142 | fi 143 | sleep 1 144 | done 145 | if [[ "${no_collections_incomplete}" = "0" ]]; then 146 | echo 147 | echo "This job line was stopped because there are no ligand collections left." 148 | echo 149 | print_job_infos_end 150 | exit 0 151 | fi 152 | } 153 | 154 | check_queue_end2() { 155 | check_queue_end1 156 | line=$(cat ${VF_CONTROLFILE} | grep "stop_after_job=") 157 | stop_after_job=${line/"stop_after_job="} 158 | if [ "${stop_after_job}" = "true" ]; then 159 | echo 160 | echo "This job line was stopped by the stop_after_job flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 161 | echo 162 | print_job_infos_end 163 | exit 0 164 | fi 165 | } 166 | 167 | # Setting important variables 168 | job_line=$(grep -m 1 "job-name=" $0) 169 | jobname="b1" 170 | export VF_OLD_JOB_NO=${jobname:2} 171 | export VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 172 | export VF_QUEUE_NO_1=${VF_OLD_JOB_NO/.*} 173 | export VF_JOBLINE_NO=${VF_QUEUE_NO_1} 174 | export VF_BATCHSYSTEM="BASH" 175 | export VF_SLEEP_TIME_1="1" 176 | export VF_STARTINGTIME=`date` 177 | export VF_START_TIME_SECONDS="$(date +%s)" 178 | job_line=$(grep -m 1 "nodes=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 179 | export VF_NODES_PER_JOB=1 180 | export LC_ALL=C 181 | export PATH="./bin:$PATH" # to give bin priority of system commands, useful for obabel sometimes for example 182 | 183 | # Determining the VF_CONTROLFILE to use for this jobline 184 | VF_CONTROLFILE="" 185 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 186 | file_basename=$(basename $file) 187 | jobline_range=${file_basename/.*} 188 | VF_JOBLINE_NO_START=${jobline_range/-*} 189 | VF_JOBLINE_NO_END=${jobline_range/*-} 190 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 191 | export VF_CONTROLFILE="${file}" 192 | break 193 | fi 194 | done 195 | if [ -z "${VF_CONTROLFILE}" ]; then 196 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 197 | fi 198 | 199 | # Verbosity 200 | VF_VERBOSITY_LOGFILES="$(grep -m 1 "^verbosity_logfiles=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 201 | export VF_VERBOSITY_LOGFILES 202 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 203 | set -x 204 | fi 205 | 206 | # Setting the error response 207 | VF_ERROR_RESPONSE="$(grep -m 1 "^error_response=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 208 | export VF_ERROR_RESPONSE 209 | 210 | # VF_TMPDIR 211 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 212 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 213 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 214 | mkdir -p ${VF_TMPDIR}/${USER} 215 | fi 216 | 217 | # VF_TMPDIR_FAST 218 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 219 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 220 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 221 | mkdir -p ${VF_TMPDIR}/${USER} 222 | fi 223 | 224 | # Setting the job letter 225 | VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 226 | export VF_JOBLETTER 227 | 228 | # Setting the error sensitivity 229 | VF_ERROR_SENSITIVITY="$(grep -m 1 "^error_sensitivity=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 230 | export VF_ERROR_SENSITIVITY 231 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 232 | set -uo pipefail 233 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 234 | fi 235 | 236 | # Checking if the queue should be stopped 237 | check_queue_end1 238 | 239 | # Getting the available wallclock time 240 | job_line=$(grep -m 1 "time=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 241 | timelimit=10000-00:00:00 242 | timelimit=${timelimit//-/:} 243 | export VF_TIMELIMIT_SECONDS="$(echo -n "${timelimit}" | awk -F ':' '{print $4 + $3 * 60 + $2 * 3600 + $1 * 3600 * 24}')" 244 | 245 | # Getting the number of queues per step 246 | line=$(cat ${VF_CONTROLFILE} | grep "queues_per_step=") 247 | export VF_QUEUES_PER_STEP=${line/"queues_per_step="} 248 | 249 | # Preparing the todo lists for the queues 250 | prepare_queue_todolists="$(grep -m 1 "^prepare_queue_todolists=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 251 | if [ "${prepare_queue_todolists^^}" == "TRUE" ]; then 252 | cd helpers 253 | bash prepare-todolists.sh ${VF_JOBLINE_NO} ${VF_NODES_PER_JOB} ${VF_QUEUES_PER_STEP} 254 | cd .. 255 | elif [ "${prepare_queue_todolists^^}" == "FALSE" ]; then 256 | echo " * Skipping the todo-list preparation as specified in the control-file." 257 | else 258 | echo "Error: The variable prepare_queue_todolists in the control file ${VF_CONTROLFILE} has an unsupported value (${prepare_queue_todolists})." 259 | false 260 | fi 261 | 262 | # Starting the individual steps on different nodes 263 | for VF_STEP_NO in $(seq 1 ${VF_NODES_PER_JOB} ); do 264 | export VF_STEP_NO 265 | echo "Starting job step ${VF_STEP_NO} on host $(hostname)." 266 | srun --relative=$((VF_STEP_NO - 1)) -n 1 -N 1 ../workflow/job-files/sub/one-step.sh & 267 | pids[$(( VF_STEP_NO - 0 ))]=$! 268 | sleep "${VF_SLEEP_TIME_1}" 269 | done 270 | 271 | # Waiting for all steps to finish 272 | 273 | # Checking if all queues exited without error ("wait" waits for all of them, but always returns 0) 274 | exit_code=0 275 | for pid in ${pids[@]}; do 276 | wait $pid || let "exit_code=1" 277 | done 278 | if [ "$exit_code" == "1" ]; then 279 | error_response_std $LINENO 280 | fi 281 | 282 | 283 | # Creating the next job 284 | ##################################################################################### 285 | echo 286 | echo 287 | echo " *** Preparing the next batch system job *** " 288 | echo "==================================================================================" 289 | echo 290 | 291 | # Checking if the queue should be stopped 292 | check_queue_end2 293 | 294 | # Syncing the new jobfile with the settings in the VF_CONTROLFILE 295 | cd helpers 296 | . sync-jobfile.sh ${VF_JOBLINE_NO} 297 | cd .. 298 | 299 | # Changing the job name 300 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 301 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 302 | 303 | # Checking how much time has passed since the job has been started 304 | end_time_seconds="$(date +%s)" 305 | time_diff="$((end_time_seconds - VF_START_TIME_SECONDS))" 306 | treshhold=120 307 | if [ "${time_diff}" -le "${treshhold}" ]; then 308 | echo "Since the beginning of the job less than ${treshhold} seconds have passed." 309 | echo "Sleeping for some while to prevent a job submission run..." 310 | sleep ${treshhold} 311 | fi 312 | 313 | # Submitting a new new job 314 | cd helpers 315 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 316 | cd .. 317 | 318 | 319 | # Finalizing the job 320 | ##################################################################################### 321 | print_job_infos_end 322 | -------------------------------------------------------------------------------- /tools/templates/template1.sge.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # SGE Settings 21 | ############################################################################### 22 | 23 | #$ -S /bin/bash 24 | #$ -cwd 25 | #$ -N h-1.1 26 | #$ -o ../workflow/output-files/jobs/job-1.1_$JOB_ID.out 27 | #$ -e ../workflow/output-files/jobs/job-1.1_$JOB_ID.out 28 | #$ -l h_rt=12:00:00 29 | #$ -q 30 | #$ -m a 31 | #$ -M 32 | #$ -notify 33 | 34 | # Job Information 35 | ################################################################################## 36 | 37 | echo 38 | echo " *** Job Information *** " 39 | echo "===============================================================" 40 | echo 41 | echo "Environment variables" 42 | echo "------------------------" 43 | env 44 | echo 45 | echo 46 | echo "*** Job Infos by checkjob and qstat -f ***" 47 | echo "--------------------------------------------------" 48 | qstat -j $JOB_ID 49 | 50 | # Running the Job - Screening of the Ligands 51 | ###################################################################### 52 | echo 53 | echo " *** Job Output *** " 54 | echo "===========================================================" 55 | echo 56 | 57 | # Functions 58 | # Standard error response 59 | error_response_std() { 60 | 61 | # Printing some informatoin 62 | echo "Error was trapped" 1>&2 63 | echo "Error in bash script $0" 1>&2 64 | echo "Error on line $1" 1>&2 65 | echo "Environment variables" 1>&2 66 | echo "----------------------------------" 1>&2 67 | env 1>&2 68 | 69 | # Checking error response type 70 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]] || [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 71 | 72 | # Printing some information 73 | echo -e "\n * Trying to continue..." 74 | 75 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 76 | 77 | # Printing some information 78 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 79 | print_job_infos_end 80 | 81 | # Exiting 82 | exit 1 83 | fi 84 | } 85 | trap 'error_response_std $LINENO' ERR 86 | 87 | # Handling signals 88 | time_near_limit() { 89 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal." 90 | } 91 | trap 'time_near_limit' 10 92 | 93 | termination_signal() { 94 | echo "The script ${BASH_SOURCE[0]} caught a termination signal. Stopping jobline." 95 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 96 | echo -e "\n Ignoring error. Trying to continue..." 97 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 98 | echo -e "\n Ignoring error. Trying to continue and start next job..." 99 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 100 | echo -e "\n Stopping the jobline." 101 | print_job_infos_end 102 | exit 1 103 | fi 104 | } 105 | trap 'termination_signal' 1 2 3 9 12 15 106 | 107 | # Printing final job information 108 | print_job_infos_end() { 109 | # Job information 110 | echo 111 | echo " *** Final Job Information *** " 112 | echo "======================================================================" 113 | echo 114 | echo "Starting time:" $VF_STARTINGTIME 115 | echo "Ending time: " $(date) 116 | echo 117 | } 118 | 119 | # Checking if the queue should be stopped 120 | check_queue_end1() { 121 | 122 | # Determining the VF_CONTROLFILE to use for this jobline 123 | VF_CONTROLFILE="" 124 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 125 | file_basename=$(basename $file) 126 | jobline_range=${file_basename/.*} 127 | VF_JOBLINE_NO_START=${jobline_range/-*} 128 | VF_JOBLINE_NO_END=${jobline_range/*-} 129 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 130 | export VF_CONTROLFILE="${file}" 131 | break 132 | fi 133 | done 134 | if [ -z "${VF_CONTROLFILE}" ]; then 135 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 136 | fi 137 | 138 | # Checking if the queue should be stopped 139 | line="$(cat ${VF_CONTROLFILE} | grep "stop_after_next_check_interval=")" 140 | stop_after_next_check_interval=${line/"stop_after_next_check_interval="} 141 | if [[ "${stop_after_next_check_interval}" == "true" ]]; then 142 | echo 143 | echo "This job line was stopped by the stop_after_next_check_interval flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 144 | echo 145 | print_job_infos_end 146 | exit 0 147 | fi 148 | 149 | # Checking if there are still ligand collections todo 150 | no_collections_incomplete="0" 151 | i=0 152 | # Using a loop to try several times if there are no ligand collections left - maybe the files where just shortly inaccessible 153 | while [ "${no_collections_incomplete}" == "0" ]; do 154 | no_collections_incomplete="$(cat ../workflow/ligand-collections/todo/todo.all* ../workflow/ligand-collections/todo/${VF_JOBLINE_NO}/*/* ../workflow/ligand-collections/current/${VF_JOBLINE_NO}/*/* 2>/dev/null | grep -c "[^[:blank:]]" || true)" 155 | i="$((i + 1))" 156 | if [ "${i}" == "5" ]; then 157 | break 158 | fi 159 | sleep 1 160 | done 161 | if [[ "${no_collections_incomplete}" = "0" ]]; then 162 | echo 163 | echo "This job line was stopped because there are no ligand collections left." 164 | echo 165 | print_job_infos_end 166 | exit 0 167 | fi 168 | } 169 | 170 | check_queue_end2() { 171 | check_queue_end1 172 | line=$(cat ${VF_CONTROLFILE} | grep "stop_after_job=") 173 | stop_after_job=${line/"stop_after_job="} 174 | if [ "${stop_after_job}" = "true" ]; then 175 | echo 176 | echo "This job line was stopped by the stop_after_job flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 177 | echo 178 | print_job_infos_end 179 | exit 0 180 | fi 181 | } 182 | 183 | # Setting important variables 184 | export VF_NODES_PER_JOB=${PBS_NUM_NODES} ###??? 185 | export VF_OLD_JOB_NO=${JOB_NAME:2} 186 | export VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 187 | export VF_QUEUE_NO_1=${VF_OLD_JOB_NO/.*} 188 | export VF_JOBLINE_NO=${VF_QUEUE_NO_1} 189 | export VF_BATCHSYSTEM="SGE" 190 | export VF_SLEEP_TIME_1="1" 191 | VF_STARTINGTIME=`date` 192 | export VF_START_TIME_SECONDS="$(date +%s)" 193 | date 194 | echo $VF_START_TIME_SECONDS 195 | export LC_ALL=C 196 | export PATH="./bin:$PATH" # to give bin priority of system commands, useful for obabel sometimes for example 197 | 198 | 199 | # Determining the VF_CONTROLFILE to use for this jobline 200 | VF_CONTROLFILE="" 201 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 202 | file_basename=$(basename $file) 203 | jobline_range=${file_basename/.*} 204 | VF_JOBLINE_NO_START=${jobline_range/-*} 205 | VF_JOBLINE_NO_END=${jobline_range/*-} 206 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 207 | export VF_CONTROLFILE="${file}" 208 | break 209 | fi 210 | done 211 | if [ -z "${VF_CONTROLFILE}" ]; then 212 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 213 | fi 214 | 215 | # Verbosity 216 | VF_VERBOSITY_LOGFILES="$(grep -m 1 "^verbosity_logfiles=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 217 | export VF_VERBOSITY_LOGFILES 218 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 219 | set -x 220 | fi 221 | 222 | # VF_TMPDIR 223 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 224 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 225 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 226 | mkdir -p ${VF_TMPDIR}/${USER} 227 | fi 228 | 229 | # VF_TMPDIR_FAST 230 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 231 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 232 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 233 | mkdir -p ${VF_TMPDIR}/${USER} 234 | fi 235 | # Setting the job letter 236 | VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 237 | export VF_JOBLETTER 238 | 239 | # Setting the error sensitivity 240 | VF_ERROR_SENSITIVITY="$(grep -m 1 "^error_sensitivity=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 241 | export VF_ERROR_SENSITIVITY 242 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 243 | set -uo pipefail 244 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 245 | fi 246 | 247 | # Setting the error response 248 | VF_ERROR_RESPONSE="$(grep -m 1 "^error_response=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 249 | export VF_ERROR_RESPONSE 250 | 251 | # Checking if queue should be stopped 252 | check_queue_end1 253 | 254 | # Getting the available wallclock time 255 | job_line=$(grep -m 1 "h_rt=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 256 | timelimit=${job_line/\#\$ -l h_rt=} 257 | export VF_TIMELIMIT_SECONDS="$(echo -n "${timelimit}" | awk -F ':' '{print $3 + $2 * 60 + $1 * 3600}')" 258 | 259 | # Getting the number of queues per step 260 | line=$(cat ${VF_CONTROLFILE} | grep -m 1 "queues_per_step=") 261 | export VF_QUEUES_PER_STEP=${line/"queues_per_step="} 262 | 263 | # Preparing the todo lists for the queues 264 | prepare_queue_todolists="$(grep -m 1 "^prepare_queue_todolists=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 265 | if [ "${prepare_queue_todolists^^}" == "TRUE" ]; then 266 | cd helpers 267 | bash prepare-todolists.sh ${VF_JOBLINE_NO} ${VF_NODES_PER_JOB} ${VF_QUEUES_PER_STEP} 268 | cd .. 269 | elif [ "${prepare_queue_todolists^^}" == "FALSE" ]; then 270 | echo " * Skipping the todo-list preparation as specified in the control-file." 271 | else 272 | echo "Error: The variable prepare_queue_todolists in the control file ${VF_CONTROLFILE} has an unsupported value (${prepare_queue_todolists})." 273 | false 274 | fi 275 | 276 | # Starting the individual steps on different nodes 277 | for VF_STEP_NO in $(seq 1 ${VF_NODES_PER_JOB} ); do 278 | export VF_STEP_NO 279 | echo "Starting job step VF_STEP_NO on host $(hostname)." 280 | bash ../workflow/job-files/sub/one-step.sh & 281 | pids[$(( VF_STEP_NO - 0 ))]=$! 282 | sleep "${VF_SLEEP_TIME_1}" 283 | done 284 | 285 | # Waiting for all steps to finish 286 | 287 | # Checking if all queues exited without error ("wait" waits for all of them, but always returns 0) 288 | exit_code=0 289 | for pid in ${pids[@]}; do 290 | wait $pid || let "exit_code=1" 291 | done 292 | if [ "$exit_code" == "1" ]; then 293 | error_response_std 294 | fi 295 | 296 | 297 | # Creating the next job 298 | ##################################################################################### 299 | echo 300 | echo 301 | echo " *** Preparing the next batch system job *** " 302 | echo "==================================================================================" 303 | echo 304 | 305 | # Checking if the queue should be stopped 306 | check_queue_end2 307 | 308 | # Syncing the new jobfile with the settings in the VF_CONTROLFILE 309 | cd helpers 310 | . sync-jobfile.sh ${VF_JOBLINE_NO} 311 | cd .. 312 | 313 | # Changing the job name 314 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 315 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 316 | sed -i "s/^#\\$ -N ${VF_JOBLETTER}-.*/#\$ -N ${VF_JOBLETTER}-${new_job_no}/g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 317 | 318 | # Changing the output filenames 319 | sed -i "s|^#\\$ -\([oe]\) .*|#\$ -\1 ../workflow/output-files/jobs/job-${new_job_no}_\${JOB_ID}.out|g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 320 | 321 | # Checking how much time has passed since the job has been started 322 | end_time_seconds="$(date +%s)" 323 | time_diff="$((end_time_seconds - VF_START_TIME_SECONDS))" 324 | treshhold=100 325 | if [ "${time_diff}" -le "${treshhold}" ]; then 326 | echo "Since the beginning of the job less than ${treshhold} seconds have passed." 327 | echo "Sleeping for some while to prevent a job submission run..." 328 | sleep 120 329 | fi 330 | 331 | 332 | # Submitting a new new job 333 | cd helpers 334 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 335 | cd .. 336 | 337 | 338 | # Finalizing the job 339 | ##################################################################################### 340 | print_job_infos_end 341 | -------------------------------------------------------------------------------- /tools/templates/template1.lsf.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # LSF Settings 21 | ############################################################################### 22 | 23 | #BSUB -J h-1.1 24 | #BSUB -W 00:30 25 | #BSUB -R "rusage[mem=400]" 26 | #BSUB -R "span[ptile=4]" 27 | #BSUB -n 4 28 | #BSUB -q medium 29 | #BSUB -oo ../workflow/output-files/jobs/job-1.1_%J.out # File to which standard out will be written 30 | #BSUB -wt '5' 31 | #BSUB -wa 'USR1' 32 | 33 | # Job Information 34 | ################################################################################## 35 | echo 36 | echo " *** Job Information *** " 37 | echo "===============================================================" 38 | echo 39 | echo "Environment variables" 40 | echo "------------------------" 41 | env 42 | echo 43 | echo 44 | echo "Job infos by bjobs" 45 | echo "------------------------" 46 | bjobs $LSB_JOBID 47 | 48 | # Running the Job - Screening of the Ligands 49 | ###################################################################### 50 | echo 51 | echo " *** Job Output *** " 52 | echo "===========================================================" 53 | echo 54 | 55 | # Functions 56 | # Standard error response 57 | error_response_std() { 58 | 59 | # Printing some informatoin 60 | echo "Error was trapped" 1>&2 61 | echo "Error in bash script $0" 1>&2 62 | echo "Error on line $1" 1>&2 63 | echo "Environment variables" 1>&2 64 | echo "----------------------------------" 1>&2 65 | env 1>&2 66 | 67 | # Checking error response type 68 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]] || [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 69 | 70 | # Printing some information 71 | echo -e "\n * Trying to continue..." 72 | 73 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 74 | 75 | # Printing some information 76 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 77 | print_job_infos_end 78 | 79 | # Exiting 80 | exit 1 81 | fi 82 | } 83 | trap 'error_response_std $LINENO' ERR 84 | 85 | # Handling signals 86 | time_near_limit() { 87 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal. Trying to start a new job." 88 | } 89 | trap 'time_near_limit' 10 90 | 91 | termination_signal() { 92 | echo "The script ${BASH_SOURCE[0]} caught a termination signal. Stopping jobline." 93 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 94 | echo -e "\n Ignoring error. Trying to continue..." 95 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 96 | echo -e "\n Ignoring error. Trying to continue and start next job..." 97 | print_job_infos_end 98 | exit 0 99 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 100 | echo -e "\n Stopping the jobline." 101 | print_job_infos_end 102 | exit 1 103 | fi 104 | } 105 | trap 'termination_signal' 1 2 3 9 12 15 106 | 107 | # Printing final job information 108 | print_job_infos_end() { 109 | # Job information 110 | echo 111 | echo " *** Final Job Information *** " 112 | echo "======================================================================" 113 | echo 114 | echo "Starting time:" $VF_STARTINGTIME 115 | echo "Ending time: " $(date) 116 | echo 117 | } 118 | 119 | # Checking if the queue should be stopped 120 | check_queue_end1() { 121 | 122 | # Determining the VF_CONTROLFILE to use for this jobline 123 | VF_CONTROLFILE="" 124 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 125 | file_basename=$(basename $file) 126 | jobline_range=${file_basename/.*} 127 | VF_JOBLINE_NO_START=${jobline_range/-*} 128 | VF_JOBLINE_NO_END=${jobline_range/*-} 129 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 130 | export VF_CONTROLFILE="${file}" 131 | break 132 | fi 133 | done 134 | if [ -z "${VF_CONTROLFILE}" ]; then 135 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 136 | fi 137 | 138 | # Checking if the queue should be stopped 139 | line="$(cat ${VF_CONTROLFILE} | grep "stop_after_next_check_interval=")" 140 | stop_after_next_check_interval=${line/"stop_after_next_check_interval="} 141 | if [ "${stop_after_next_check_interval}" = "true" ]; then 142 | echo 143 | echo "This job line was stopped by the stop_after_next_check_interval flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 144 | echo 145 | print_job_infos_end 146 | exit 0 147 | fi 148 | 149 | # Checking if there are still ligand collections to-do 150 | no_collections_incomplete="0" 151 | i=0 152 | # Using a loop to try several times if there are no ligand collections left - maybe the files where just shortly inaccessible 153 | while [ "${no_collections_incomplete}" == "0" ]; do 154 | no_collections_incomplete="$(cat ../workflow/ligand-collections/todo/todo.all* ../workflow/ligand-collections/todo/${VF_JOBLINE_NO}/*/* ../workflow/ligand-collections/current/${VF_JOBLINE_NO}/*/* 2>/dev/null | grep -c "[^[:blank:]]" || true)" 155 | i="$((i + 1))" 156 | if [ "${i}" == "5" ]; then 157 | break 158 | fi 159 | sleep 1 160 | done 161 | if [[ "${no_collections_incomplete}" = "0" ]]; then 162 | echo 163 | echo "This job line was stopped because there are no ligand collections left." 164 | echo 165 | print_job_infos_end 166 | exit 0 167 | fi 168 | } 169 | 170 | check_queue_end2() { 171 | check_queue_end1 172 | line=$(cat ${VF_CONTROLFILE} | grep "stop_after_job=") 173 | stop_after_job=${line/"stop_after_job="} 174 | if [ "${stop_after_job}" = "true" ]; then 175 | echo 176 | echo "This job line was stopped by the stop_after_job flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 177 | echo 178 | print_job_infos_end 179 | exit 0 180 | fi 181 | } 182 | 183 | 184 | # Setting important variables 185 | job_line=$(grep -m 1 -- "-J " $0) 186 | jobname=${job_line/"#BSUB -J "} 187 | export VF_OLD_JOB_NO=${jobname/[a-zA-Z]-} 188 | export VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 189 | export VF_QUEUE_NO_1=${VF_OLD_JOB_NO/.*} 190 | export VF_JOBLINE_NO=${VF_QUEUE_NO_1} 191 | export VF_BATCHSYSTEM="LSF" 192 | export VF_SLEEP_TIME_1="1" 193 | export VF_STARTINGTIME=`date` 194 | export VF_START_TIME_SECONDS="$(date +%s)" 195 | #export job_line=$(grep -m 1 "nodes=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 196 | #export VF_NODES_PER_JOB=${job_line/"#SBATCH --nodes="} 197 | #export VF_NODES_PER_JOB=${SLURM_JOB_NUM_NODES} 198 | export VF_NODES_PER_JOB=1 199 | export LC_ALL=C 200 | export LANG=C 201 | export PATH="./bin:$PATH" # to give bin priority of system commands, useful for obabel sometimes for example 202 | 203 | 204 | # Determining the VF_CONTROLFILE to use for this jobline 205 | VF_CONTROLFILE="" 206 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 207 | file_basename=$(basename $file) 208 | jobline_range=${file_basename/.*} 209 | VF_JOBLINE_NO_START=${jobline_range/-*} 210 | VF_JOBLINE_NO_END=${VF_JOBLINE_NO_END/-*} 211 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 212 | export VF_CONTROLFILE="${file}" 213 | break 214 | fi 215 | done 216 | if [ -z "${VF_CONTROLFILE}" ]; then 217 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 218 | fi 219 | 220 | # Verbosity 221 | VF_VERBOSITY_LOGFILES="$(grep -m 1 "^verbosity_logfiles=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 222 | export VF_VERBOSITY_LOGFILES 223 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 224 | set -x 225 | fi 226 | 227 | # VF_TMPDIR 228 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 229 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 230 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 231 | mkdir -p ${VF_TMPDIR}/${USER} 232 | fi 233 | 234 | # VF_TMPDIR_FAST 235 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 236 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 237 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 238 | mkdir -p ${VF_TMPDIR}/${USER} 239 | fi 240 | 241 | # Setting the job letter 242 | VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 243 | export VF_JOBLETTER 244 | 245 | # Setting the error sensitivity 246 | VF_ERROR_SENSITIVITY="$(grep -m 1 "^error_sensitivity=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 247 | export VF_ERROR_SENSITIVITY 248 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 249 | set -uo pipefail 250 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 251 | fi 252 | 253 | # Setting the error response 254 | VF_ERROR_RESPONSE="$(grep -m 1 "^error_response=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 255 | export VF_ERROR_RESPONSE 256 | 257 | # Checking if the queue should be stopped 258 | check_queue_end1 259 | 260 | # Getting the available wallclock time 261 | job_line=$(grep -m 1 -- "-W " ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 262 | timelimit=${job_line/"#BSUB -W"} 263 | export VF_TIMELIMIT_SECONDS="$(echo -n "${timelimit}" | awk -F ':' '{print $2 * 60 + $1 * 3600 }')" 264 | 265 | # Getting the number of queues per step 266 | line=$(cat ${VF_CONTROLFILE} | grep -m 1 "queues_per_step=") 267 | export VF_QUEUES_PER_STEP=${line/"queues_per_step="} 268 | 269 | # Preparing the to-do lists for the queues 270 | prepare_queue_todolists="$(grep -m 1 "^prepare_queue_todolists=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 271 | if [ "${prepare_queue_todolists^^}" == "TRUE" ]; then 272 | cd helpers 273 | bash prepare-todolists.sh ${VF_JOBLINE_NO} ${VF_NODES_PER_JOB} ${VF_QUEUES_PER_STEP} 274 | cd .. 275 | elif [ "${prepare_queue_todolists^^}" == "FALSE" ]; then 276 | echo " * Skipping the todo-list preparation as specified in the control-file." 277 | else 278 | echo "Error: The variable prepare_queue_todolists in the control file ${VF_CONTROLFILE} has an unsupported value (${prepare_queue_todolists})." 279 | false 280 | fi 281 | 282 | # Starting the individual steps on different nodes 283 | for VF_STEP_NO in $(seq 1 ${VF_NODES_PER_JOB} ); do 284 | export VF_STEP_NO 285 | echo "Starting job step VF_STEP_NO on host $(hostname)." 286 | bash ../workflow/job-files/sub/one-step.sh & 287 | sleep "${VF_SLEEP_TIME_1}" 288 | done 289 | 290 | 291 | # Waiting for all steps to finish 292 | wait 293 | 294 | 295 | # Creating the next job 296 | ##################################################################################### 297 | echo 298 | echo 299 | echo " *** Preparing the next batch system job *** " 300 | echo "==================================================================================" 301 | echo 302 | 303 | # Checking if the queue should be stopped 304 | check_queue_end2 305 | 306 | # Syncing the new jobfile with the settings in the VF_CONTROLFILE 307 | cd helpers 308 | . sync-jobfile.sh ${VF_JOBLINE_NO} 309 | cd .. 310 | 311 | # Changing the job name 312 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 313 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 314 | sed -i "s/^#BSUB -J ${VF_JOBLETTER}-.*/#BSUB -J ${VF_JOBLETTER}-${new_job_no}/g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 315 | 316 | # Changing the output filenames 317 | sed -i "s|^#BSUB -oo ../workflow/output-files/jobs/job-${VF_OLD_JOB_NO}_%J.out|#BSUB -oo ../workflow/output-files/jobs/job-${new_job_no}_%J.out|g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 318 | 319 | # Checking how much time has passed since the job has been started 320 | end_time_seconds="$(date +%s)" 321 | time_diff="$((end_time_seconds - VF_START_TIME_SECONDS))" 322 | treshhold=100 323 | if [ "${time_diff}" -le "${treshhold}" ]; then 324 | echo "Since the beginning of the job less than ${treshhold} seconds have passed." 325 | echo "Sleeping for some while to prevent a job submission run..." 326 | sleep 120 327 | fi 328 | 329 | # Submitting a new new job 330 | cd helpers 331 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 332 | cd .. 333 | 334 | 335 | # Finalizing the job 336 | ##################################################################################### 337 | print_job_infos_end 338 | -------------------------------------------------------------------------------- /tools/templates/template1.torque.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # PBS/Moab Settings 21 | ############################################################################### 22 | 23 | #PBS -N h-1.1 24 | #PBS -l nodes=1:ppn=24 25 | #PBS -l naccesspolicy=singlejob 26 | #PBS -o ../workflow/output-files/jobs/job-1.1_${PBS_JOBID}.out 27 | #PBS -e ../workflow/output-files/jobs/job-1.1_${PBS_JOBID}.out 28 | #PBS -l walltime=00:12:00 29 | ##PBS -A 30 | #PBS -q mpp2testq 31 | #PBS -m a 32 | #PBS -M 33 | #PBS -l signal=10@300 34 | 35 | # Job Information 36 | ################################################################################## 37 | 38 | echo 39 | echo " *** Job Information *** " 40 | echo "===============================================================" 41 | echo 42 | echo "Environment variables" 43 | echo "------------------------" 44 | env 45 | echo 46 | echo 47 | echo "*** Job Infos by checkjob and qstat -f ***" 48 | echo "--------------------------------------------------" 49 | checkjob $PBS_JOBID 50 | qstat -f $PBS_JOBID 51 | 52 | # Running the Job - Screening of the Ligands 53 | ###################################################################### 54 | echo 55 | echo " *** Job Output *** " 56 | echo "===========================================================" 57 | echo 58 | 59 | # Functions 60 | # Standard error response 61 | error_response_std() { 62 | 63 | # Printing some informatoin 64 | echo "Error was trapped" 1>&2 65 | echo "Error in bash script $0" 1>&2 66 | echo "Error on line $1" 1>&2 67 | echo "Environment variables" 1>&2 68 | echo "----------------------------------" 1>&2 69 | env 1>&2 70 | 71 | # Checking error response type 72 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]] || [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 73 | 74 | # Printing some information 75 | echo -e "\n * Trying to continue..." 76 | 77 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 78 | 79 | # Printing some information 80 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 81 | print_job_infos_end 82 | 83 | # Exiting 84 | exit 1 85 | fi 86 | } 87 | trap 'error_response_std $LINENO' ERR 88 | 89 | # Handling signals 90 | time_near_limit() { 91 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal." 92 | } 93 | trap 'time_near_limit' 10 94 | 95 | termination_signal() { 96 | echo "The script ${BASH_SOURCE[0]} caught a termination signal. Stopping jobline." 97 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 98 | echo -e "\n Ignoring error. Trying to continue..." 99 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 100 | echo -e "\n Ignoring error. Trying to continue and start next job..." 101 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 102 | echo -e "\n Stopping the jobline." 103 | print_job_infos_end 104 | exit 1 105 | fi 106 | } 107 | trap 'termination_signal' 1 2 3 9 12 15 108 | 109 | # Printing final job information 110 | print_job_infos_end() { 111 | # Job information 112 | echo 113 | echo " *** Final Job Information *** " 114 | echo "======================================================================" 115 | echo 116 | echo "Starting time:" $VF_STARTINGTIME 117 | echo "Ending time: " $(date) 118 | echo 119 | } 120 | 121 | # Checking if the queue should be stopped 122 | check_queue_end1() { 123 | 124 | # Determining the VF_CONTROLFILE to use for this jobline 125 | VF_CONTROLFILE="" 126 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 127 | file_basename=$(basename $file) 128 | jobline_range=${file_basename/.*} 129 | VF_JOBLINE_NO_START=${jobline_range/-*} 130 | VF_JOBLINE_NO_END=${jobline_range/*-} 131 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 132 | export VF_CONTROLFILE="${file}" 133 | break 134 | fi 135 | done 136 | if [ -z "${VF_CONTROLFILE}" ]; then 137 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 138 | fi 139 | 140 | # Checking if the queue should be stopped 141 | line="$(cat ${VF_CONTROLFILE} | grep "stop_after_next_check_interval=")" 142 | stop_after_next_check_interval=${line/"stop_after_next_check_interval="} 143 | if [[ "${stop_after_next_check_interval}" == "true" ]]; then 144 | echo 145 | echo "This job line was stopped by the stop_after_next_check_interval flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 146 | echo 147 | print_job_infos_end 148 | exit 0 149 | fi 150 | 151 | # Checking if there are still ligand collections todo 152 | no_collections_incomplete="0" 153 | i=0 154 | # Using a loop to try several times if there are no ligand collections left - maybe the files where just shortly inaccessible 155 | while [ "${no_collections_incomplete}" == "0" ]; do 156 | no_collections_incomplete="$(cat ../workflow/ligand-collections/todo/todo.all* ../workflow/ligand-collections/todo/${VF_JOBLINE_NO}/*/* ../workflow/ligand-collections/current/${VF_JOBLINE_NO}/*/* 2>/dev/null | grep -c "[^[:blank:]]" || true)" 157 | i="$((i + 1))" 158 | if [ "${i}" == "5" ]; then 159 | break 160 | fi 161 | sleep 1 162 | done 163 | if [[ "${no_collections_incomplete}" = "0" ]]; then 164 | echo 165 | echo "This job line was stopped because there are no ligand collections left." 166 | echo 167 | print_job_infos_end 168 | exit 0 169 | fi 170 | } 171 | 172 | check_queue_end2() { 173 | check_queue_end1 174 | line=$(cat ${VF_CONTROLFILE} | grep "stop_after_job=") 175 | stop_after_job=${line/"stop_after_job="} 176 | if [ "${stop_after_job}" = "true" ]; then 177 | echo 178 | echo "This job line was stopped by the stop_after_job flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 179 | echo 180 | print_job_infos_end 181 | exit 0 182 | fi 183 | } 184 | 185 | 186 | # Setting important variables 187 | export VF_NODES_PER_JOB=${PBS_NUM_NODES} 188 | export VF_OLD_JOB_NO=${PBS_JOBNAME:2} 189 | export VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 190 | export VF_QUEUE_NO_1=${VF_OLD_JOB_NO/.*} 191 | export VF_JOBLINE_NO=${VF_QUEUE_NO_1} 192 | export VF_BATCHSYSTEM="TORQUE" 193 | export VF_SLEEP_TIME_1="1" 194 | export VF_STARTINGTIME=`date` 195 | export VF_START_TIME_SECONDS="$(date +%s)" 196 | export LC_ALL=C 197 | export PATH="./bin:$PATH" # to give bin priority of system commands, useful for obabel sometimes for example 198 | 199 | # Determining the VF_CONTROLFILE to use for this jobline 200 | VF_CONTROLFILE="" 201 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 202 | file_basename=$(basename $file) 203 | jobline_range=${file_basename/.*} 204 | VF_JOBLINE_NO_START=${jobline_range/-*} 205 | VF_JOBLINE_NO_END=${jobline_range/*-} 206 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 207 | export VF_CONTROLFILE="${file}" 208 | break 209 | fi 210 | done 211 | if [ -z "${VF_CONTROLFILE}" ]; then 212 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 213 | fi 214 | 215 | # Verbosity 216 | VF_VERBOSITY_LOGFILES="$(grep -m 1 "^verbosity_logfiles=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 217 | export VF_VERBOSITY_LOGFILES 218 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 219 | set -x 220 | fi 221 | 222 | # VF_TMPDIR 223 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 224 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 225 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 226 | mkdir -p ${VF_TMPDIR}/${USER} 227 | fi 228 | 229 | # VF_TMPDIR_FAST 230 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 231 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 232 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 233 | mkdir -p ${VF_TMPDIR}/${USER} 234 | fi 235 | 236 | # Setting the job letter 237 | VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 238 | export VF_JOBLETTER 239 | 240 | # Setting the error sensitivity 241 | VF_ERROR_SENSITIVITY="$(grep -m 1 "^error_sensitivity=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 242 | export VF_ERROR_SENSITIVITY 243 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 244 | set -uo pipefail 245 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 246 | fi 247 | 248 | # Setting the error response 249 | VF_ERROR_RESPONSE="$(grep -m 1 "^error_response=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 250 | export VF_ERROR_RESPONSE 251 | 252 | # Checking if queue should be stopped 253 | check_queue_end1 254 | 255 | # Getting the available wallclock time 256 | job_line=$(grep -m 1 "walltime=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 257 | timelimit=${job_line/\#PBS -l walltime=} 258 | export VF_TIMELIMIT_SECONDS="$(echo -n "${timelimit}" | awk -F ':' '{print $3 + $2 * 60 + $1 * 3600}')" 259 | 260 | # Getting the number of queues per step 261 | line=$(cat ${VF_CONTROLFILE} | grep -m 1 "queues_per_step=") 262 | export VF_QUEUES_PER_STEP=${line/"queues_per_step="} 263 | 264 | # Preparing the todo lists for the queues 265 | prepare_queue_todolists="$(grep -m 1 "^prepare_queue_todolists=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 266 | if [ "${prepare_queue_todolists^^}" == "TRUE" ]; then 267 | cd helpers 268 | bash prepare-todolists.sh ${VF_JOBLINE_NO} ${VF_NODES_PER_JOB} ${VF_QUEUES_PER_STEP} 269 | cd .. 270 | elif [ "${prepare_queue_todolists^^}" == "FALSE" ]; then 271 | echo " * Skipping the todo-list preparation as specified in the control-file." 272 | else 273 | echo "Error: The variable prepare_queue_todolists in the control file ${VF_CONTROLFILE} has an unsupported value (${prepare_queue_todolists})." 274 | false 275 | fi 276 | 277 | # Starting the individual steps on different nodes 278 | for VF_STEP_NO in $(seq 1 ${VF_NODES_PER_JOB} ); do 279 | export VF_STEP_NO 280 | echo "Starting job step VF_STEP_NO on host $(hostname)." 281 | aprun -n 1 -cc none ../workflow/job-files/sub/one-step.sh & 282 | pids[$(( VF_STEP_NO - 0 ))]=$! 283 | sleep "${VF_SLEEP_TIME_1}" 284 | done 285 | 286 | # Waiting for all steps to finish 287 | 288 | # Checking if all queues exited without error ("wait" waits for all of them, but always returns 0) 289 | exit_code=0 290 | for pid in ${pids[@]}; do 291 | wait $pid || let "exit_code=1" 292 | done 293 | if [ "$exit_code" == "1" ]; then 294 | error_response_std 295 | fi 296 | 297 | 298 | # Creating the next job 299 | ##################################################################################### 300 | echo 301 | echo 302 | echo " *** Preparing the next batch system job *** " 303 | echo "==================================================================================" 304 | echo 305 | 306 | # Checking if the queue should be stopped 307 | check_queue_end2 308 | 309 | # Syncing the new jobfile with the settings in the VF_CONTROLFILE 310 | cd helpers 311 | . sync-jobfile.sh ${VF_JOBLINE_NO} 312 | cd .. 313 | 314 | # Changing the job name 315 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 316 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 317 | sed -i "s/^#PBS -N ${VF_JOBLETTER}-.*/#PBS -N ${VF_JOBLETTER}-${new_job_no}/g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 318 | 319 | # Changing the output filenames 320 | sed -i "s|^#PBS -\([oe]\) .*|#PBS -\1 ../workflow/output-files/jobs/job-${new_job_no}_\${PBS_JOBID}.out|g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 321 | 322 | # Checking how much time has passed since the job has been started 323 | end_time_seconds="$(date +%s)" 324 | time_diff="$((end_time_seconds - VF_START_TIME_SECONDS))" 325 | treshhold=100 326 | if [ "${time_diff}" -le "${treshhold}" ]; then 327 | echo "Since the beginning of the job less than ${treshhold} seconds have passed." 328 | echo "Sleeping for some while to prevent a job submission run..." 329 | sleep 120 330 | fi 331 | 332 | 333 | # Submitting a new new job 334 | cd helpers 335 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 336 | cd .. 337 | 338 | 339 | # Finalizing the job 340 | ##################################################################################### 341 | print_job_infos_end 342 | -------------------------------------------------------------------------------- /tools/templates/template1.pbs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # PBS/Moab Settings 21 | ############################################################################### 22 | 23 | #PBS -N h-1.1 24 | #PBS -l nodes=1:ppn=24 25 | #PBS -l naccesspolicy=singlejob 26 | #PBS -o ../workflow/output-files/jobs/job-1.1_${PBS_JOBID}.out 27 | #PBS -e ../workflow/output-files/jobs/job-1.1_${PBS_JOBID}.out 28 | #PBS -l walltime=00:12:00 29 | #PBS -A bec00123 30 | #PBS -q mpp2testq 31 | #PBS -m a 32 | #PBS -M 33 | #PBS -l signal=10@300 34 | 35 | # Job Information 36 | ################################################################################## 37 | 38 | echo 39 | echo " *** Job Information *** " 40 | echo "===============================================================" 41 | echo 42 | echo "Environment variables" 43 | echo "------------------------" 44 | env 45 | echo 46 | echo 47 | echo "*** Job Infos by checkjob and qstat -f ***" 48 | echo "--------------------------------------------------" 49 | checkjob $PBS_JOBID 50 | qstat -f $PBS_JOBID 51 | 52 | # Running the Job - Screening of the Ligands 53 | ###################################################################### 54 | echo 55 | echo " *** Job Output *** " 56 | echo "===========================================================" 57 | echo 58 | 59 | # Functions 60 | # Standard error response 61 | error_response_std() { 62 | 63 | # Printing some informatoin 64 | echo "Error was trapped" 1>&2 65 | echo "Error in bash script $0" 1>&2 66 | echo "Error on line $1" 1>&2 67 | echo "Environment variables" 1>&2 68 | echo "----------------------------------" 1>&2 69 | env 1>&2 70 | 71 | # Checking error response type 72 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]] || [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 73 | 74 | # Printing some information 75 | echo -e "\n * Trying to continue..." 76 | 77 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 78 | 79 | # Printing some information 80 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 81 | print_job_infos_end 82 | 83 | # Exiting 84 | exit 1 85 | fi 86 | } 87 | trap 'error_response_std $LINENO' ERR 88 | 89 | # Handling signals 90 | time_near_limit() { 91 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal." 92 | } 93 | trap 'time_near_limit' 10 94 | 95 | termination_signal() { 96 | echo "The script ${BASH_SOURCE[0]} caught a termination signal. Stopping jobline." 97 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 98 | echo -e "\n Ignoring error. Trying to continue..." 99 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 100 | echo -e "\n Ignoring error. Trying to continue and start next job..." 101 | print_job_infos_end 102 | exit 0 103 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 104 | echo -e "\n Stopping the jobline." 105 | print_job_infos_end 106 | exit 1 107 | fi 108 | } 109 | trap 'termination_signal' 1 2 3 9 12 15 110 | 111 | # Printing final job information 112 | print_job_infos_end() { 113 | # Job information 114 | echo 115 | echo " *** Final Job Information *** " 116 | echo "======================================================================" 117 | echo 118 | echo "Starting time:" $VF_STARTINGTIME 119 | echo "Ending time: " $(date) 120 | echo 121 | } 122 | 123 | # Checking if the queue should be stopped 124 | check_queue_end1() { 125 | 126 | # Determining the VF_CONTROLFILE to use for this jobline 127 | VF_CONTROLFILE="" 128 | for file in $(ls ../workflow/control/*-* 2>/dev/null || true); do 129 | file_basename=$(basename $file) 130 | jobline_range=${file_basename/.*} 131 | VF_JOBLINE_NO_START=${jobline_range/-*} 132 | VF_JOBLINE_NO_END=${jobline_range/*-} 133 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 134 | export VF_CONTROLFILE="${file}" 135 | break 136 | fi 137 | done 138 | if [ -z "${VF_CONTROLFILE}" ]; then 139 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 140 | fi 141 | 142 | # Checking if the queue should be stopped 143 | line="$(cat ${VF_CONTROLFILE} | grep "stop_after_next_check_interval=")" 144 | stop_after_next_check_interval=${line/"stop_after_next_check_interval="} 145 | if [[ "${stop_after_next_check_interval}" == "true" ]]; then 146 | echo 147 | echo "This job line was stopped by the stop_after_next_check_interval flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 148 | echo 149 | print_job_infos_end 150 | exit 0 151 | fi 152 | 153 | # Checking if there are still ligand collections todo 154 | no_collections_incomplete="0" 155 | i=0 156 | # Using a loop to try several times if there are no ligand collections left - maybe the files where just shortly inaccessible 157 | while [ "${no_collections_incomplete}" == "0" ]; do 158 | no_collections_incomplete="$(cat ../workflow/ligand-collections/todo/todo.all* ../workflow/ligand-collections/todo/${VF_JOBLINE_NO}/*/* ../workflow/ligand-collections/current/${VF_JOBLINE_NO}/*/* 2>/dev/null | grep -c "[^[:blank:]]" || true)" 159 | i="$((i + 1))" 160 | if [ "${i}" == "5" ]; then 161 | break 162 | fi 163 | sleep 1 164 | done 165 | if [[ "${no_collections_incomplete}" = "0" ]]; then 166 | echo 167 | echo "This job line was stopped because there are no ligand collections left." 168 | echo 169 | print_job_infos_end 170 | exit 0 171 | fi 172 | } 173 | 174 | check_queue_end2() { 175 | check_queue_end1 176 | line=$(cat ${VF_CONTROLFILE} | grep "stop_after_job=") 177 | stop_after_job=${line/"stop_after_job="} 178 | if [ "${stop_after_job}" = "true" ]; then 179 | echo 180 | echo "This job line was stopped by the stop_after_job flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 181 | echo 182 | print_job_infos_end 183 | exit 0 184 | fi 185 | } 186 | 187 | # Setting important variables 188 | export VF_NODES_PER_JOB=${PBS_NUM_NODES} 189 | export VF_OLD_JOB_NO=${PBS_JOBNAME:2} 190 | export VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 191 | export VF_QUEUE_NO_1=${VF_OLD_JOB_NO/.*} 192 | export VF_JOBLINE_NO=${VF_QUEUE_NO_1} 193 | export VF_BATCHSYSTEM="PBS" 194 | export VF_SLEEP_TIME_1="1" 195 | export VF_STARTINGTIME=`date` 196 | export VF_START_TIME_SECONDS="$(date +%s)" 197 | export LC_ALL=C 198 | export PATH="./bin:$PATH" # to give bin priority of system commands, useful for obabel sometimes for example 199 | 200 | 201 | # Determining the VF_CONTROLFILE to use for this jobline 202 | VF_CONTROLFILE="" 203 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 204 | file_basename=$(basename $file) 205 | jobline_range=${file_basename/.*} 206 | VF_JOBLINE_NO_START=${jobline_range/-*} 207 | VF_JOBLINE_NO_END=${jobline_range/*-} 208 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 209 | export VF_CONTROLFILE="${file}" 210 | break 211 | fi 212 | done 213 | if [ -z "${VF_CONTROLFILE}" ]; then 214 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 215 | fi 216 | 217 | # Verbosity 218 | VF_VERBOSITY_LOGFILES="$(grep -m 1 "^verbosity_logfiles=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 219 | export VF_VERBOSITY_LOGFILES 220 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 221 | set -x 222 | fi 223 | 224 | # VF_TMPDIR 225 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 226 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 227 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 228 | mkdir -p ${VF_TMPDIR}/${USER} 229 | fi 230 | 231 | # VF_TMPDIR_FAST 232 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 233 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 234 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 235 | mkdir -p ${VF_TMPDIR}/${USER} 236 | fi 237 | 238 | # Setting the job letter 239 | VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 240 | export VF_JOBLETTER 241 | 242 | # Setting the error sensitivity 243 | VF_ERROR_SENSITIVITY="$(grep -m 1 "^error_sensitivity=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 244 | export VF_ERROR_SENSITIVITY 245 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 246 | set -uo pipefail 247 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 248 | fi 249 | 250 | # Setting the error response 251 | VF_ERROR_RESPONSE="$(grep -m 1 "^error_response=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 252 | export VF_ERROR_RESPONSE 253 | 254 | # Checking if queue should be stopped 255 | check_queue_end1 256 | 257 | # Getting the available wallclock time 258 | job_line=$(grep -m 1 "walltime=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 259 | timelimit=${job_line/\#PBS -l walltime=} 260 | export VF_TIMELIMIT_SECONDS="$(echo -n "${timelimit}" | awk -F ':' '{print $3 + $2 * 60 + $1 * 3600}')" 261 | 262 | # Getting the number of queues per step 263 | line=$(cat ${VF_CONTROLFILE} | grep -m 1 "queues_per_step=") 264 | export VF_QUEUES_PER_STEP=${line/"queues_per_step="} 265 | 266 | # Preparing the todo lists for the queues 267 | prepare_queue_todolists="$(grep -m 1 "^prepare_queue_todolists=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 268 | if [ "${prepare_queue_todolists^^}" == "TRUE" ]; then 269 | cd helpers 270 | bash prepare-todolists.sh ${VF_JOBLINE_NO} ${VF_NODES_PER_JOB} ${VF_QUEUES_PER_STEP} 271 | cd .. 272 | elif [ "${prepare_queue_todolists^^}" == "FALSE" ]; then 273 | echo " * Skipping the todo-list preparation as specified in the control-file." 274 | else 275 | echo "Error: The variable prepare_queue_todolists in the control file ${VF_CONTROLFILE} has an unsupported value (${prepare_queue_todolists})." 276 | false 277 | fi 278 | 279 | # Starting the individual steps on different nodes 280 | for VF_STEP_NO in $(seq 1 ${VF_NODES_PER_JOB} ); do 281 | export VF_STEP_NO 282 | echo "Starting job step VF_STEP_NO on host $(hostname)." 283 | bash ../workflow/job-files/sub/one-step.sh & 284 | pids[$(( VF_STEP_NO - 0 ))]=$! 285 | sleep "${VF_SLEEP_TIME_1}" 286 | done 287 | 288 | # Waiting for all steps to finish 289 | 290 | # Checking if all queues exited without error ("wait" waits for all of them, but always returns 0) 291 | exit_code=0 292 | for pid in ${pids[@]}; do 293 | wait $pid || let "exit_code=1" 294 | done 295 | if [ "$exit_code" == "1" ]; then 296 | error_response_std 297 | fi 298 | 299 | 300 | # Creating the next job 301 | ##################################################################################### 302 | echo 303 | echo 304 | echo " *** Preparing the next batch system job *** " 305 | echo "==================================================================================" 306 | echo 307 | 308 | # Checking if the queue should be stopped 309 | check_queue_end2 310 | 311 | # Syncing the new jobfile with the settings in the VF_CONTROLFILE 312 | cd helpers 313 | . sync-jobfile.sh ${VF_JOBLINE_NO} 314 | cd .. 315 | 316 | # Changing the job name 317 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 318 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 319 | sed -i "s/^#PBS -N ${VF_JOBLETTER}-.*/#PBS -N ${VF_JOBLETTER}-${new_job_no}/g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 320 | 321 | # Changing the output filenames 322 | sed -i "s|^#PBS -\([oe]\) .*|#PBS -\1 ../workflow/output-files/jobs/job-${new_job_no}_\${PBS_JOBID}.out|g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 323 | 324 | # Checking how much time has passed since the job has been started 325 | end_time_seconds="$(date +%s)" 326 | time_diff="$((end_time_seconds - VF_START_TIME_SECONDS))" 327 | treshhold=100 328 | if [ "${time_diff}" -le "${treshhold}" ]; then 329 | echo "Since the beginning of the job less than ${treshhold} seconds have passed." 330 | echo "Sleeping for some while to prevent a job submission run..." 331 | sleep 120 332 | fi 333 | 334 | 335 | # Submitting a new new job 336 | cd helpers 337 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 338 | cd .. 339 | 340 | 341 | # Finalizing the job 342 | ##################################################################################### 343 | print_job_infos_end 344 | -------------------------------------------------------------------------------- /tools/templates/template1.slurm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Slurm Settings 21 | ############################################################################### 22 | 23 | #SBATCH --job-name=h-1.1 24 | #SBATCH --mail-user= 25 | #SBATCH --mail-type=fail 26 | #SBATCH --time=00-12:00:00 27 | #SBATCH --mem-per-cpu=800M 28 | #SBATCH --nodes=1 29 | #SBATCH --cpus-per-task=1 30 | #SBATCH --partition=main 31 | #SBATCH --output=../workflow/output-files/jobs/job-1.1_%j.out # File to which standard out will be written 32 | #SBATCH --error=../workflow/output-files/jobs/job-1.1_%j.out # File to which standard err will be written 33 | #SBATCH --signal=10@300 34 | 35 | # Loading Modules 36 | # Odyssey 37 | #module load openbabel/2.4.1-fasrc01 38 | #module load jdk/10.0.1-fasrc01 39 | # O2 40 | #module load java 41 | 42 | # Job Information 43 | ################################################################################## 44 | 45 | echo 46 | echo " *** Job Information *** " 47 | echo "===============================================================" 48 | echo 49 | echo "Environment variables" 50 | echo "------------------------" 51 | env 52 | echo 53 | echo 54 | echo "Job infos by scontrol" 55 | echo "------------------------" 56 | scontrol show job $SLURM_JOB_ID 57 | 58 | # Running the Job - Screening of the Ligands 59 | ###################################################################### 60 | echo 61 | echo " *** Job Output *** " 62 | echo "===========================================================" 63 | echo 64 | 65 | # Functions 66 | # Standard error response 67 | error_response_std() { 68 | 69 | # Printing some informatoin 70 | echo "Error was trapped" 1>&2 71 | echo "Error in bash script $0" 1>&2 72 | echo "Error on line $1" 1>&2 73 | echo "Environment variables" 1>&2 74 | echo "----------------------------------" 1>&2 75 | env 1>&2 76 | 77 | # Checking error response type 78 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]] || [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 79 | 80 | # Printing some information 81 | echo -e "\n * Trying to continue..." 82 | 83 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 84 | 85 | # Printing some information 86 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 87 | print_job_infos_end 88 | 89 | # Exiting 90 | exit 1 91 | fi 92 | } 93 | trap 'error_response_std $LINENO' ERR 94 | 95 | # Handling signals 96 | time_near_limit() { 97 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal. Trying to start a new job." 98 | } 99 | trap 'time_near_limit' 10 100 | 101 | termination_signal() { 102 | echo "The script ${BASH_SOURCE[0]} caught a termination signal. Stopping jobline." 103 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 104 | echo -e "\n Ignoring error. Trying to continue..." 105 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 106 | echo -e "\n Ignoring error. Trying to continue and start next job..." 107 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 108 | echo -e "\n Stopping the jobline." 109 | print_job_infos_end 110 | exit 1 111 | fi 112 | } 113 | trap 'termination_signal' 1 2 3 9 12 15 114 | 115 | # Printing final job information 116 | print_job_infos_end() { 117 | # Job information 118 | echo 119 | echo " *** Final Job Information *** " 120 | echo "======================================================================" 121 | echo 122 | echo "Starting time:" $VF_STARTINGTIME 123 | echo "Ending time: " $(date) 124 | echo 125 | } 126 | 127 | # Checking if the queue should be stopped 128 | check_queue_end1() { 129 | 130 | # Determining the VF_CONTROLFILE to use for this jobline 131 | VF_CONTROLFILE="" 132 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 133 | file_basename=$(basename $file) 134 | jobline_range=${file_basename/.*} 135 | VF_JOBLINE_NO_START=${jobline_range/-*} 136 | VF_JOBLINE_NO_END=${jobline_range/*-} 137 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 138 | export VF_CONTROLFILE="${file}" 139 | break 140 | fi 141 | done 142 | if [ -z "${VF_CONTROLFILE}" ]; then 143 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 144 | fi 145 | 146 | # Checking if the queue should be stopped 147 | line="$(cat ${VF_CONTROLFILE} | grep "stop_after_next_check_interval=")" 148 | stop_after_next_check_interval=${line/"stop_after_next_check_interval="} 149 | if [ "${stop_after_next_check_interval}" = "true" ]; then 150 | echo 151 | echo "This job line was stopped by the stop_after_next_check_interval flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 152 | echo 153 | print_job_infos_end 154 | exit 0 155 | fi 156 | 157 | # Checking if there are still ligand collections todo 158 | no_collections_incomplete="0" 159 | i=0 160 | # Using a loop to try several times if there are no ligand collections left - maybe the files where just shortly inaccessible 161 | while [ "${no_collections_incomplete}" == "0" ]; do 162 | no_collections_incomplete="$(cat ../workflow/ligand-collections/todo/todo.all* ../workflow/ligand-collections/todo/${VF_JOBLINE_NO}/*/* ../workflow/ligand-collections/current/${VF_JOBLINE_NO}/*/* 2>/dev/null | grep -c "[^[:blank:]]" || true)" 163 | i="$((i + 1))" 164 | if [ "${i}" == "5" ]; then 165 | break 166 | fi 167 | sleep 1 168 | done 169 | if [[ "${no_collections_incomplete}" = "0" ]]; then 170 | echo 171 | echo "This job line was stopped because there are no ligand collections left." 172 | echo 173 | print_job_infos_end 174 | exit 0 175 | fi 176 | } 177 | 178 | check_queue_end2() { 179 | check_queue_end1 180 | line=$(cat ${VF_CONTROLFILE} | grep "stop_after_job=") 181 | stop_after_job=${line/"stop_after_job="} 182 | if [ "${stop_after_job}" = "true" ]; then 183 | echo 184 | echo "This job line was stopped by the stop_after_job flag in the VF_CONTROLFILE ${VF_CONTROLFILE}." 185 | echo 186 | print_job_infos_end 187 | exit 0 188 | fi 189 | } 190 | 191 | # Setting important variables 192 | job_line=$(grep -m 1 "job-name=" $0) 193 | jobname=${job_line/"#SBATCH --job-name="} 194 | export VF_OLD_JOB_NO=${jobname:2} 195 | #export VF_OLD_JOB_NO=${SLURM_JOB_NAME/h-} 196 | export VF_VF_OLD_JOB_NO_2=${VF_OLD_JOB_NO/*.} 197 | export VF_QUEUE_NO_1=${VF_OLD_JOB_NO/.*} 198 | export VF_JOBLINE_NO=${VF_QUEUE_NO_1} 199 | export VF_BATCHSYSTEM="SLURM" 200 | export VF_SLEEP_TIME_1="1" 201 | export VF_STARTINGTIME=`date` 202 | export VF_START_TIME_SECONDS="$(date +%s)" 203 | job_line=$(grep -m 1 "nodes=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 204 | export VF_NODES_PER_JOB=${job_line/"#SBATCH --nodes="} 205 | #export VF_NODES_PER_JOB=${SLURM_JOB_NUM_NODES} 206 | export LC_ALL=C 207 | export PATH="./bin:$PATH" # to give bin priority of system commands, useful for obabel sometimes for example 208 | 209 | # Determining the VF_CONTROLFILE to use for this jobline 210 | VF_CONTROLFILE="" 211 | for file in $(ls ../workflow/control/*-* 2>/dev/null|| true); do 212 | file_basename=$(basename $file) 213 | jobline_range=${file_basename/.*} 214 | VF_JOBLINE_NO_START=${jobline_range/-*} 215 | VF_JOBLINE_NO_END=${jobline_range/*-} 216 | if [[ "${VF_JOBLINE_NO_START}" -le "${VF_JOBLINE_NO}" && "${VF_JOBLINE_NO}" -le "${VF_JOBLINE_NO_END}" ]]; then 217 | export VF_CONTROLFILE="${file}" 218 | break 219 | fi 220 | done 221 | if [ -z "${VF_CONTROLFILE}" ]; then 222 | export VF_CONTROLFILE="../workflow/control/all.ctrl" 223 | fi 224 | 225 | # Verbosity 226 | VF_VERBOSITY_LOGFILES="$(grep -m 1 "^verbosity_logfiles=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 227 | export VF_VERBOSITY_LOGFILES 228 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 229 | set -x 230 | fi 231 | 232 | # Setting the error response 233 | VF_ERROR_RESPONSE="$(grep -m 1 "^error_response=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 234 | export VF_ERROR_RESPONSE 235 | 236 | # VF_TMPDIR 237 | export VF_TMPDIR="$(grep -m 1 "^tempdir_default=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 238 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 239 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 240 | mkdir -p ${VF_TMPDIR}/${USER} 241 | fi 242 | 243 | # VF_TMPDIR_FAST 244 | export VF_TMPDIR_FAST="$(grep -m 1 "^tempdir_fast=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 245 | # Creating the ${VF_TMPDIR}/${USER} folder if not present 246 | if [ ! -d "${VF_TMPDIR}/${USER}" ]; then 247 | mkdir -p ${VF_TMPDIR}/${USER} 248 | fi 249 | 250 | # Setting the job letter 251 | VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 252 | export VF_JOBLETTER 253 | 254 | # Setting the error sensitivity 255 | VF_ERROR_SENSITIVITY="$(grep -m 1 "^error_sensitivity=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 256 | export VF_ERROR_SENSITIVITY 257 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 258 | set -uo pipefail 259 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 260 | fi 261 | 262 | # Checking if the queue should be stopped 263 | check_queue_end1 264 | 265 | # Getting the available wallclock time 266 | job_line=$(grep -m 1 "time=" ../workflow/job-files/main/${VF_JOBLINE_NO}.job) 267 | timelimit=${job_line/"#SBATCH --time="} 268 | timelimit=${timelimit//-/:} 269 | export VF_TIMELIMIT_SECONDS="$(echo -n "${timelimit}" | awk -F ':' '{print $4 + $3 * 60 + $2 * 3600 + $1 * 3600 * 24}')" 270 | 271 | # Getting the number of queues per step 272 | line=$(cat ${VF_CONTROLFILE} | grep "queues_per_step=") 273 | export VF_QUEUES_PER_STEP=${line/"queues_per_step="} 274 | 275 | # Preparing the todo lists for the queues 276 | prepare_queue_todolists="$(grep -m 1 "^prepare_queue_todolists=" ${VF_CONTROLFILE} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 277 | if [ "${prepare_queue_todolists^^}" == "TRUE" ]; then 278 | cd helpers 279 | bash prepare-todolists.sh ${VF_JOBLINE_NO} ${VF_NODES_PER_JOB} ${VF_QUEUES_PER_STEP} 280 | cd .. 281 | elif [ "${prepare_queue_todolists^^}" == "FALSE" ]; then 282 | echo " * Skipping the todo-list preparation as specified in the control-file." 283 | else 284 | echo "Error: The variable prepare_queue_todolists in the control file ${VF_CONTROLFILE} has an unsupported value (${prepare_queue_todolists})." 285 | false 286 | fi 287 | 288 | # Starting the individual steps on different nodes 289 | for VF_STEP_NO in $(seq 1 ${VF_NODES_PER_JOB} ); do 290 | export VF_STEP_NO 291 | echo "Starting job step ${VF_STEP_NO} on host $(hostname)." 292 | srun --relative=$((VF_STEP_NO - 1)) -n 1 -N 1 ../workflow/job-files/sub/one-step.sh & 293 | pids[$(( VF_STEP_NO - 0 ))]=$! 294 | sleep "${VF_SLEEP_TIME_1}" 295 | done 296 | 297 | # Waiting for all steps to finish 298 | 299 | # Checking if all queues exited without error ("wait" waits for all of them, but always returns 0) 300 | exit_code=0 301 | for pid in ${pids[@]}; do 302 | wait $pid || let "exit_code=1" 303 | done 304 | if [ "$exit_code" == "1" ]; then 305 | error_response_std $LINENO 306 | fi 307 | 308 | 309 | # Creating the next job 310 | ##################################################################################### 311 | echo 312 | echo 313 | echo " *** Preparing the next batch system job *** " 314 | echo "==================================================================================" 315 | echo 316 | 317 | # Checking if the queue should be stopped 318 | check_queue_end2 319 | 320 | # Syncing the new jobfile with the settings in the VF_CONTROLFILE 321 | cd helpers 322 | . sync-jobfile.sh ${VF_JOBLINE_NO} 323 | cd .. 324 | 325 | # Changing the job name 326 | new_job_no_2=$((VF_VF_OLD_JOB_NO_2 + 1)) 327 | new_job_no="${VF_JOBLINE_NO}.${new_job_no_2}" 328 | sed -i "s/^#SBATCH --job-name=${VF_JOBLETTER}.*/#SBATCH --job-name=${VF_JOBLETTER}-${new_job_no}/g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 329 | 330 | # Changing the output filenames 331 | sed -i "s|^#SBATCH --output=.*|#SBATCH --output=../workflow/output-files/jobs/job-${new_job_no}_%j.out|g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 332 | sed -i "s|^#SBATCH --error=.*|#SBATCH --output=../workflow/output-files/jobs/job-${new_job_no}_%j.err|g" ../workflow/job-files/main/${VF_JOBLINE_NO}.job 333 | 334 | # Checking how much time has passed since the job has been started 335 | end_time_seconds="$(date +%s)" 336 | time_diff="$((end_time_seconds - VF_START_TIME_SECONDS))" 337 | treshhold=120 338 | if [ "${time_diff}" -le "${treshhold}" ]; then 339 | echo "Since the beginning of the job less than ${treshhold} seconds have passed." 340 | echo "Sleeping for some while to prevent a job submission run..." 341 | sleep ${treshhold} 342 | fi 343 | 344 | # Submitting a new new job 345 | cd helpers 346 | . submit.sh ../workflow/job-files/main/${VF_JOBLINE_NO}.job 347 | cd .. 348 | 349 | 350 | # Finalizing the job 351 | ##################################################################################### 352 | print_job_infos_end 353 | -------------------------------------------------------------------------------- /tools/vf_report.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Displaying the banner 21 | echo 22 | echo 23 | . helpers/show_banner.sh 24 | echo 25 | echo 26 | 27 | # Function definitions 28 | # Standard error response 29 | error_response_nonstd() { 30 | echo "Error was trapped which is a nonstandard error." 31 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 32 | echo "Error on line $1" 33 | echo -e "Cleaning up and exiting...\n\n" 34 | exit 1 35 | } 36 | trap 'error_response_nonstd $LINENO' ERR 37 | 38 | # Clean up 39 | clean_up() { 40 | rm -r ${tempdir}/ 2>/dev/null || true 41 | } 42 | trap 'clean_up' EXIT 43 | 44 | # Variables 45 | usage="\nUsage: vf_report.sh [-h] -c category [-v verbosity] 46 | 47 | Options: 48 | -h: Display this help 49 | -c: Possible categories are: 50 | workflow: Shows information about the status of the workflow and the batchsystem. 51 | -v: Specifies the verbosity level of the output. Possible values are 1-2 (default 1) 52 | 53 | " 54 | help_info="The -h option can be used to get more information on how to use this script." 55 | controlfile="../workflow/control/all.ctrl" 56 | collection_folder="$(grep -m 1 "^collection_folder=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 57 | outputfiles_level="$(grep -m 1 "^outputfiles_level=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 58 | export LC_ALL=C 59 | export LANG=C 60 | vf_tempdir="$(grep -m 1 "^tempdir_default=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 61 | export VF_JOBLETTER="$(grep -m 1 "^job_letter=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 62 | batchsystem="$(grep -m 1 "^batchsystem=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 63 | job_letter="$(grep -m 1 "^job_letter=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 64 | 65 | # Tempdir 66 | vf_tempdir="$(grep -m 1 "^tempdir_default=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 67 | tempdir=${vf_tempdir}/$USER/VFLP/${VF_JOBLETTER}/vf_report_$(date | tr " :" "_") 68 | mkdir -p ${tempdir} 69 | 70 | # Verbosity 71 | verbosity="$(grep -m 1 "^verbosity_commands=" ${controlfile} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 72 | if [ "${verbosity}" = "debug" ]; then 73 | set -x 74 | fi 75 | 76 | # Folders 77 | mkdir -p ${tempdir} 78 | 79 | # Treating the input arguments 80 | category_flag="false" 81 | while getopts ':hc:v:' option; do 82 | case "$option" in 83 | h) echo -e "$usage" 84 | exit 0 85 | ;; 86 | c) category=$OPTARG 87 | if ! [[ "${category}" == "workflow" || "${category}" == "vs" ]]; then 88 | echo -e "\nAn unsupported category (${category}) has been specified via the -c option." 89 | echo -e "${help_info}\n" 90 | echo -e "Cleaning up and exiting...\n\n" 91 | exit 1 92 | fi 93 | category_flag=true 94 | ;; 95 | v) verbosity=$OPTARG 96 | if ! [[ "${verbosity}" == [1-2] ]]; then 97 | echo -e "\nAn unsupported verbosity level (${verbosity}) has been specified via the -v option." 98 | echo -e "${help_info}\n" 99 | echo -e "Cleaning up and exiting...\n\n" 100 | exit 1 101 | fi 102 | verbosity_flag=true 103 | ;; 104 | :) printf "\nMissing argument for option -%s\n" "$OPTARG" >&2 105 | echo -e "\n${help_info}\n" 106 | echo -e "Cleaning up and exiting...\n\n" 107 | exit 1 108 | ;; 109 | \?) printf "\nUnrecognized option: -%s\n" "$OPTARG" >&2 110 | echo -e "\n${help_info}\n" 111 | echo -e "Cleaning up and exiting...\n\n" 112 | exit 1 113 | ;; 114 | *) echo "Unimplemented option: -$OPTARG" >&2; 115 | echo -e "\n${help_info}\n" 116 | exit 1 117 | ;; 118 | esac 119 | done 120 | if [ "${category_flag}" == "false" ]; then 121 | echo -e "\nThe mandatory option -c which specifies the category to report on was not specified." 122 | echo -e "${help_info}\n" 123 | echo -e "Cleaning up and exiting...\n\n" 124 | exit 1 125 | fi 126 | if [ "${verbosity_flag}" == "false" ]; then 127 | verbosity=1 128 | fi 129 | 130 | # Checking the category 131 | if [[ "${category}" = "workflow" ]]; then 132 | 133 | # Displaying the information 134 | echo 135 | echo " $(date) " 136 | echo 137 | echo 138 | echo " Workflow Status " 139 | echo "::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" 140 | echo 141 | echo 142 | echo " Joblines " 143 | echo "................................................................................................" 144 | echo 145 | echo " Number of jobfiles in the workflow/jobfiles/main folder: $(ls ../workflow/job-files/main | wc -l)" 146 | if [[ "${batchsystem}" == "SLURM" || "${batchsystem}" == "LSF" ]]; then 147 | echo " Number of joblines in the batch system: $(bin/sqs 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -c "" 2>/dev/null || true)" 148 | fi 149 | if [ "${batchsystem}" = "SLURM" ]; then 150 | queues=$(squeue -l 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | awk '{print $2}' | sort | uniq | tr "\n" " " ) 151 | echo " Number of joblines in the batch system currently running: $(squeue -l 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i "RUNNING" | grep -c "" 2>/dev/null || true)" 152 | for queue in ${queues}; do 153 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently running: $(squeue -l | grep "${queue}.*RUN" | grep "${job_letter}\-" | grep ${USER:0:8} | wc -l)" 154 | done 155 | echo " Number of joblines in the batch system currently not running: $(squeue -l 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i -v "RUNNING" | grep -c "" 2>/dev/null || true)" 156 | for queue in ${queues}; do 157 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently not running: $(squeue -l | grep ${USER:0:8} | grep "${job_letter}\-" | grep "${queue}" | grep -v RUN | grep -v COMPL | wc -l)" 158 | done 159 | elif [[ "${batchsystem}" = "TORQUE" ]] || [[ "${batchsystem}" = "PBS" ]]; then 160 | echo " Number of joblines in the batch system currently running: $(qstat 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i " R " | grep -c "" 2>/dev/null || true)" 161 | echo " Number of joblines in the batch system currently not running: $(qstat 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i -v " R " | grep -c "" 2>/dev/null || true)" 162 | 163 | queues=$(qstat 2>/dev/null | grep "${job_letter}\-" 2>/dev/null | grep "${USER:0:8}" | awk '{print $6}' | sort | uniq | tr "\n" " " ) 164 | echo " Number of joblines in the batch system currently running: $(qstat 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i " R " | grep -c "" 2>/dev/null || true)" 165 | for queue in ${queues}; do 166 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently running: $(qstat | grep ${USER:0:8} | grep "${job_letter}\-" | grep " R .*${queue}" | wc -l)" 167 | done 168 | echo " Number of joblines in the batch system currently not running: $(qstat 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i -v " R " | grep -c "" 2>/dev/null || true)" 169 | for queue in ${queues}; do 170 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently not running: $(qstat | grep ${USER:0:8} | grep "${queue}" | grep "${job_letter}\-" | grep -v " R " | wc -l)" 171 | done 172 | elif [ "${batchsystem}" = "LSF" ]; then 173 | echo " Number of joblines in the batch system currently running: $(bin/sqs 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i "RUN" | grep -c "" 2>/dev/null || true)" 174 | queues=$(bin/sqs 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | awk '{print $4}' | sort | uniq | tr "\n" " " ) 175 | for queue in ${queues}; do 176 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently running: $(bin/sqs | grep "RUN.*${queue}" | grep "${job_letter}\-" | wc -l)" 177 | done 178 | echo " Number of joblines in the batch system currently not running: $(bin/sqs 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i -v "RUN" | grep -c "" 2>/dev/null || true)" 179 | for queue in ${queues}; do 180 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently not running: $(bin/sqs | grep -v "RUN" | grep "${queue}" | grep "${job_letter}\-" | wc -l)" 181 | done 182 | 183 | elif [ "${batchsystem}" = "SGE" ]; then 184 | echo " Number of joblines in the batch system currently running: $(bin/sqs 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i " r " | grep -c "" 2>/dev/null || true)" 185 | queues=$(qconf -sql) 186 | for queue in ${queues}; do 187 | echo ' * Number of joblines in queue "'"${queue}"'"'" currently running: $(bin/sqs | grep " r .*${queue}" | grep "${job_letter}\-" | wc -l)" 188 | done 189 | echo " Number of joblines in the batch system currently not running: $(bin/sqs 2>/dev/null | grep "${job_letter}\-" | grep "${USER:0:8}" | grep -i " qw " | grep -c "" 2>/dev/null || true)" 190 | fi 191 | if [[ "$verbosity" -gt "2" ]]; then 192 | echo " Number of collections which are currently assigned to more than one queue: $(awk -F '.' '{print $1}' ../workflow/ligand-collections/current/*/*/* 2>/dev/null | sort -S 80% | uniq -c | grep " [2-9] " | grep -c "" 2>/dev/null || true)" 193 | fi 194 | if [[ "${batchsystem}" == "LSF" || "${batchsystem}" == "SLURM" || "{batchsystem}" == "SGE" ]]; then 195 | if [[ "${batchsystem}" == "SLURM" ]]; then 196 | squeue -o "%.18i %.9P %.8j %.8u %.8T %.10M %.9l %.6D %R %C" | grep RUN | grep "${USER:0:8}" | grep "${job_letter}\-" | awk '{print $10}' > ${tempdir}/report.tmp 197 | elif [[ "${batchsystem}" == "LSF" ]]; then 198 | bin/sqs | grep RUN | grep "${USER:0:8}" | grep "${job_letter}\-" | awk -F " *" '{print $6}' > ${tempdir}/report.tmp 199 | elif [[ "${batchsystem}" == "SGE" ]]; then 200 | bin/sqs | grep " r " | grep "${USER:0:8}" | grep "${job_letter}\-" | awk '{print $7}' > ${tempdir}/report.tmp 201 | fi 202 | sumCores='0' 203 | while IFS='' read -r line || [[ -n "${line}" ]]; do 204 | if [ "${line:0:1}" -eq "${line:0:1}" ] 2>/dev/null ; then 205 | coreNumber=$(echo $line | awk -F '*' '{print $1}') 206 | else 207 | coreNumber=1 208 | fi 209 | sumCores=$((sumCores + coreNumber)) 210 | done < ${tempdir}/report.tmp 211 | echo " Number of cores/slots currently used by the workflow: ${sumCores}" 212 | rm ${tempdir}/report.tmp || true 213 | fi 214 | 215 | echo 216 | echo 217 | echo " Collections " 218 | echo "................................................................................................" 219 | echo 220 | echo " Total number of ligand collections: $(grep -c "" ../workflow/ligand-collections/var/todo.original 2>/dev/null || true )" 221 | 222 | ligand_collections_completed=0 223 | for folder1 in $(find ../workflow/ligand-collections/done/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 224 | for folder2 in $(find ../workflow/ligand-collections/done/$folder1/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 225 | ligand_collections_completed_toadd="$(grep -ch "" ../workflow/ligand-collections/done/$folder1/$folder2/* 2>/dev/null | paste -sd+ 2>/dev/null | bc )" 226 | if [[ -z "${ligand_collections_completed_toadd// }" ]]; then 227 | ligand_collections_completed_toadd=0 228 | fi 229 | ligand_collections_completed=$((ligand_collections_completed + ligand_collections_completed_toadd)) 230 | done 231 | done 232 | echo " Number of ligand collections completed: ${ligand_collections_completed}" 233 | 234 | ligand_collections_processing=0 235 | for folder1 in $(find ../workflow/ligand-collections/current/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 236 | for folder2 in $(find ../workflow/ligand-collections/current/$folder1/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 237 | ligand_collections_processing_toadd=$(grep -ch "" ../workflow/ligand-collections/current/$folder1/$folder2/* 2>/dev/null | paste -sd+ 2>/dev/null | bc ) 238 | if [[ -z "${ligand_collections_processing_toadd// }" ]]; then 239 | ligand_collections_processing_toadd=0 240 | fi 241 | ligand_collections_processing=$((ligand_collections_processing + ligand_collections_processing_toadd)) 242 | done 243 | done 244 | echo " Number of ligand collections in state \"processing\": ${ligand_collections_processing}" 245 | 246 | ligand_collections_todo=0 247 | for folder1 in $(find ../workflow/ligand-collections/todo/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 248 | for folder2 in $(find ../workflow/ligand-collections/todo/$folder1/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 249 | ligand_collections_todo_toadd=$(grep -ch "" ../workflow/ligand-collections/todo/$folder1/$folder2/* 2>/dev/null | paste -sd+ 2>/dev/null | bc ) 250 | if [[ -z "${ligand_collections_todo_toadd// }" ]]; then 251 | ligand_collections_todo_toadd=0 252 | fi 253 | ligand_collections_todo=$((ligand_collections_todo + ligand_collections_todo_toadd)) 254 | done 255 | done 256 | echo " Number of ligand collections not yet started: ${ligand_collections_todo}" 257 | echo 258 | echo 259 | 260 | echo " Ligands (in completed collections) " 261 | echo "................................................................................................" 262 | echo 263 | 264 | ligands_total=0 265 | if [ -s ../workflow/ligand-collections/var/todo.original ]; then 266 | ligands_total="$(awk '{print $2}' ../workflow/ligand-collections/var/todo.original | paste -sd+ | bc -l 2>/dev/null || true)" 267 | if [[ -z "${ligands_total// }" ]]; then 268 | ligands_total=0 269 | fi 270 | fi 271 | echo " Total number of ligands: ${ligands_total}" 272 | 273 | ligands_started=0 274 | for folder1 in $(find ../workflow/ligand-collections/done/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 275 | for folder2 in $(find ../workflow/ligand-collections/done/$folder1/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 276 | ligands_started_to_add="$(grep -ho "Ligands-started:[0-9]\+" ../workflow/ligand-collections/done/$folder1/$folder2/* 2>/dev/null | awk -F ':' '{print $2}' | sed "/^$/d" | paste -sd+ | bc -l 2>/dev/null || true)" 277 | if [[ -z "${ligands_started_to_add// }" ]]; then 278 | ligands_started_to_add=0 279 | fi 280 | ligands_started=$((ligands_started + ligands_started_to_add)) 281 | done 282 | done 283 | echo " Number of ligands started: ${ligands_started}" 284 | 285 | ligands_success=0 286 | for folder1 in $(find ../workflow/ligand-collections/done/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 287 | for folder2 in $(find ../workflow/ligand-collections/done/$folder1/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 288 | ligands_success_to_add="$(grep -ho "Ligands-succeeded:[0-9]\+" ../workflow/ligand-collections/done/$folder1/$folder2/* 2>/dev/null | awk -F ':' '{print $2}' | sed "/^$/d" | paste -sd+ | bc -l 2>/dev/null || true)" 289 | if [[ -z "${ligands_success_to_add// }" ]]; then 290 | ligands_success_to_add=0 291 | fi 292 | ligands_success=$((ligands_success + ligands_success_to_add)) 293 | done 294 | done 295 | echo " Number of ligands successfully completed: ${ligands_success}" 296 | 297 | ligands_failed=0 298 | for folder1 in $(find ../workflow/ligand-collections/done/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 299 | for folder2 in $(find ../workflow/ligand-collections/done/$folder1/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n"); do 300 | ligands_failed_to_add="$(grep -ho "Ligands-failed:[0-9]\+" ../workflow/ligand-collections/done/$folder1/$folder2/* 2>/dev/null | awk -F ':' '{print $2}' | sed "/^$/d" | paste -sd+ | bc -l 2>/dev/null || true)" 301 | if [[ -z "${ligands_failed_to_add// }" ]]; then 302 | ligands_failed_to_add=0 303 | fi 304 | ligands_failed=$((ligands_failed + ligands_failed_to_add)) 305 | done 306 | done 307 | echo " Number of ligands failed: ${ligands_failed}" 308 | 309 | echo 310 | echo 311 | fi -------------------------------------------------------------------------------- /tools/templates/one-step.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # Functions 21 | # Standard error response 22 | error_response_std() { 23 | 24 | # Printint some information 25 | echo "Error was trapped" 1>&2 26 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" 1>&2 27 | echo "Error on line $1" 1>&2 28 | echo "Environment variables" 1>&2 29 | echo "----------------------------------" 1>&2 30 | env 1>&2 31 | 32 | # Copying again the queue output files back to the shared filesystem (was done already in the one-queue.sh file, but during job abortions it can fail due to a shortage of time) 33 | for i in $(seq 1 ${VF_QUEUES_PER_STEP}); do 34 | VF_QUEUE_NO_3="${i}" 35 | VF_QUEUE_NO="${VF_QUEUE_NO_12}-${VF_QUEUE_NO_3}" 36 | cp ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.* ../workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 2>/dev/null || true 37 | done 38 | 39 | # Checking error response 40 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 41 | 42 | # Printing some information 43 | echo -e "\n * Ignoring error. Trying to continue..." 44 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 45 | 46 | # Printing some information 47 | echo -e "\n * Trying to stop this queue without causing the jobline to fail..." 48 | 49 | # Exiting 50 | exit 0 51 | 52 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 53 | 54 | # Printing some information 55 | echo -e "\n * Trying to stop this queue and causing the jobline to fail..." 56 | 57 | # Exiting 58 | exit 1 59 | fi 60 | } 61 | trap 'error_response_std $LINENO' ERR 62 | 63 | time_near_limit() { 64 | echo "The script one-step.sh caught a time limit signal." 65 | echo "Sending this signal to all the queues started by this step." 66 | kill -s 10 ${pids[*]} || true 67 | wait 68 | } 69 | trap 'time_near_limit' 10 70 | 71 | another_signal() { 72 | echo "The script one-step.sh caught a terminating signal." 73 | echo "Sending terminating signal to all the queues started by this step." 74 | kill -s 1 ${pids[*]} || true 75 | wait 76 | } 77 | trap 'time_near_limit' 1 2 3 9 15 78 | 79 | 80 | clean_up() { 81 | 82 | 83 | # Copying again the queue output files back to the shared filesystem (was done already in the one-queue.sh file, but during job abortions it can fail due to a shortage of time) 84 | for i in $(seq 1 ${VF_QUEUES_PER_STEP}); do 85 | VF_QUEUE_NO_3="${i}" 86 | VF_QUEUE_NO="${VF_QUEUE_NO_12}-${VF_QUEUE_NO_3}" 87 | cp ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.* ../workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ || true 88 | done 89 | } 90 | trap 'clean_up' EXIT 91 | 92 | # Sourcing bashrc 93 | source ~/.bashrc || true 94 | 95 | prepare_queue_files_tmp() { 96 | 97 | # Creating the required folders 98 | if [ -d "${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/" ]; then 99 | rm -r ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/ 100 | fi 101 | mkdir -p ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 102 | 103 | # Copying the requires files 104 | if ls -1 ../workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.* > /dev/null 2>&1; then 105 | cp ../workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.* ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 106 | fi 107 | } 108 | 109 | # Verbosity 110 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 111 | set -x 112 | fi 113 | 114 | start_ng_server() { 115 | 116 | # Starting the ng-server 117 | java -Xmx${java_max_heap_size}G com.martiansoftware.nailgun.NGServer localhost:${NG_PORT} & 118 | sleep 10 # Loading the ng server takes a few seconds 119 | } 120 | 121 | ng_server_check() { 122 | 123 | # Testing if the ng-server is still working 124 | ng --dun-server localhost --nailgun-port ${NG_PORT} com.martiansoftware.nailgun.examples.Exit 0 125 | exit_code=$? 126 | 127 | # Checking the exit status 128 | if [[ ${exit_code} != 0 && ${exit_code} != 130 ]]; then 129 | 130 | # Printing warning message 131 | echo " * Warning: The NG Server seems to be not running anymore. Trying to restart..." 132 | 133 | # Starting the ng-server 134 | start_ng_server 135 | 136 | # Testing if the ng-server is still working 137 | ng --nailgun-server localhost --nailgun-port ${NG_PORT} com.martiansoftware.nailgun.examples.Exit 0 138 | exit_code=$? 139 | 140 | # Checking the exit status 141 | if [[ ${exit_code} != 0 && ${exit_code} != 130 ]]; then 142 | 143 | # Testing if it is running again already - e.g. because another queue already restarted it 144 | sleep 3 145 | ng --dun-server localhost --nailgun-port ${NG_PORT} com.martiansoftware.nailgun.examples.Exit 0 146 | exit_code=$? 147 | 148 | # Checking the exit status 149 | if [[ ${exit_code} != 0 && ${exit_code} != 130 ]]; then 150 | 151 | # Printing warning message 152 | echo " * Error: Restarting of the NG Server has failed... " 153 | error_response_std $LINENO 154 | else 155 | 156 | # Printing message 157 | echo " * The NG Server is already running again... " 158 | fi 159 | 160 | # Printing message 161 | echo " * The NG Server was successfully restarted... " 162 | fi 163 | fi 164 | } 165 | 166 | # Preparing the temporary controlfile 167 | export VF_CONTROLFILE_TEMP=${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/controlfile 168 | mkdir -p ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/ 169 | cp ${VF_CONTROLFILE} ${VF_CONTROLFILE_TEMP} 170 | 171 | # Setting and exporting variables 172 | export VF_QUEUE_NO_2=${VF_STEP_NO} 173 | export VF_QUEUE_NO_12="${VF_QUEUE_NO_1}-${VF_QUEUE_NO_2}" 174 | export VF_LITTLE_TIME="false"; 175 | export VF_START_TIME_SECONDS 176 | export VF_TIMELIMIT_SECONDS 177 | pids="" 178 | protonation_state_generation="$(grep -m 1 "^protonation_state_generation=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 179 | protonation_program_1="$(grep -m 1 "^protonation_program_1=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 180 | protonation_program_2="$(grep -m 1 "^protonation_program_2=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 181 | conformation_generation="$(grep -m 1 "^conformation_generation=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 182 | conformation_program_1="$(grep -m 1 "^conformation_program_1=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 183 | conformation_program_2="$(grep -m 1 "^conformation_program_2=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 184 | store_queue_log_files="$(grep -m 1 "^store_queue_log_files=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 185 | 186 | # Creating required folders 187 | mkdir -p ../workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 188 | mkdir -p ../workflow/ligand-collections/todo/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 189 | mkdir -p ../workflow/ligand-collections/current/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 190 | mkdir -p ../workflow/ligand-collections/done/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/ 191 | 192 | # Preparing the JChem and associated packages if JChem is used 193 | if [[ ( "${protonation_state_generation}" == "true" && ( "${protonation_program_1}" == "cxcalc" || "${protonation_program_2}" == "cxcalc" )) || ( "${conformation_generation}" == "true" && ( "${conformation_program_1}" == "molconvert" || "${conformation_program_2}" == "molconvert" )) ]]; then 194 | 195 | # Preparing the JChem Package 196 | chemaxon_license_filename="$(grep -m 1 "^chemaxon_license_filename=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 197 | jchem_package_filename="$(grep -m 1 "^jchem_package_filename=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 198 | mkdir -p ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/ 199 | tar -C ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/ -xvzf packages/${jchem_package_filename} 200 | #chmod u+x ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/jchemsuite/bin/* 201 | #export PATH="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/jchemsuite/bin:$PATH" 202 | 203 | # Copying the ChemAxon license file if needed 204 | if [[ "${chemaxon_license_filename}" == "none" ]] || [[ -z "${chemaxon_license_filename}" ]]; then 205 | 206 | # Error 207 | echo " Error: The ChemAxon license file was not specified, but is required..." 208 | 209 | # Causing an error 210 | error_response_std $LINENO 211 | 212 | elif [[ ! -f "packages/${chemaxon_license_filename}" ]]; then 213 | 214 | # Error 215 | echo " Error: The specified ChemAxon license file was not found..." 216 | 217 | # Causing an error 218 | error_response_std $LINENO 219 | 220 | else 221 | # Copying the ChemAxon licence file 222 | cp packages/${chemaxon_license_filename} ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/license.cxl 223 | 224 | # Adjusting the CHEMAXON_LICENSE_URL environment variable 225 | export CHEMAXON_LICENSE_URL="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/license.cxl" 226 | 227 | # Removing the Chemaxon folder if existent which ChemAxon's software automatically creates to store the license file, because if it exists it will not use the variable CHEMAXON_LICENSE_URL 228 | if [ -d ${HOME}/.chemaxon ]; then 229 | rm -r ${HOME}/.chemaxon || true 230 | mkdir -p ${HOME}/.chemaxon 231 | chmod 000 ${HOME}/.chemaxon 232 | fi 233 | fi 234 | 235 | # Preparing the JVM 236 | java_package_filename="$(grep -m 1 "^java_package_filename=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 237 | 238 | if [[ ${java_package_filename} != "none" ]]; then 239 | if [[ -f packages/${java_package_filename} ]]; then 240 | tar -C ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/ -xvzf packages/${java_package_filename} 241 | 242 | # Checking the folder structure 243 | if [ ! -d ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/java/ ]; then 244 | 245 | # Error 246 | echo " Error: The root folder of the java package does not seem to have the name \'java\', which is required..." 247 | 248 | # Causing an error 249 | error_response_std $LINENO 250 | fi 251 | 252 | export JAVA_HOME="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/java" 253 | chmod u+x ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/java/bin/* 254 | export PATH="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/java/bin:$PATH" 255 | else 256 | 257 | # Error 258 | echo " Error: The Java package file (java_package_filename=${java_package_filename}) which was specified in the relevant controlfile (${VF_CONTROLFILE_TEMP}) was was not found..." 259 | 260 | # Causing an error 261 | error_response_std $LINENO 262 | fi 263 | fi 264 | 265 | # Preparing ng 266 | java_max_heap_size="$(grep -m 1 "^java_max_heap_size=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 267 | ng_package_filename="$(grep -m 1 "^ng_package_filename=" ${VF_CONTROLFILE_TEMP} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 268 | tar -C ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/ -xvzf packages/${ng_package_filename} 269 | export CLASSPATH="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/nailgun/nailgun-server/target/classes:${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/nailgun/nailgun-examples/target/classes:${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/chemaxon/jchemsuite/lib/*" 270 | chmod u+x ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/nailgun/nailgun-client/target/ng 271 | export PATH="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/nailgun/nailgun-client/target/:$PATH" 272 | # Getting the first free port (Source: https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port ) 273 | read lowerport upperport < /proc/sys/net/ipv4/ip_local_port_range 274 | while : 275 | do 276 | test_port="$(shuf -i $lowerport-$upperport -n 1)" 277 | bin/ss -lpn | grep -q ":${test_port} " || break 278 | done 279 | export NG_PORT=${test_port} 280 | # Starting the ng server 281 | # Java 10/11 debugging options: -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError="echo %p" -XX:OnError="echo %p" -Xlog:gc*:file=${PWD}/java.gc.${VF_QUEUE_NO_12}.log -Xlog:all=warning:file=${PWD}/java.warning.${VF_QUEUE_NO_12}.log 282 | # Java 8 debugging options: -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${PWD}/java.gc.${VF_QUEUE_NO_12}.log -XX:-PrintConcurrentLocks -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${PWD}/java.gc.${VF_QUEUE_NO_12}.log 283 | start_ng_server 284 | 285 | # Preparing the other binary files 286 | mkdir -p ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/bin 287 | cp bin/* ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/bin/ 288 | chmod u+x ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/bin/* 289 | export PATH="${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/packages/bin/:$PATH" 290 | fi 291 | 292 | # Starting the individual queues 293 | for i in $(seq 1 ${VF_QUEUES_PER_STEP}); do 294 | export VF_QUEUE_NO_3="${i}" 295 | export VF_QUEUE_NO="${VF_QUEUE_NO_12}-${VF_QUEUE_NO_3}" 296 | prepare_queue_files_tmp 297 | echo "Job step ${VF_STEP_NO} is starting queue ${VF_QUEUE_NO} on host $(hostname)." 298 | if [ ${store_queue_log_files} == "all_uncompressed" ]; then 299 | source ../workflow/job-files/sub/one-queue.sh >> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.all 2>&1 & 300 | elif [ ${store_queue_log_files} == "all_compressed" ]; then 301 | source ../workflow/job-files/sub/one-queue.sh 2>&1 | gzip >> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.all.gz & 302 | elif [ ${store_queue_log_files} == "only_error_uncompressed" ]; then 303 | source ../workflow/job-files/sub/one-queue.sh 2> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.err & 304 | elif [ ${store_queue_log_files} == "only_error_compressed" ]; then 305 | source ../workflow/job-files/sub/one-queue.sh 2> >(gzip >> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.err.gz) & 306 | elif [ ${store_queue_log_files} == "std_compressed_error_uncompressed" ]; then 307 | source ../workflow/job-files/sub/one-queue.sh 1> >(gzip >> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.std.gz) 2>> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.err & 308 | elif [ ${store_queue_log_files} == "all_compressed_error_uncompressed" ]; then 309 | source ../workflow/job-files/sub/one-queue.sh 1> >(gzip >> ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.all.gz) 2> >(tee ${VF_TMPDIR}/${USER}/VFLP/${VF_JOBLETTER}/${VF_QUEUE_NO_12}/${VF_QUEUE_NO}/workflow/output-files/queues/${VF_QUEUE_NO_1}/${VF_QUEUE_NO_2}/queue-${VF_QUEUE_NO}.out.err) & 310 | elif [ ${store_queue_log_files} == "none" ]; then 311 | source ../workflow/job-files/sub/one-queue.sh 2>&1 >/dev/null & 312 | else 313 | echo "Error: The variable store_log_file in the control file ${VF_CONTROLFILE_TEMP} has an unsupported value (${store_queue_log_files})." 314 | false 315 | fi 316 | pids[$(( i - 1 ))]=$! 317 | done 318 | 319 | # Checking if all queues exited without error ("wait" waits for all of them, but always returns 0) 320 | exit_code=0 321 | for pid in "${pids[@]}"; do 322 | wait $pid || let "exit_code=1" 323 | done 324 | if [ "$exit_code" == "1" ]; then 325 | error_response_std $LINENO 326 | fi 327 | 328 | # Cleaning up 329 | exit 0 330 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /tools/helpers/prepare-todolists.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright (C) 2019 Christoph Gorgulla 4 | # 5 | # This file is part of VirtualFlow. 6 | # 7 | # VirtualFlow is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # VirtualFlow is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with VirtualFlow. If not, see . 19 | 20 | # ---------------------------- 21 | # 22 | # Usage: . prepare-todolists.sh jobline_no steps_per_job queues_per_step [quiet] 23 | # 24 | # Description: prepares the todolists for the queues. The tasks are taken from the central todo list ../../workflow/ligand-collections/todo/todo.all 25 | # 26 | # Option: quiet (optional) 27 | # Possible values: 28 | # quiet: No information is displayed on the screen. 29 | # 30 | # --------------------------------------------------------------------------- 31 | 32 | # Idea: improve second backup mecha (copying back) 33 | 34 | # Displaying help if the first argument is -h 35 | usage="Usage: . prepare-todolists.sh jobline_no steps_per_job queues_per_step [quiet]" 36 | if [ "${1}" = "-h" ]; then 37 | echo "${usage}" 38 | return 39 | fi 40 | 41 | # Setting the error sensitivity 42 | if [[ "${VF_ERROR_SENSITIVITY}" == "high" ]]; then 43 | set -uo pipefail 44 | trap '' PIPE # SIGPIPE = exit code 141, means broken pipe. Happens often, e.g. if head is listening and got all the lines it needs. 45 | fi 46 | 47 | # Variables 48 | queue_no_1="${1}" 49 | steps_per_job="${2}" 50 | queues_per_step="${3}" 51 | export LC_ALL=C 52 | todo_file_temp=${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.all 53 | 54 | # Verbosity (the script is only called by the job scripts) 55 | if [ "${VF_VERBOSITY_LOGFILES}" = "debug" ]; then 56 | set -x 57 | fi 58 | 59 | # Printing some information 60 | echo -e "\n * Preparing the to-do lists for jobline ${queue_no_1}\n" 61 | 62 | # Standard error response 63 | error_response_std() { 64 | echo "Error has been trapped." | tee -a /dev/stderr 65 | echo "Error in bash script $(basename ${BASH_SOURCE[0]})" | tee -a /dev/stderr 66 | echo "Error on line $1" | tee -a /dev/stderr 67 | 68 | #clean_up 69 | if [[ "${VF_ERROR_RESPONSE}" == "ignore" ]]; then 70 | echo -e "\n * Ignoring error. Trying to continue..." 71 | elif [[ "${VF_ERROR_RESPONSE}" == "next_job" ]]; then 72 | echo -e "\n * Trying to stop this job and to start a new job..." 73 | kill -9 ${touch_locked_pid} &>/dev/null || true 74 | exit 0 exit 0 75 | elif [[ "${VF_ERROR_RESPONSE}" == "fail" ]]; then 76 | echo -e "\n * Stopping this jobline." 77 | kill -9 ${touch_locked_pid} &>/dev/null || true 78 | exit 1 79 | else 80 | echo -e "\n * Stopping this jobline." 81 | kill -9 ${touch_locked_pid} &>/dev/null || true 82 | exit 1 83 | fi 84 | } 85 | # Trapping only after we got hold of the to-do.all file (the wait command seems to fail when catching USR1, and thus causes the general error response rather than a time_near_limit response) 86 | 87 | # Handling signals 88 | time_near_limit() { 89 | echo "The script ${BASH_SOURCE[0]} caught a time limit signal." 90 | # clean_up 91 | exit 0 92 | } 93 | trap 'time_near_limit' 10 94 | 95 | termination_signal() { 96 | echo "The script ${BASH_SOURCE[0]} caught a termination signal." 97 | # clean_up 98 | exit 1 99 | } 100 | trap 'termination_signal' 1 2 3 9 15 101 | 102 | next_todo_list1() { 103 | 104 | # Checking if current_todo_list_index is a number 105 | if [ ${current_todo_list_index} -eq ${current_todo_list_index} ]; then 106 | 107 | # Variables 108 | next_todo_list_index=$(printf "%04d" $((10#${current_todo_list_index}+1)) ) 109 | next_todo_list=../../workflow/ligand-collections/todo/todo.all.${next_todo_list_index} 110 | 111 | # Checking if another todo file exists 112 | if [ -f ${next_todo_list} ]; then 113 | 114 | # Printing information 115 | echo "The next todo list will be used (todo.all.${next_todo_list_index})" 116 | 117 | # Changing the symlink 118 | rm ../../workflow/ligand-collections/todo/todo.all.locked || true 119 | ln -s todo.all.${next_todo_list_index} ../../workflow/ligand-collections/todo/todo.all.locked 120 | 121 | # Copying the new todo list to temp 122 | cp ${next_todo_list} ${todo_file_temp} 123 | 124 | # Emptying the old todo list 125 | echo -n "" > ../../workflow/ligand-collections/todo/todo.all.${current_todo_list_index} 126 | 127 | # Changing variables 128 | current_todo_list_index=${next_todo_list_index} 129 | current_todo_list=${next_todo_list} 130 | no_collections_remaining="$(grep -cv '^\s*$' ${todo_file_temp} || true)" 131 | #no_collections_remaining="$(cat ${todo_file_temp} 2>/dev/null | grep -c "[^[:blank:]]" || true)" 132 | no_collections_assigned=0 133 | no_collections_beginning=${no_collections_remaining} 134 | initial_todolist=false 135 | else 136 | next_todo_list_index=$(printf "%04d" $((10#${current_todo_list_index})) ) 137 | next_todo_list=../../workflow/ligand-collections/todo/todo.all.${next_todo_list_index} 138 | no_collections_remaining="0" 139 | rm ../../workflow/ligand-collections/todo/todo.all.locked || true 140 | echo " * Info: No more todo lists." 141 | fi 142 | else 143 | echo " * Warning: current_todo_list_index is not a number. Trying to compensate..." 144 | fi 145 | } 146 | 147 | next_todo_list2() { 148 | 149 | # Changing the locked file 150 | if [[ -f ../../workflow/ligand-collections/todo/todo.all.locked ]] && [[ ${initial_todolist} == "true" ]]; then 151 | 152 | echo " * Warning: There exists an old (locked) todo file. Trying to take care of it..." 153 | if [[ ! -L ../../workflow/ligand-collections/todo/todo.all.locked ]] && [[ -s ../../workflow/ligand-collections/todo/todo.all.locked ]]; then 154 | echo " * Warning: The old todo file is not a symlink and not a empty, trying to preserve it..." 155 | mv ../../workflow/ligand-collections/todo/todo.all.locked ${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.all.old 156 | else 157 | echo " * Warning: The old todo file is a symlink or an empty file. Removing it..." 158 | rm ../../workflow/ligand-collections/todo/todo.all.locked 159 | fi 160 | fi 161 | 162 | # Determining the next todo list 163 | next_todo_list=$(wc -l ../../workflow/ligand-collections/todo/todo.all.[0-9]* | grep -v total | grep -v " 0 " | head -n 1 | awk '{print $2}') 164 | next_todo_list_index=${next_todo_list/*.} 165 | if [ -n ${next_todo_list_index} ]; then 166 | 167 | ln -s todo.all.${next_todo_list_index} ../../workflow/ligand-collections/todo/todo.all.locked 168 | 169 | # Printing information 170 | echo "The next todo list will be used (todo.all.${next_todo_list_index})" 171 | 172 | # Copying the new todo list to temp 173 | cp ${next_todo_list} ${todo_file_temp} 174 | 175 | # Emptying the old todo list 176 | echo -n "" > ../../workflow/ligand-collections/todo/todo.all.${current_todo_list_index} 177 | 178 | # Adding the old list contents if present 179 | if [ -f ${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.all.old ]; then 180 | cat ${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.all.old > ${todo_file_temp} 181 | sort -u ${todo_file_temp} > ${todo_file_temp}.tmp # In case that the old todo file was part of the new one 182 | mv ${todo_file_temp}.tmp ${todo_file_temp} 183 | rm ${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.all.old 184 | fi 185 | 186 | # Changing variables 187 | current_todo_list_index=${next_todo_list_index} 188 | current_todo_list=${next_todo_list} 189 | no_collections_remaining="$(grep -cv '^\s*$' ${todo_file_temp} || true)" 190 | no_collections_assigned=0 191 | no_collections_beginning=${no_collections_remaining} 192 | initial_todolist=false 193 | fi 194 | } 195 | 196 | 197 | # Clean up when exiting 198 | clean_up() { 199 | 200 | # # Moving the to-do.all file to its original place 201 | # other_todofile_exists="false" 202 | # if [ -f ../../workflow/ligand-collections/todo/todo.all ]; then 203 | # echo "Warning: The file ../../workflow/ligand-collections/todo/todo.all already exists." 204 | # no_of_lines_1=$(fgrep -c "" ../../workflow/ligand-collections/todo/todo.all) 205 | # no_of_lines_2=$(fgrep -c "" "${todo_file_temp}") 206 | # other_todofile_exists="true" 207 | # other_todofile_is_larger="false" 208 | # if [ "${no_of_lines_1}" -ge "${no_of_lines_2}" ]; then 209 | # echo "The number of lines in the found todo file is larger than in our one. Discarding our version." 210 | # other_todofile_is_larger="true" 211 | # else 212 | # echo "The number of lines in the found todo file is smaller than in our one. Using our version." 213 | # fi 214 | # fi 215 | # 216 | # # Checking if our to-do file has size zero and the locked one is very large 217 | # copy_flag="true" 218 | # #if [[ ! -s ${todo_file_temp} ]] && [[ -f ../../workflow/ligand-collections/todo/todo.all.locked ]]; then 219 | # # no_of_lines_1=$(fgrep -c "" ../../workflow/ligand-collections/todo/todo.all.locked 2>/dev/null || true) 220 | # # if [[ "${no_of_lines_1}" -ge "1000" ]]; then 221 | # # copy_flag="false" 222 | # # fi 223 | # #fi 224 | # 225 | # if [[ "${other_todofile_exists}" == "false" ]] || [[ "${other_todofile_exists}" == "true" && "${other_todofile_is_larger}" == "false" ]]; then 226 | # if [[ -f "${todo_file_temp}" && "${copy_flag}" == "true" ]]; then 227 | # mv ${todo_file_temp} ../../workflow/ligand-collections/todo/ 228 | # echo -e "\nThe file ${todo_file_temp} has been moved back to the original folder (../../workflow/ligand-collections/todo/).\n" 229 | # rm ../../workflow/ligand-collections/todo/todo.all.locked || true 230 | # 231 | # elif [[ -f ../../workflow/ligand-collections/todo/todo.all.locked ]]; then 232 | # mv ../../workflow/ligand-collections/todo/todo.all.locked ../../workflow/ligand-collections/todo/todo.all 233 | # echo -e "The file ../../workflow/ligand-collections/todo/todo.all.locked has been moved back to ../../workflow/ligand-collections/todo/" 234 | # 235 | # else 236 | # echo -e "\nThe file ${todo_file_temp} could not be moved back to the original folder (../../workflow/ligand-collections/todo/)." 237 | # echo -e "Also the file ../../workflow/ligand-collections/todo/todo.all.locked could not be moved back to ../../workflow/ligand-collections/todo/" 238 | # fi 239 | # fi 240 | cp ${todo_file_temp} ../../workflow/ligand-collections/todo/todo.all.locked 241 | mv ../../workflow/ligand-collections/todo/todo.all.locked ../../workflow/ligand-collections/todo/todo.all 242 | rm -r ${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/ || true 243 | kill ${touch_locked_pid} &>/dev/null || true 244 | } 245 | trap 'clean_up' EXIT 246 | 247 | 248 | # Creating the working directory 249 | mkdir -p ${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/ 250 | 251 | # Copying the control to temp 252 | vf_controlfile_temp=${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/controlfile 253 | cp ../${VF_CONTROLFILE} ${vf_controlfile_temp} 254 | 255 | # Variables 256 | collection_folder="$(grep -m 1 "^collection_folder=" ${vf_controlfile_temp} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 257 | collection_folder=${collection_folder%/} 258 | ligands_todo_per_queue="$(grep -m 1 "^ligands_todo_per_queue=" ${vf_controlfile_temp} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 259 | ligands_per_refilling_step="$(grep -m 1 "^ligands_per_refilling_step=" ${vf_controlfile_temp} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 260 | initial_todolist=true 261 | 262 | # Screen formatting output 263 | if [[ ! "$*" = *"quiet"* ]]; then 264 | echo 265 | fi 266 | 267 | # Getting the number of ligands which are already in the local to-do lists 268 | ligands_todo="" 269 | queue_collection_numbers="" 270 | todofile_queue_old_temp="${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.queue.old" 271 | for queue_no_2 in $(seq 1 ${steps_per_job}); do 272 | # Loop for each queue of the node 273 | for queue_no_3 in $(seq 1 ${queues_per_step}); do 274 | 275 | # Variables 276 | queue_no="${queue_no_1}-${queue_no_2}-${queue_no_3}" 277 | ligands_todo[${queue_no_2}0000${queue_no_3}]=0 278 | queue_collection_numbers[${queue_no_2}0000${queue_no_3}]=0 279 | 280 | # Creating a temporary to-do file with the new ligand collections 281 | todofile_queue_new_temp[${queue_no_2}0000${queue_no_3}]="${VF_TMPDIR_FAST}/${USER}/VFLP/${VF_JOBLETTER}/${VF_JOBLINE_NO}/prepare-todolists/todo.queue.new.${queue_no}" 282 | 283 | # Maybe to test: Checking if it works (job run on test). Read the entire list into memory as bash array. 10K package size during refilling. Test the new ligand-list mechanism during breaks. 284 | 285 | # Checking the number of ligands in the queue todo lists 286 | if [ -s "../../workflow/ligand-collections/todo/${queue_no_1}/${queue_no_2}/${queue_no}" ]; then 287 | cp ../../workflow/ligand-collections/todo/${queue_no_1}/${queue_no_2}/${queue_no} ${todofile_queue_old_temp} 288 | queue_collection_numbers[${queue_no_2}0000${queue_no_3}]=$(grep -c "" ${todofile_queue_old_temp}) 289 | ligands_to_add=$(awk '{print $2}' ${todofile_queue_old_temp} | paste -sd+ | bc -l) 290 | if [ ! ${ligands_to_add} -eq ${ligands_to_add} ]; then 291 | ligands_to_add=0 292 | fi 293 | ligands_todo[${queue_no_2}0000${queue_no_3}]=${ligands_to_add} 294 | fi 295 | 296 | # Checking the number of ligands in the current ligand collection 297 | if [ -s "../../workflow/ligand-collections/current/${queue_no_1}/${queue_no_2}/${queue_no}" ]; then 298 | cp ../../workflow/ligand-collections/current/${queue_no_1}/${queue_no_2}/${queue_no} ${todofile_queue_old_temp} 299 | ligands_to_add=$(awk '{print $2}' ${todofile_queue_old_temp}) 300 | if [ ! ${ligands_to_add} -eq ${ligands_to_add} ]; then 301 | ligands_to_add=0 302 | fi 303 | ligands_todo[${queue_no_2}0000${queue_no_3}]=$((ligands_todo[${queue_no_2}0000${queue_no_3}] + ${ligands_to_add} )) 304 | queue_collection_numbers[${queue_no_2}0000${queue_no_3}]=$((queue_collection_numbers[${queue_no_2}0000${queue_no_3}] + 1 )) 305 | fi 306 | done 307 | done 308 | 309 | # Printing some infos about the to-do lists of this queue before the refilling 310 | if [[ ! "$*" = *"quiet"* ]]; then 311 | echo "Starting the (re)filling of the todolists of the queues." 312 | echo 313 | for queue_no_2 in $(seq 1 ${steps_per_job}); do 314 | # Loop for each queue of the node 315 | for queue_no_3 in $(seq 1 ${queues_per_step}); do 316 | queue_no="${queue_no_1}-${queue_no_2}-${queue_no_3}" 317 | echo "Before (re)filling the todolists the queue ${queue_no} had ${ligands_todo[${queue_no_2}0000${queue_no_3}]} ligands todo distributed in ${queue_collection_numbers[${queue_no_2}0000${queue_no_3}]} collections." 318 | done 319 | done 320 | echo 321 | fi 322 | 323 | 324 | # Hiding the to-do.all list 325 | status="false"; 326 | k="1" 327 | max_iter=1000 328 | modification_time_difference=0 329 | start_time_waiting="$(date +%s)" 330 | dispersion_time_min="$(grep -m 1 "^dispersion_time_min=" ${vf_controlfile_temp} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 331 | dispersion_time_max="$(grep -m 1 "^dispersion_time_max=" ${vf_controlfile_temp} | tr -d '[[:space:]]' | awk -F '[=#]' '{print $2}')" 332 | modification_time_treshhold=$(shuf -i ${dispersion_time_min}-${dispersion_time_max} -n1) 333 | modification_time_treshhold_lockedfile="300" # 5 minute 334 | 335 | # Loop for hiding the todo.all file 336 | while [[ "${status}" = "false" ]]; do 337 | modification_time=$(stat -c %Z ../../workflow/ligand-collections/todo/todo.all 2>/dev/null || true) 338 | if [ "${modification_time}" -eq "${modification_time}" 2>/dev/null ]; then 339 | modification_time_difference="$(($(date +%s) - modification_time))" 340 | else 341 | modification_time_difference=0 342 | fi 343 | if [ "${modification_time_difference}" -ge "${modification_time_treshhold}" 2>/dev/null ]; then 344 | if mv ../../workflow/ligand-collections/todo/todo.all ../../workflow/ligand-collections/todo/todo.all.locked 2>/dev/null; then 345 | cp ../../workflow/ligand-collections/todo/todo.all.locked ${todo_file_temp} 346 | current_todo_list_index="$(realpath ../../workflow/ligand-collections/todo/todo.all.locked | xargs basename | xargs basename | awk -F '.' '{print $3}')" 347 | if ! [ "${current_todo_list_index}" -eq "${current_todo_list_index}" ]; then 348 | echo " * Warning: The current todo file is not a symlink. Trying to compensate..." 349 | next_todo_list2 350 | fi 351 | cp ${todo_file_temp} ../../workflow/ligand-collections/var/todo.all.${current_todo_list_index}.bak.${queue_no_1} 352 | status="true" 353 | trap 'error_response_std $LINENO' ERR 354 | 355 | watch -m -n 1 touch ../../workflow/ligand-collections/var/todo.all.locked &>/dev/null & 356 | touch_locked_pid=#! 357 | else 358 | sleep 1."$(shuf -i 0-9 -n1)" 359 | fi 360 | else 361 | 362 | echo "The ligand-collections/todo/todo.all (if existent) did not meet the requirements for continuation (trial ${k})." 363 | sleep "$(shuf -i 10-30 -n1).$(shuf -i 0-9 -n1)" 364 | if [ -f ../../workflow/ligand-collections/todo/todo.all.locked ]; then 365 | # Checking the locked file 366 | modification_time=$(stat -c %Z ../../workflow/ligand-collections/todo/todo.all.locked 2>/dev/null || true) 367 | if [ "${modification_time}" -eq "${modification_time}" 2>/dev/null ]; then 368 | modification_time_difference="$(($(date +%s) - modification_time))" 369 | else 370 | modification_time_difference=0 371 | fi 372 | if [ "${modification_time_difference}" -ge "${modification_time_treshhold_lockedfile}" ]; then 373 | echo " * The file ../../workflow/ligand-collections/todo/todo.all.locked does exist, and probably it was abandoned because the locked file is quite old (${modification_time_difference} seconds)." 374 | echo " * Adopting the locked file to this jobline." 375 | next_todo_list2 376 | status="true" 377 | trap 'error_response_std $LINENO' ERR 378 | elif [ "${k}" = "${max_iter}" ]; then 379 | echo "Reached iteration ${max_iter}. Also the file ../../workflow/ligand-collections/todo/todo.all.locked does not exit." 380 | echo "This seems to be hopeless. Stopping the refilling process." 381 | error_response_std 382 | fi 383 | fi 384 | k=$((k+1)) 385 | fi 386 | done 387 | end_time_waiting="$(date +%s)" 388 | 389 | # Checking if there are tasks left in the to-do file 390 | no_collections_incomplete="$(cat ${todo_file_temp} 2>/dev/null | grep -c "[^[:blank:]]" || true)" 391 | if [[ "${no_collections_incomplete}" = "0" ]]; then 392 | 393 | # Checking if there is one more todo list 394 | next_todo_list1 395 | no_collections_incomplete="$(cat ${todo_file_temp} 2>/dev/null | grep -c "[^[:blank:]]" || true)" 396 | 397 | # Checking if no more collections 398 | if [[ "${no_collections_incomplete}" = "0" ]]; then 399 | 400 | # Using the alternative method 401 | next_todo_list2 402 | no_collections_incomplete="$(cat ${todo_file_temp} 2>/dev/null | grep -c "[^[:blank:]]" || true)" 403 | 404 | # If no more new todo list, quitting 405 | if [[ "${no_collections_incomplete}" = "0" ]]; then 406 | echo "There is no more ligand collection in the todo.all file. Stopping the refilling procedure." 407 | exit 0 408 | fi 409 | fi 410 | fi 411 | 412 | # Removing empty lines 413 | grep '[^[:blank:]]' < ${todo_file_temp} > ${todo_file_temp}.tmp || true 414 | mv ${todo_file_temp}.tmp ${todo_file_temp} 415 | 416 | # Loop for each refilling step 417 | no_of_refilling_steps="$((${ligands_todo_per_queue} / ${ligands_per_refilling_step}))" 418 | no_collections_remaining="$(grep -cv '^\s*$' ${todo_file_temp} || true)" 419 | no_collections_assigned=0 420 | no_collections_beginning=${no_collections_remaining} 421 | start_time_seconds="$(date +%s)" 422 | for refill_step in $(seq 1 ${no_of_refilling_steps}); do 423 | step_limit=$((${refill_step} * ${ligands_per_refilling_step})) 424 | # Loop for each node 425 | for queue_no_2 in $(seq 1 ${steps_per_job}); do 426 | # Loop for each queue of the node 427 | for queue_no_3 in $(seq 1 ${queues_per_step}); do 428 | queue_no="${queue_no_1}-${queue_no_2}-${queue_no_3}" 429 | 430 | while [ "${ligands_todo[${queue_no_2}0000${queue_no_3}]}" -lt "${step_limit}" ]; do 431 | 432 | # Checking if there is one more ligand collection to be done 433 | if [ "${no_collections_remaining}" -eq "0" ]; then 434 | 435 | # Checking if there is one more todo list 436 | next_todo_list1 437 | 438 | # Checking if no more collections 439 | if [[ "${no_collections_remaining}" = "0" ]]; then 440 | 441 | # Using the alternative method 442 | next_todo_list2 443 | 444 | # If no more new collections, quitting 445 | if [[ "${no_collections_remaining}" = "0" ]]; then 446 | echo "There is no more ligand collection in the todo.all file. Stopping the refilling procedure." 447 | break 4 448 | fi 449 | fi 450 | fi 451 | 452 | # Setting some variables 453 | next_ligand_collection_and_length="$(head -n 1 ${todo_file_temp})" 454 | next_ligand_collection=${next_ligand_collection_and_length// *} 455 | 456 | # Checking for the collection name. Very few times the current todo_list contains the content "Binary file (standard input) matches", and nothing else. In this case, we just go to the next todolist. 457 | if [ "${next_ligand_collection}" = "Binary" ]; then 458 | 459 | # Clearing the faulty file (so that the other queues don't stumple over it as well in case this queue fails to prepare the next todolist) 460 | echo -n "" > ../../workflow/ligand-collections/todo/todo.all 461 | 462 | # Checking if there is one more todo list 463 | next_todo_list1 464 | 465 | # Checking if no more collections 466 | if [[ "${no_collections_remaining}" = "0" ]]; then 467 | 468 | # Using the alternative method 469 | next_todo_list2 470 | 471 | # If no more new collections, quitting 472 | if [[ "${no_collections_remaining}" = "0" ]]; then 473 | echo "There is no more ligand collection in the todo.all file. Stopping the refilling procedure." 474 | break 4 475 | fi 476 | fi 477 | fi 478 | 479 | no_to_add=${next_ligand_collection_and_length//* } 480 | if ! [ "${no_to_add}" -eq "${no_to_add}" ]; then 481 | sleep 1 482 | next_ligand_collection_and_length="$(head -n 1 ${todo_file_temp})" 483 | no_to_add=${next_ligand_collection_and_length//* } 484 | if ! [ "${no_to_add}" -eq "${no_to_add}" ]; then 485 | echo " * Warning: Could not get the length of collection ${next_ligand_collection}. Found value is: ${no_to_add}. Exiting." 486 | exit 1 487 | fi 488 | fi 489 | echo "${next_ligand_collection_and_length}" >> ${todofile_queue_new_temp[${queue_no_2}0000${queue_no_3}]} 490 | ligands_todo[${queue_no_2}0000${queue_no_3}]=$(( ${ligands_todo[${queue_no_2}0000${queue_no_3}]} + ${no_to_add} )) 491 | queue_collection_numbers[${queue_no_2}0000${queue_no_3}]=$((queue_collection_numbers[${queue_no_2}0000${queue_no_3}] + 1 )) 492 | # Removing the new collection from the ligand-collections-to-do file 493 | tail -n +2 ${todo_file_temp} > ${todo_file_temp}.tmp || true 494 | mv ${todo_file_temp}.tmp ${todo_file_temp} 495 | # Updating the variable no_collections_remaining 496 | no_collections_remaining=$((no_collections_remaining-1)) 497 | no_collections_assigned=$((no_collections_assigned+1)) 498 | done 499 | done 500 | done 501 | done 502 | 503 | # Adding the new collections from the temporary to-do file to the permanent one of the queue 504 | for queue_no_2 in $(seq 1 ${steps_per_job}); do 505 | for queue_no_3 in $(seq 1 ${queues_per_step}); do 506 | queue_no="${queue_no_1}-${queue_no_2}-${queue_no_3}" 507 | mkdir -p ../../workflow/ligand-collections/todo/${queue_no_1}/${queue_no_2}/ 508 | if [ -f ${todofile_queue_new_temp[${queue_no_2}0000${queue_no_3}]} ]; then 509 | cat ${todofile_queue_new_temp[${queue_no_2}0000${queue_no_3}]} >> ../../workflow/ligand-collections/todo/${queue_no_1}/${queue_no_2}/${queue_no} || true 510 | rm ${todofile_queue_new_temp[${queue_no_2}0000${queue_no_3}]} || true 511 | fi 512 | done 513 | done 514 | 515 | # Printing some infos about the to-do lists of this queue after the refilling 516 | if [[ ! "$*" = *"quiet"* ]]; then 517 | for queue_no_2 in $(seq 1 ${steps_per_job}); do 518 | # Loop for each queue of the node 519 | for queue_no_3 in $(seq 1 ${queues_per_step}); do 520 | queue_no="${queue_no_1}-${queue_no_2}-${queue_no_3}" 521 | echo "After (re)filling the todolists the queue ${queue_no} has ${ligands_todo[${queue_no_2}0000${queue_no_3}]} ligands todo distributed in ${queue_collection_numbers[${queue_no_2}0000${queue_no_3}]} collections." 522 | done 523 | done 524 | fi 525 | 526 | # Displaying some information 527 | if [[ ! "$*" = *"quiet"* ]]; then 528 | end_time_seconds="$(date +%s)" 529 | echo 530 | echo "The todo lists for the queues were (re)filled in $((end_time_seconds-start_time_seconds)) second(s) (waiting time not included)." 531 | echo "The waiting time was $((end_time_waiting-start_time_waiting)) second(s)." 532 | echo 533 | fi 534 | --------------------------------------------------------------------------------