├── README.asc ├── app ├── mpiwrap ├── simulate ├── simulate.py ├── simulate.sh ├── stats ├── stats.py └── stats.sh ├── bin ├── cleanup ├── hosts ├── package_tutorial.sh └── plot.sh ├── doc ├── README ├── TODO ├── activeplot.png ├── amazon_ec2.asc ├── amazon_ec2.html ├── asciidoc.css ├── build_docs.sh ├── cumulativeplot.png ├── figs │ ├── modis.dia │ └── modis.png ├── images │ └── icons │ │ ├── caution.png │ │ ├── example.png │ │ ├── home.png │ │ ├── important.png │ │ ├── next.png │ │ ├── note.png │ │ ├── prev.png │ │ ├── tip.png │ │ ├── up.png │ │ └── warning.png ├── part01.png ├── part02.png ├── part03.png ├── part04.png ├── part05.png ├── part06.png ├── push.sh ├── swift-config.html ├── swift.conf.asc └── tutorial.html ├── ec2.asc ├── local.conf ├── mandelbrot ├── README ├── bin │ ├── Makefile │ ├── assemble │ ├── driver.c │ ├── makefile.edison │ ├── mandelbrot.c │ ├── mandelbrot.h │ └── run_mandelbrot ├── cleanup.sh ├── ensemble_mandel.swift ├── setup.sh └── swift.conf ├── mpi ├── Makefile ├── buildcray.sh ├── do_mpi.swift ├── mock.sh ├── mpi_hello.c ├── pi.c ├── setup.sh ├── submit.sbatch ├── submit_scripts │ └── pbs.submit └── swift.conf ├── nersc.conf ├── part01 ├── p1.swift └── swift.conf ├── part02 ├── p2.swift └── swift.conf ├── part03 ├── p3.swift └── swift.conf ├── part04 ├── p4.swift ├── swift.conf ├── unsorted.txt └── xsede.conf ├── part05 ├── p5.swift ├── simulate.sh ├── stats.sh ├── swift.conf └── xsede.conf ├── part06 ├── p6.swift ├── simulate.sh ├── stats.sh ├── swift.conf └── xsede.conf ├── part07 ├── Makefile ├── mpi_hello.c ├── nersc.conf ├── p7.swift └── swift.conf ├── part08 ├── Makefile ├── hipi.c ├── hipiwrap ├── nersc.conf ├── p8.swift ├── swift.conf └── xsede.conf ├── setup.sh ├── swift.conf ├── test_all.sh └── xsede.conf /README.asc: -------------------------------------------------------------------------------- 1 | doc/README -------------------------------------------------------------------------------- /app/mpiwrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is a wrapper that launches the mpi binary using the appropriate mpi launcher. 4 | # On edison/cori the launch will be done using srun 5 | 6 | MPI_RANKS=$1 7 | MPI_APP_PATH=$2 8 | shift 2 9 | 10 | if which srun &> /dev/null ; then 11 | 12 | echo "Executing with srun :" 13 | srun -n $MPI_RANKS $MPI_APP_PATH $* 14 | 15 | elif which mpirun &> /dev/null ; then 16 | 17 | echo "Executing with mpirun :" 18 | mpirun -np $MPI_RANKS $MPI_APP_PATH $* 19 | 20 | fi 21 | 22 | 23 | -------------------------------------------------------------------------------- /app/simulate: -------------------------------------------------------------------------------- 1 | simulate.sh -------------------------------------------------------------------------------- /app/simulate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | import socket 5 | import time 6 | import math 7 | import random 8 | import getpass 9 | 10 | def parse(): 11 | from optparse import OptionParser 12 | parser=OptionParser() 13 | parser.add_option("-b", "--bias", type="int", dest="bias", default=0, 14 | help="offset bias: add this integer to all results"); 15 | 16 | parser.add_option("-B", "--biasfile", type="string", dest="biasfile", 17 | default="none", help="file of integer biases to add to results" ); 18 | 19 | parser.add_option("-l", "--log", type="string", dest="log", default="yes", 20 | help="generate a log in stderr if not nul"); 21 | 22 | parser.add_option("-n", "--nvalues", type="int", dest="nvalues",default=1, 23 | help="print this many values per simulation" ); 24 | 25 | parser.add_option("-s", "--seed", type="int", dest="initseed", default=0, 26 | help="use this integer [0..32767] as a seed"); 27 | 28 | parser.add_option("-S", "--seedfile", type="string", dest="seedfile", 29 | default="none", help="use this file (containing integer seeds [0..32767]) one per line" ); 30 | 31 | parser.add_option("-t", "--timesteps", type="int", dest="timesteps", 32 | default=0, help='number of simulated "timesteps" in seconds (determines runtime)' ); 33 | 34 | parser.add_option("-r", "--range", type="int", dest="range", default=100, 35 | help="range (limit) of generated results" ); 36 | 37 | parser.add_option("-w", "--width", type="int", dest="width", default=8, 38 | help="Width ?" ); 39 | 40 | parser.add_option("-x", "--scale", type="int", dest="scale", default=1, 41 | help="scale the results by this integer" ); 42 | # Not implemented yet 43 | parser.add_option("-p", "--paramfile", type="string", dest="paramfile", default="none", 44 | help="Not implemented yet" ); 45 | return (parser.parse_args()); 46 | 47 | def log(): 48 | import datetime 49 | print >> sys.stderr, "Called as: ", str(sys.argv) 50 | print >> sys.stderr, "Start time: ", datetime.datetime.now() 51 | print >> sys.stderr, "Running on node: ", socket.gethostname() 52 | print >> sys.stderr, "Running as user: ", getpass.getuser() 53 | print >> sys.stderr, "\nEnvironment:\n\n" 54 | print >> sys.stderr, os.environ 55 | 56 | def printparams(options): 57 | print 'bias =',options.bias 58 | print 'biasfile = ',options.biasfile 59 | print 'log = ', options.log 60 | print 'nvalues = ',options.nvalues 61 | print 'seed = ', options.initseed 62 | print 'seedfile = ', options.seedfile 63 | print 'timesteps = ', options.timesteps 64 | print 'range = ', options.range 65 | print 'width = ', options.width 66 | print 'scale = ', options.scale 67 | print 'paramfile = ', options.paramfile 68 | 69 | def simulate(options): 70 | time.sleep(options.timesteps) 71 | bias=[]; 72 | if (options.biasfile != "none"): 73 | try: 74 | with open(options.biasfile) as biasfile: 75 | lines = biasfile.read().splitlines() 76 | for line in lines: 77 | bias.append(int(line)) 78 | except IOError: 79 | print "Error accessing content from file: ", options.biasfile 80 | bias_count = len(bias) 81 | 82 | for i in range(options.nvalues): 83 | value = (random.random() + 84 | random.random()*math.pow(2,16) + 85 | random.random()*math.pow(2,32) + 86 | random.random()*math.pow(2,48)) 87 | value=( (int(value)%options.range) * options.scale + options.bias) 88 | if ( i < bias_count ): 89 | value = value + bias[i] 90 | elif ( bias_count > 0 ): 91 | value = value + bias[bias_count-1] 92 | 93 | print '{num:{fill}{width}}'.format(num=value, fill=' ', width=options.width) 94 | 95 | 96 | def seed(options): 97 | if (options.initseed != 0 ): 98 | random.seed(options.initseed) 99 | if (options.seedfile != "none"): 100 | try: 101 | with open(options.seedfile) as seedfile: 102 | lines=seedfile.read().splitlines() 103 | seed = 0 104 | for line in lines: 105 | seed = seed + int(line) 106 | random.seed(seed) 107 | except IOError: 108 | print "Could not open file: ", options.seedfile 109 | 110 | 111 | if __name__ == "__main__": 112 | (options, args) = parse() 113 | printparams(options) 114 | seed(options) 115 | simulate(options) 116 | log() 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | -------------------------------------------------------------------------------- /app/simulate.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | sleep 1 4 | printparams() 5 | { 6 | printf "\nSimulation parameters:\n\n" 7 | echo bias=$bias 8 | echo biasfile=$biasfile 9 | echo initseed=$initseed 10 | echo log=$log 11 | echo paramfile=$paramfile 12 | echo range=$range 13 | echo scale=$scale 14 | echo seedfile=$seedfile 15 | echo timesteps=$timesteps 16 | echo output width=$width 17 | } 18 | 19 | log() { 20 | printf "\nCalled as: $0: $cmdargs\n\n" 21 | printf "Start time: "; /bin/date 22 | printf "Running as user: "; /usr/bin/id 23 | printf "Running on node: "; /bin/hostname 24 | printf "Node IP address: "; /bin/hostname -I 25 | printparams 26 | printf "\nEnvironment:\n\n" 27 | printenv | sort 28 | } 29 | 30 | addsims() { 31 | while read f1 ; do 32 | read -u 3 f2 33 | if [ _$f1 = _ ]; then f1=$lastf1; fi 34 | if [ _$f2 = _ ]; then f2=$lastf2; fi 35 | printf "%${width}d\n" $(($f1+$f2)) 36 | lastf1=$f1 37 | lastf2=$f2 38 | done <$1 3<$2 39 | } 40 | 41 | # set defaults 42 | 43 | bias=0 44 | biasfile=none 45 | initseed=none 46 | log=yes 47 | paramfile=none 48 | range=100 49 | scale=1 50 | seedfile=none 51 | timesteps=0 52 | nvalues=1 53 | width=8 54 | cmdargs="$*" 55 | 56 | usage() 57 | { 58 | echo $0: usage: 59 | cat <$simout 122 | 123 | # process file of biases 124 | 125 | if [ $biasfile != none ]; then 126 | addsims $simout $biasfile 127 | else 128 | cat $simout 129 | fi 130 | rm $simout 131 | 132 | # log environmental data 133 | 134 | if [ $log != off ]; then 135 | log 1>&2 136 | fi 137 | -------------------------------------------------------------------------------- /app/stats: -------------------------------------------------------------------------------- 1 | stats.sh -------------------------------------------------------------------------------- /app/stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import socket 3 | import sys 4 | import os 5 | 6 | def log(): 7 | import datetime 8 | print >> sys.stderr, "Called as: ", str(sys.argv) 9 | print >> sys.stderr, "Start time: ", datetime.datetime.now() 10 | print >> sys.stderr, "Running on node: ", socket.gethostname() 11 | print >> sys.stderr, "Running as user: ",os.getlogin() 12 | print >> sys.stderr, "\nEnvironment:\n\n" 13 | print >> sys.stderr, os.environ 14 | 15 | def stat(): 16 | total = 0 17 | count = 0 18 | for f in sys.argv[1:]: 19 | try: 20 | with open(f) as ifile: 21 | lines = ifile.read().splitlines() 22 | for line in lines: 23 | total += int(line) 24 | count += 1 25 | except IOError: 26 | print "Error accessing content from file: ", options.biasfile 27 | print total/count 28 | 29 | if __name__ == "__main__": 30 | log() 31 | stat() 32 | -------------------------------------------------------------------------------- /app/stats.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | log() { 4 | printf "\nCalled as: $0: $cmdargs\n\n" 5 | printf "Start time: "; /bin/date 6 | printf "Running as user: "; /usr/bin/id 7 | printf "Running on node: "; /bin/hostname 8 | printf "Node IP address: "; /bin/hostname -I 9 | printf "\nEnvironment:\n\n" 10 | printenv | sort 11 | } 12 | 13 | awk ' 14 | 15 | { sum += $1} 16 | 17 | END { printf("%d\n",sum/NR) } 18 | ' $* 19 | log 1>&2 20 | -------------------------------------------------------------------------------- /bin/cleanup: -------------------------------------------------------------------------------- 1 | rm -rf *.log *.rlog *.d p?-*-*-* *.kml *.swiftx *.out output outdir logs hi.* _concurrent .swift/tmp run??? sorted.txt 2 | 3 | -------------------------------------------------------------------------------- /bin/hosts: -------------------------------------------------------------------------------- 1 | for h in $(cat hosts.txt); do ssh $h hostname -f; done 2 | -------------------------------------------------------------------------------- /bin/package_tutorial.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | BIN=$(cd $(dirname $0); pwd) 4 | BASE=$(cd $BIN/..; pwd) 5 | LOC=$(cd $BASE/..; pwd) 6 | 7 | echo executing $0 from BIN=$BIN 8 | echo making tutorial package of BASE=$BASE 9 | echo placing tutorial package in LOC=$LOC 10 | 11 | ( 12 | cd $LOC 13 | tar zcf swift-cray-tutorial.tgz --exclude-vcs swift-cray-tutorial 14 | scp swift-cray-tutorial.tgz p01532@raven.cray.com: 15 | ) 16 | -------------------------------------------------------------------------------- /bin/plot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #usage: ./plotswiftlogs.ketan 4 | 5 | SWIFTLOGFILE=$1 6 | 7 | #TMPDIR=`mktemp -d plotlog.XXX` 8 | 9 | grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out 10 | 11 | SWIFTOUTFILE=swiftoutfile.out 12 | 13 | #extract start time 14 | TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` 15 | START_TIME=`date +%s -d "$TMPDATE"` 16 | 17 | #extract end time 18 | TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` 19 | END_TIME=`date +%s -d "$TMPDATE"` 20 | 21 | #duration 22 | DIFFTIME=$((END_TIME - START_TIME)) 23 | 24 | #extract active runs in a file 25 | (grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt) 26 | 27 | #extract successful completions in a file 28 | (grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) 29 | 30 | #prepare tics 31 | activelines=`wc -l active.txt | awk '{print $1}'` 32 | cumulines=`wc -l cumulative.txt | awk '{print $1}'` 33 | 34 | if [ $activelines -ne 0 ] 35 | then 36 | activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` 37 | fi 38 | 39 | seq 0 $activelinespertic $DIFFTIME > activetics.txt 40 | 41 | if [ $cumulines -ne 0 ] 42 | then 43 | cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` 44 | fi 45 | 46 | seq 0 $cumulinespertic $DIFFTIME > cumultics.txt 47 | 48 | #final plot data 49 | paste activetics.txt active.txt > plot_active.txt 50 | paste cumultics.txt cumulative.txt > plot_cumulative.txt 51 | 52 | cat << EOF1 > plotit.gp 53 | set terminal png enhanced 54 | set nokey 55 | set output "cumulativeplot.png" 56 | set xlabel "Time in sec" 57 | set ylabel "number of completed apps" 58 | set title "Cumulative Apps Completed" 59 | plot "plot_cumulative.txt" using 1:2 with lines 60 | set output "activeplot.png" 61 | set xlabel "Time in sec" 62 | set ylabel "number of active apps" 63 | set title "Active Apps" 64 | plot "plot_active.txt" using 1:2 with line 65 | EOF1 66 | 67 | gnuplot plotit.gp 2>/dev/null 68 | rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt 69 | -------------------------------------------------------------------------------- /doc/TODO: -------------------------------------------------------------------------------- 1 | PLAN 2 | 3 | basics 4 | cloud 5 | uc3 6 | multisystems 7 | (modis) 8 | 9 | get hosts.txt for cloud dynamically 10 | 11 | Show variety of UC3 activity and what ran where 12 | 13 | get hostname and app_resources into provider staging (hostname into all _swiftwrap) 14 | 15 | Show gridftp and go connect to process the modis dataset 16 | 17 | Show SwiftR for R users, SwiftPy for Enthought PY? 18 | 19 | fetching apps vs sending apps vs parrot/cvmfs/oasis etc ???? 20 | 21 | Demo script-aps, bin apps, building apps, caching apps. etc. 22 | 23 | Do a Swift BLAST workflow -- with viz !!! (spice it up with help from Dina) 24 | 25 | OSG page from Mats: "Is your job HTC-ready" 26 | 27 | Use new Trunk config in tutorial? 28 | 29 | 30 | Add viz and status display; add node probing to tutorial. 31 | 32 | Debug failure with pinfiles 33 | 34 | 35 | What "exercises to suggest?" 36 | 37 | - run stats app locally 38 | - installed vs transferred apps 39 | - transferred programs vs transferred scripts 40 | - summarize things about the env from the .log files, ala stats. 41 | 42 | Y-style dag: 2 sims and average 43 | 44 | Show how to find temp files at runtime... 45 | 46 | Show how to find hung processes at runtime... (or get other trace data...) 47 | 48 | make your owb versions of the programs; verify that you can substitute them 49 | 50 | Add a layer to isolate shell version v installed version 51 | 52 | Show how to handle a paramfile 53 | 54 | Test if OSGCOnnect works and account string (PorjectName) works OK for both! 55 | 56 | SHow how to handle long lists of files (eg writeData and other swift.properties) 57 | 58 | Expand arith to arb prec using bc 59 | 60 | 61 | ISSUES 62 | 63 | error in pin files? debug and turn back on 64 | PATH issues for various providers 65 | 66 | bin dir? 67 | 68 | setup.csh and .csh testing 69 | 70 | IMPROVEMENTS 71 | 72 | 1. Cleanup.sh mentioned in the tutorial 73 | [fixed] 74 | 75 | 2. cd ../part02 76 | [fixed] 77 | 78 | 3. support for zsh, csh 79 | [pending] 80 | 81 | 4. What results are created and where did the outputs go ? 82 | and pointers to what the log files mean 83 | [pending] 84 | 85 | 5. Part4, 86 | - cd ../part04 is missing. Cleanup the README 87 | [fixed] 88 | 89 | 6. Part5 90 | - cd ../part05 is missing. Cleanup 91 | [fixed] 92 | 93 | 7. Part7 94 | -Change from running on Tukey analysis cluster compute nodes 95 | [fixed] 96 | 97 | 8. Part6 98 | command line args for nsim and steps, tell user the different behavior 99 | that can be expected with different arg options 100 | [pending] 101 | 102 | 9. Draw workflow for the modis demos 103 | [pending] 104 | 105 | 10. Link to more technical swift-lang references 106 | [pending] 107 | 108 | 11. Links to detailed language constructs, when explaining 109 | apps, link to the section on apps in the userguide. 110 | [pending] 111 | 112 | 113 | 114 | --- 115 | 116 | From Yadu: 117 | 118 | I've committed all the changes that I made on Friday and you can get them 119 | from -> https://svn.ci.uchicago.edu/svn/vdl2/SwiftTutorials/CIC_2013-08-09 120 | 121 | I'm trying to make this tutorial easily work on several sites so that, we 122 | do not have to tweak things for every single tutorial. Once this is done, 123 | I can also add it to the test-battery as well. I think this has value 124 | considering that we spend atleast 1-2 days for every demo. 125 | 126 | Now, the major pending item is the Modis demo, specifically the issue of 127 | bringing data to any site and scripts. Slight tweaks to allow for sending 128 | the scripts as args to apps would also be needed. 129 | 130 | So the major changes are: 131 | 1. Source the setup.sh script with the target site name 132 | eg. source setups.sh 133 | 2. No separate cloud folder 134 | 3. [pending] fix modis data and scripts 135 | 4. [pending] support for zsh and other shells? 136 | 137 | --- 138 | -------------------------------------------------------------------------------- /doc/activeplot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/activeplot.png -------------------------------------------------------------------------------- /doc/amazon_ec2.asc: -------------------------------------------------------------------------------- 1 | Setting up Amazon EC2 2 | --------------------- 3 | 4 | Amazon Elastic Compute Cloud (Amazon EC2) is a web service that provides resizable compute capacity in the cloud. 5 | Getting cloud resources on-demand from Amazon is now as simple as setting up an account and attaching a credit card. 6 | 7 | Sign up online 8 | ~~~~~~~~~~~~~~ 9 | 10 | Sign up for Amazon Web Services (AWS) and create an IAM user by following instructions 11 | from http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/get-set-up-for-amazon-ec2.html[Amazon's Setup Documentation]. 12 | From the documentation you would only need to do the following two steps: 13 | 14 | 1. http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/get-set-up-for-amazon-ec2.html#sign-up-for-aws[Sign up for AWS] 15 | 16 | 2. http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/get-set-up-for-amazon-ec2.html#create-an-iam-user[Create and IAM User] 17 | 18 | NOTE: To create access keys, you must have permissions to perform the required 19 | IAM actions. For more information, see http://docs.aws.amazon.com/IAM/latest/UserGuide/ManagingCredentials.html[Managing Credentials using IAM]. 20 | 21 | If you already have an account, here's the steps to get you IAM access keys: 22 | 23 | 1. Go to the IAM console. 24 | 25 | 2. From the navigation menu, click Users. 26 | 27 | 3. Select your IAM user name, or create a new one. 28 | 29 | 4. Click User Actions, and then click Manage Access Keys. 30 | 31 | 5. Click Create Access Key. 32 | + 33 | NOTE: Your keys will look something like this: + 34 | Access key ID example : AKIAIOSFODNN7*EXAMPLE* + 35 | Secret access key example: wJalrXUtnFEMI/K7MDENG/bPxRfiCY*EXAMPLEKEY* + 36 | + 37 | 38 | 6. Click Download Credentials, and store the keys (a .csv file) in a secure location. 39 | 40 | NOTE: Your secret key present in the credentials file will no longer be available 41 | through the AWS Management Console; you will have the only copy. Keep it confidential 42 | in order to protect your account. 43 | 44 | 45 | Setting up local-dependencies 46 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 47 | 48 | Swift requires python and apache-libcloud to talk to the Amazon EC2. 49 | To install these packages follow these steps: 50 | 51 | Install python and libcloud library on a *debian* based linux distributions such as *Ubuntu*: 52 | [source,bash] 53 | ----- 54 | sudo apt-get instal python python-libcloud 55 | ----- 56 | 57 | If you machine has python and pip installed, you can install libcloud with *pip*: 58 | [source,bash] 59 | ----- 60 | # With super-user privileges do following 61 | pip install apache-libcloud 62 | ----- 63 | 64 | For *OSX*, do the following: 65 | [source,bash] 66 | ----- 67 | port install py-libcloud 68 | ----- -------------------------------------------------------------------------------- /doc/amazon_ec2.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 552 | 746 | 747 | 748 | 750 |
751 |
752 |

Setting up Amazon EC2

753 |
754 |

Amazon Elastic Compute Cloud (Amazon EC2) is a web service that provides resizable compute capacity in the cloud. 755 | Getting cloud resources on-demand from Amazon is now as simple as setting up an account and attaching a credit card.

756 |
757 |

Sign up online

758 |

Sign up for Amazon Web Services (AWS) and create an IAM user by following instructions 759 | from Amazon’s Setup Documentation. 760 | From the documentation you would only need to do the following two steps:

761 |
    762 |
  1. 763 |

    764 | Sign up for AWS 765 |

    766 |
    2. 767 |
  2. 768 |

    769 | Create and IAM User 770 |

    771 |
    3. 772 |
773 |
774 | 775 | 778 | 780 |
776 | Note 777 | To create access keys, you must have permissions to perform the required 779 | IAM actions. For more information, see Managing Credentials using IAM.
781 |
782 |

If you already have an account, here’s the steps to get you IAM access keys:

783 |
    784 |
  1. 785 |

    786 | Go to the IAM console. 787 |

    788 |
    2. 789 |
  2. 790 |

    791 | From the navigation menu, click Users. 792 |

    793 |
    3. 794 |
  3. 795 |

    796 | Select your IAM user name, or create a new one. 797 |

    798 |
    4. 799 |
  4. 800 |

    801 | Click User Actions, and then click Manage Access Keys. 802 |

    803 |
    5. 804 |
  5. 805 |

    806 | Click Create Access Key. 807 |

    808 |
    809 | 810 | 813 | 816 |
    811 | Note 812 | Your keys will look something like this:
    814 | Access key ID example : AKIAIOSFODNN7*EXAMPLE*
    815 | Secret access key example: wJalrXUtnFEMI/K7MDENG/bPxRfiCY*EXAMPLEKEY*
    817 |
    818 |
    6. 819 |
  6. 820 |

    821 | Click Download Credentials, and store the keys (a .csv file) in a secure location. 822 |

    823 |
    7. 824 |
825 |
826 | 827 | 830 | 833 |
828 | Note 829 | Your secret key present in the credentials file will no longer be available 831 | through the AWS Management Console; you will have the only copy. Keep it confidential 832 | in order to protect your account.
834 |
835 |
836 |
837 |

Setting up local-dependencies

838 |

Swift requires python and apache-libcloud to talk to the Amazon EC2. 839 | To install these packages follow these steps:

840 |

Install python and libcloud library on a debian based linux distributions such as Ubuntu:

841 |
842 |
846 | 
sudo apt-get instal python python-libcloud
847 |

If you machine has python and pip installed, you can install libcloud with pip:

848 |
849 |
853 | 
# With super-user privileges do following
854 | pip install apache-libcloud
855 |

For OSX, do the following:

856 |
857 |
861 | 
port install py-libcloud
862 |
863 |
864 |
865 |
866 |
867 | 872 | 873 | 874 | -------------------------------------------------------------------------------- /doc/asciidoc.css: -------------------------------------------------------------------------------- 1 | a:link { color:navy; } 2 | a:visited { color:navy; } 3 | 4 | #content .listingblock code { 5 | /* margin-left: 32px; */ 6 | line-height: 140%; 7 | /* padding: 8px; */ 8 | background-color: #f7f7f7; 9 | /* border: thin solid #d0d0d0; */ 10 | border-radius: 3px; 11 | width: 780px; 12 | display: block; 13 | } 14 | -------------------------------------------------------------------------------- /doc/build_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | asciidoc -a icons -a toc -a toplevels=3 -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o tutorial.html README 4 | asciidoc -a icons -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o amazon_ec2.html amazon_ec2.asc 5 | asciidoc -a icons -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o swift-config.html swift.conf.asc 6 | 7 | 8 | -------------------------------------------------------------------------------- /doc/cumulativeplot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/cumulativeplot.png -------------------------------------------------------------------------------- /doc/figs/modis.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/figs/modis.dia -------------------------------------------------------------------------------- /doc/figs/modis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/figs/modis.png -------------------------------------------------------------------------------- /doc/images/icons/caution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/caution.png -------------------------------------------------------------------------------- /doc/images/icons/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/example.png -------------------------------------------------------------------------------- /doc/images/icons/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/home.png -------------------------------------------------------------------------------- /doc/images/icons/important.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/important.png -------------------------------------------------------------------------------- /doc/images/icons/next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/next.png -------------------------------------------------------------------------------- /doc/images/icons/note.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/note.png -------------------------------------------------------------------------------- /doc/images/icons/prev.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/prev.png -------------------------------------------------------------------------------- /doc/images/icons/tip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/tip.png -------------------------------------------------------------------------------- /doc/images/icons/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/up.png -------------------------------------------------------------------------------- /doc/images/icons/warning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/images/icons/warning.png -------------------------------------------------------------------------------- /doc/part01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/part01.png -------------------------------------------------------------------------------- /doc/part02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/part02.png -------------------------------------------------------------------------------- /doc/part03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/part03.png -------------------------------------------------------------------------------- /doc/part04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/part04.png -------------------------------------------------------------------------------- /doc/part05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/part05.png -------------------------------------------------------------------------------- /doc/part06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swift-lang/swift-tutorial/801e132ed3a7eb34daa4ad268f227fbdf1eb5f71/doc/part06.png -------------------------------------------------------------------------------- /doc/push.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # scp -r swift-cray-tutorial.html images *png login.ci.uchicago.edu:/ci/www/projects/swift/tutorials/cray 4 | 5 | tar zcf - --exclude-vcs *html *png images | ssh login.ci.uchicago.edu "cd /ci/www/projects/swift/tutorials/cray; tar zxf -" 6 | -------------------------------------------------------------------------------- /doc/swift-config.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | Swift Configuration Simplified 8 | 552 | 746 | 747 | 748 | 751 |
752 |
753 |
754 |

The swift configuration file (swift.conf) is a description of how swift should execute your 755 | swift workflow on one or more of your target computation resources. The swift config file 756 | therefore contains three aspects:

757 |
    758 |
  • 759 |

    760 | A description of the target resource, and what you want from it, such as: 761 |

    762 |
      763 |
    • 764 |

      765 | URL of the site, scheduler the system uses, what queue you want to run on, how many nodes you need, etc 766 |

      767 |
      • 768 |
    769 |
    • 770 |
  • 771 |

    772 | A description of application that you will run on these resources: 773 |

    774 |
      775 |
    • 776 |

      777 | Executable for each app, walltimes for apps etc 778 |

      779 |
      • 780 |
    781 |
    • 782 |
  • 783 |

    784 | Configuration for the workflow execution behavior: 785 |

    786 |
      787 |
    • 788 |

      789 | handling errors, network port to use, leave files for debug etc. 790 |

      791 |
      • 792 |
    793 |
    • 794 |
795 |

Simple canned configurations work great for a vast majority of cases. Nevertheless, Swift provides 796 | the user with several different paramenters to tune and tweak your workflow to perfection.

797 |

Here’s the skeleton of a swift.conf:

798 |
799 |
803 | 
site.<name> {
804 |     # How to negotiate resources with the site
805 |     execution {
806 |         [URL        : < URL_OF_COMPUTE_RESOURCE > ]
807 |         type        : < local | coaster | coaster-persistent >
808 |         [jobmanager : < <local|ssh-cl>:<local,pbs,sge,slurm..>  >]
809 |     }
810 | 
811 |     # Directory for placing intermediate work files
812 |     workDirectory: <path>
813 | 
814 |     # Site specific options (eg. queue to use,nodes to request)
815 |     [<site options>]
816 | 
817 |     # How to move data to/from the compute nodes
818 |     [staging: "swift" | "local" | "service-local" | "shared-fs" | "wrapper"]
819 | 
820 |     # Describe the application to run on this site
821 |     [<application declarations>]
822 | }
823 |

Here’s a simple config file for a cluster which uses slurm scheduler:

824 |
825 |
829 | 
sites : midway
830 | 
831 | site.midway {
832 |     execution {
833 |         type      : "coaster"                         # Use coasters to run on remote sites
834 |         URL       : "swift.rcc.uchicago.edu"          # Midway login node | Alternatively use midway.rcc.uchicago.edu
835 |         jobManager: "ssh-cl:slurm"                    # Use ssh-cl to connect, slurm is the Local resource manager
836 |         options {
837 |             jobQueue        : "sandyb"                # Select queue from (sandyb, westmere, ...)
838 |         }
839 |     }
840 |     staging             : "local"                     # Stage files from "local" system to Midway
841 |     workDirectory       : "/tmp/"${env.MIDWAY_USERNAME} # Location for intermediate files
842 |     maxParallelTasks    : 101                         # Maximum number of parallel tasks
843 |     initialParallelTasks: 100                         # Maximum number of tasks at start
844 |     app.ALL { executable: "*" }                       # All tasks to be found from commandline
845 | }
846 |
847 |
848 |
849 |
850 | 855 | 856 | 857 | -------------------------------------------------------------------------------- /doc/swift.conf.asc: -------------------------------------------------------------------------------- 1 | Swift Configuration Simplified 2 | ============================== 3 | 4 | 5 | The swift configuration file (swift.conf) is a description of how swift should execute your 6 | swift workflow on one or more of your target computation resources. The swift config file 7 | therefore contains three aspects: 8 | 9 | - A description of the target resource, and what you want from it, such as: 10 | * URL of the site, scheduler the system uses, what queue you want to run on, how many nodes you need, etc 11 | 12 | - A description of application that you will run on these resources: 13 | * Executable for each app, walltimes for apps etc 14 | 15 | - Configuration for the workflow execution behavior: 16 | * handling errors, network port to use, leave files for debug etc. 17 | 18 | Simple canned configurations work great for a vast majority of cases. Nevertheless, Swift provides 19 | the user with several different paramenters to tune and tweak your workflow to perfection. 20 | 21 | Here's the skeleton of a swift.conf: 22 | 23 | [source,python] 24 | ---- 25 | site. { 26 | # How to negotiate resources with the site 27 | execution { 28 | [URL : < URL_OF_COMPUTE_RESOURCE > ] 29 | type : < local | coaster | coaster-persistent > 30 | [jobmanager : < : >] 31 | } 32 | 33 | # Directory for placing intermediate work files 34 | workDirectory: 35 | 36 | # Site specific options (eg. queue to use,nodes to request) 37 | [] 38 | 39 | # How to move data to/from the compute nodes 40 | [staging: "swift" | "local" | "service-local" | "shared-fs" | "wrapper"] 41 | 42 | # Describe the application to run on this site 43 | [] 44 | } 45 | ---- 46 | 47 | Here's a simple config file for a cluster which uses slurm scheduler: 48 | [source, python] 49 | ----- 50 | sites : midway 51 | 52 | site.midway { 53 | execution { 54 | type : "coaster" # Use coasters to run on remote sites 55 | URL : "swift.rcc.uchicago.edu" # Midway login node | Alternatively use midway.rcc.uchicago.edu 56 | jobManager: "ssh-cl:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 57 | options { 58 | jobQueue : "sandyb" # Select queue from (sandyb, westmere, ...) 59 | } 60 | } 61 | staging : "local" # Stage files from "local" system to Midway 62 | workDirectory : "/tmp/"${env.MIDWAY_USERNAME} # Location for intermediate files 63 | maxParallelTasks : 101 # Maximum number of parallel tasks 64 | initialParallelTasks: 100 # Maximum number of tasks at start 65 | app.ALL { executable: "*" } # All tasks to be found from commandline 66 | } 67 | ----- 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /ec2.asc: -------------------------------------------------------------------------------- 1 | Swift for the Cloud 2 | ------------------- 3 | 4 | Modes of operation 5 | ------------------ 6 | 7 | 1. Static mode: You define your cluster ahead of swift runs. 8 | 2. Dynamic mode: Cloud resources provisioned dynamically. 9 | 10 | Swift installation 11 | ~~~~~~~~~~~~~~~~~~ 12 | 13 | Prerequisites: 14 | Java 1.7 15 | Ant 16 | Python 2.7 17 | The following steps 18 | [source, bash] 19 | ----- 20 | # Install swift-trunk from git 21 | https://github.com/swift-lang/swift-k.git 22 | # Extract package 23 | tar xfz swift-0.95-RC6.tar.gz 24 | # Add swift to the PATH environment variable 25 | export PATH=$PATH:/path/to/swift-0.95-RC6/bin 26 | ----- 27 | 28 | Get the swift-on-cloud repository 29 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 30 | 31 | Clone the repository from github 32 | [source,bash] 33 | ---- 34 | git clone https://github.com/yadudoc/swift-on-cloud.git 35 | cd swift-on-cloud 36 | ---- 37 | 38 | Or, download the zip file from github and unpack. 39 | [source,bash] 40 | ---- 41 | # Download 42 | wget https://github.com/yadudoc/swift-on-cloud/archive/master.zip 43 | unzip master.zip 44 | mv swift-on-cloud-master swift-on-cloud 45 | cd swift-on-cloud 46 | ---- 47 | 48 | Run the swift-cloud-tutorial from the cloud 49 | ------------------------------------------- 50 | 51 | To run the tutorial on Google Compute Engine (GCE), follow the instructions here: + 52 | https://github.com/yadudoc/swift-on-cloud/tree/master/compute-engine + 53 | or, follow instructions for GCE, in the compute-engine folder of the swift-on-cloud 54 | repository. Once your instances are running, connect to the headnode. Everthing 55 | that you require for the swift-cloud-tutorial is already set up for you on the headnode. 56 | 57 | 58 | Simple "science applications" for the workflow tutorial 59 | ------------------------------------------------------- 60 | 61 | This tutorial is based on two intentionally trivial example programs, 62 | `simulation.sh` and `stats.sh`, (implemented as bash shell scripts) 63 | that serve as easy-to-understand proxies for real science 64 | applications. These "programs" behave as follows. 65 | 66 | simulate.sh 67 | ~~~~~~~~~~~ 68 | 69 | The simulation.sh script serves as a trivial proxy for any more 70 | complex scientific simulation application. It generates and prints a 71 | set of one or more random integers in the range [0-2^62) as controlled 72 | by its command line arguments, which are: 73 | 74 | ----- 75 | $ ./app/simulate.sh --help 76 | ./app/simulate.sh: usage: 77 | -b|--bias offset bias: add this integer to all results [0] 78 | -B|--biasfile file of integer biases to add to results [none] 79 | -l|--log generate a log in stderr if not null [y] 80 | -n|--nvalues print this many values per simulation [1] 81 | -r|--range range (limit) of generated results [100] 82 | -s|--seed use this integer [0..32767] as a seed [none] 83 | -S|--seedfile use this file (containing integer seeds [0..32767]) one per line [none] 84 | -t|--timesteps number of simulated "timesteps" in seconds (determines runtime) [1] 85 | -x|--scale scale the results by this integer [1] 86 | -h|-?|?|--help print this help 87 | $ 88 | ----- 89 | 90 | All of thess arguments are optional, with default values indicated above as `[n]`. 91 | 92 | //// 93 | .simulation.sh arguments 94 | [width="80%",cols="^2,10",options="header"] 95 | 96 | |======================= 97 | |Argument|Short|Description 98 | |1 |runtime: sets run time of simulation.sh in seconds 99 | |2 |range: limits generated values to the range [0,range-1] 100 | |3 |biasfile: add the integer contained in this file to each value generated 101 | |4 |scale: multiplies each generated value by this integer 102 | |5 |count: number of values to generate in the simulation 103 | |======================= 104 | //// 105 | 106 | With no arguments, simulate.sh prints 1 number in the range of 107 | 1-100. Otherwise it generates n numbers of the form (R*scale)+bias 108 | where R is a random integer. By default it logs information about its 109 | execution environment to stderr. Here's some examples of its usage: 110 | 111 | ----- 112 | $ simulate.sh 2>log 113 | 5 114 | $ head -4 log 115 | 116 | Called as: /home/wilde/swift/tut/CIC_2013-08-09/app/simulate.sh: 117 | Start time: Thu Aug 22 12:40:24 CDT 2013 118 | Running on node: login01.osgconnect.net 119 | 120 | $ simulate.sh -n 4 -r 1000000 2>log 121 | 239454 122 | 386702 123 | 13849 124 | 873526 125 | 126 | $ simulate.sh -n 3 -r 1000000 -x 100 2>log 127 | 6643700 128 | 62182300 129 | 5230600 130 | 131 | $ simulate.sh -n 2 -r 1000 -x 1000 2>log 132 | 565000 133 | 636000 134 | 135 | $ time simulate.sh -n 2 -r 1000 -x 1000 -t 3 2>log 136 | 336000 137 | 320000 138 | real 0m3.012s 139 | user 0m0.005s 140 | sys 0m0.006s 141 | ----- 142 | 143 | stats.sh 144 | ~~~~~~~~ 145 | 146 | The stats.sh script serves as a trivial model of an "analysis" 147 | program. It reads N files each containing M integers and simply prints 148 | the\ average of all those numbers to stdout. Similarly to simulate.sh 149 | it logs environmental information to the stderr. 150 | 151 | ----- 152 | $ ls f* 153 | f1 f2 f3 f4 154 | 155 | $ cat f* 156 | 25 157 | 60 158 | 40 159 | 75 160 | 161 | $ stats.sh f* 2>log 162 | 50 163 | ----- 164 | 165 | 166 | Basic of the Swift language with local execution 167 | ------------------------------------------------ 168 | 169 | A Summary of Swift in a nutshell 170 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 171 | 172 | * Swift scripts are text files ending in `.swift` The `swift` command 173 | runs on any host, and executes these scripts. `swift` is a Java 174 | application, which you can install almost anywhere. On Linux, just 175 | unpack the distribution `tar` file and add its `bin/` directory to 176 | your `PATH`. 177 | 178 | * Swift scripts run ordinary applications, just like shell scripts 179 | do. Swift makes it easy to run these applications on parallel and 180 | remote computers (from laptops to supercomputers). If you can `ssh` to 181 | the system, Swift can likely run applications there. 182 | 183 | * The details of where to run applications and how to get files back 184 | and forth are described in configuration files separate from your 185 | program. Swift speaks ssh, PBS, Condor, SLURM, LSF, SGE, Cobalt, and 186 | Globus to run applications, and scp, http, ftp, and GridFTP to move 187 | data. 188 | 189 | * The Swift language has 5 main data types: `boolean`, `int`, 190 | `string`, `float`, and `file`. Collections of these are dynamic, 191 | sparse arrays of arbitrary dimension and structures of scalars and/or 192 | arrays defined by the `type` declaration. 193 | 194 | * Swift file variables are "mapped" to external files. Swift sends 195 | files to and from remote systems for you automatically. 196 | 197 | * Swift variables are "single assignment": once you set them you can't 198 | change them (in a given block of code). This makes Swift a natural, 199 | "parallel data flow" language. This programming model keeps your 200 | workflow scripts simple and easy to write and understand. 201 | 202 | * Swift lets you define functions to "wrap" application programs, and 203 | to cleanly structure more complex scripts. Swift `app` functions take 204 | files and parameters as inputs and return files as outputs. 205 | 206 | * A compact set of built-in functions for string and file 207 | manipulation, type conversions, high level IO, etc. is provided. 208 | Swift's equivalent of `printf()` is `tracef()`, with limited and 209 | slightly different format codes. 210 | 211 | * Swift's `foreach {}` statement is the main parallel workhorse of the 212 | language, and executes all iterations of the loop concurrently. The 213 | actual number of parallel tasks executed is based on available 214 | resources and settable "throttles". 215 | 216 | * In fact, Swift conceptually executes *all* the statements, 217 | expressions and function calls in your program in parallel, based on 218 | data flow. These are similarly throttled based on available resources 219 | and settings. 220 | 221 | * Swift also has `if` and `switch` statements for conditional 222 | execution. These are seldom needed in simple workflows but they enable 223 | very dynamic workflow patterns to be specified. 224 | 225 | We'll see many of these points in action in the examples below. Lets 226 | get started! 227 | 228 | Part 1: Run a single application under Swift 229 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 230 | 231 | The first swift script, p1.swift, runs simulate.sh to generate a 232 | single random number. It writes the number to a file. 233 | 234 | image::part01.png["p1 workflow",align="center"] 235 | 236 | .p1.swift 237 | ----- 238 | sys::[cat ../part01/p1.swift] 239 | ----- 240 | 241 | 242 | To run this script, run the following command: 243 | ----- 244 | $ cd part01 245 | $ swift p1.swift 246 | Swift 0.94.1 RC2 swift-r6895 cog-r3765 247 | 248 | RunID: 20130827-1413-oa6fdib2 249 | Progress: time: Tue, 27 Aug 2013 14:13:33 -0500 250 | Final status: Tue, 27 Aug 2013 14:13:33 -0500 Finished successfully:1 251 | $ cat sim.out 252 | 84 253 | $ swift p1.swift 254 | $ cat sim.out 255 | 36 256 | ----- 257 | 258 | To cleanup the directory and remove all outputs (including the log 259 | files and directories that Swift generates), run the cleanup script 260 | which is located in the tutorial PATH: 261 | 262 | [source,bash] 263 | ----- 264 | $ cleanup 265 | ----- 266 | 267 | NOTE: You'll also find two Swift configuration files in each `partNN` 268 | directory of this tutorial. These specify the environment-specific 269 | details of where to find application programs (file `apps`) and where 270 | to run them (file `sites.xml`). These files will be explained in more 271 | detail in parts 4-6, and can be ignored for now. 272 | 273 | //// 274 | It defines 275 | things like the work directory, the scheduler to use, and how to 276 | control parallelism. The sites.xml file below will tell Swift to run 277 | on the local machine only, and run just 1 task at a time. 278 | 279 | .swift.properties 280 | ----- 281 | sys::[cat ../part01/swift.properties] 282 | ----- 283 | 284 | In this case, it 285 | indicates that the app "simulate" (the first token in the command line 286 | declaration of the function `simulation`, at line NNN) is located in the file 287 | simulate.sh and (since the path `simulate.sh` is specified with no 288 | directory components) Swift expects that the `simulate.sh` executable 289 | will be available in your $PATH. 290 | 291 | .apps 292 | ----- 293 | sys::[cat ../part01/apps] 294 | ----- 295 | 296 | //// 297 | 298 | Part 2: Running an ensemble of many apps in parallel with a "foreach" loop 299 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 300 | 301 | The `p2.swift` script introduces the `foreach` parallel iteration 302 | construct to run many concurrent simulations. 303 | 304 | image::part02.png[align="center"] 305 | 306 | .p2.swift 307 | ----- 308 | sys::[cat ../part02/p2.swift] 309 | ----- 310 | 311 | The script also shows an 312 | example of naming the output files of an ensemble run. In this case, the output files will be named 313 | `output/sim_N.out`. 314 | 315 | In part 2, we also update the apps file. Instead of using shell script (simulate.sh), we use 316 | the equivalent python version (simulate.py). The new apps file now looks like this: 317 | 318 | ----- 319 | sys::[cat ../part02/apps] 320 | ----- 321 | 322 | Swift does not need to know anything about the language an application is written in. The application 323 | can be written in Perl, Python, Java, Fortran, or any other language. 324 | 325 | To run the script and view the output: 326 | ----- 327 | $ cd ../part02 328 | $ swift p2.swift 329 | $ ls output 330 | sim_0.out sim_1.out sim_2.out sim_3.out sim_4.out sim_5.out sim_6.out sim_7.out sim_8.out sim_9.out 331 | $ more output/* 332 | :::::::::::::: 333 | output/sim_0.out 334 | :::::::::::::: 335 | 44 336 | :::::::::::::: 337 | output/sim_1.out 338 | :::::::::::::: 339 | 55 340 | ... 341 | :::::::::::::: 342 | output/sim_9.out 343 | :::::::::::::: 344 | 82 345 | ----- 346 | 347 | Part 3: Analyzing results of a parallel ensemble 348 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 349 | 350 | After all the parallel simulations in an ensemble run have completed, 351 | its typically necessary to gather and analyze their results with some 352 | kind of post-processing analysis program or script. p3.swift 353 | introduces such a postprocessing step. In this case, the files created 354 | by all of the parallel runs of `simulation.sh` will be averaged by by 355 | the trivial "analysis application" `stats.sh`: 356 | 357 | image::part03.png[align="center"] 358 | 359 | .p3.swift 360 | ---- 361 | sys::[cat ../part03/p3.swift] 362 | ---- 363 | 364 | To run: 365 | ---- 366 | $ cd part03 367 | $ swift p3.swift 368 | ---- 369 | 370 | Note that in `p3.swift` we expose more of the capabilities of the 371 | `simulate.sh` application to the `simulation()` app function: 372 | 373 | ----- 374 | app (file o) simulation (int sim_steps, int sim_range, int sim_values) 375 | { 376 | simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=filename(o); 377 | } 378 | ----- 379 | 380 | `p3.swift` also shows how to fetch application-specific values from 381 | the `swift` command line in a Swift script using `arg()` which 382 | accepts a keyword-style argument and its default value: 383 | 384 | ----- 385 | int nsim = toInt(arg("nsim","10")); 386 | int steps = toInt(arg("steps","1")); 387 | int range = toInt(arg("range","100")); 388 | int values = toInt(arg("values","5")); 389 | ----- 390 | 391 | Now we can specify that more runs should be performed and that each should run for more timesteps, and produce more that one value each, within a specified range, using command line arguments placed after the Swift script name in the form `-parameterName=value`: 392 | 393 | ----- 394 | $ swift p3.swift -nsim=3 -steps=10 -values=4 -range=1000000 395 | 396 | Swift 0.94.1 RC2 swift-r6895 cog-r3765 397 | 398 | RunID: 20130827-1439-s3vvo809 399 | Progress: time: Tue, 27 Aug 2013 14:39:42 -0500 400 | Progress: time: Tue, 27 Aug 2013 14:39:53 -0500 Active:2 Stage out:1 401 | Final status: Tue, 27 Aug 2013 14:39:53 -0500 Finished successfully:4 402 | 403 | $ ls output/ 404 | average.out sim_0.out sim_1.out sim_2.out 405 | $ more output/* 406 | :::::::::::::: 407 | output/average.out 408 | :::::::::::::: 409 | 651368 410 | :::::::::::::: 411 | output/sim_0.out 412 | :::::::::::::: 413 | 735700 414 | 886206 415 | 997391 416 | 982970 417 | :::::::::::::: 418 | output/sim_1.out 419 | :::::::::::::: 420 | 260071 421 | 264195 422 | 869198 423 | 933537 424 | :::::::::::::: 425 | output/sim_2.out 426 | :::::::::::::: 427 | 201806 428 | 213540 429 | 527576 430 | 944233 431 | ----- 432 | 433 | Now try running (`-nsim=`) 100 simulations of (`-steps=`) 1 second each: 434 | 435 | ----- 436 | $ swift p3.swift -nsim=100 -steps=1 437 | Swift 0.94.1 RC2 swift-r6895 cog-r3765 438 | 439 | RunID: 20130827-1444-rq809ts6 440 | Progress: time: Tue, 27 Aug 2013 14:44:55 -0500 441 | Progress: time: Tue, 27 Aug 2013 14:44:56 -0500 Selecting site:79 Active:20 Stage out:1 442 | Progress: time: Tue, 27 Aug 2013 14:44:58 -0500 Selecting site:58 Active:20 Stage out:1 Finished successfully:21 443 | Progress: time: Tue, 27 Aug 2013 14:44:59 -0500 Selecting site:37 Active:20 Stage out:1 Finished successfully:42 444 | Progress: time: Tue, 27 Aug 2013 14:45:00 -0500 Selecting site:16 Active:20 Stage out:1 Finished successfully:63 445 | Progress: time: Tue, 27 Aug 2013 14:45:02 -0500 Active:15 Stage out:1 Finished successfully:84 446 | Progress: time: Tue, 27 Aug 2013 14:45:03 -0500 Finished successfully:101 447 | Final status: Tue, 27 Aug 2013 14:45:03 -0500 Finished successfully:101 448 | ----- 449 | 450 | We can see from Swift's "progress" status that the tutorial's default 451 | `swift.properties` parameters for local execution allow Swift to run up to 20 452 | application invocations concurrently on the login node. We'll look at 453 | this in more detail in the next sections where we execute applications 454 | on the site's compute nodes. 455 | 456 | 457 | Running applications on compute nodes with Swift 458 | ------------------------------------------------ 459 | 460 | Part 4: Running a parallel ensemble on compute nodes 461 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 462 | 463 | `p4.swift` will run our mock "simulation" 464 | applications on compute nodes. The script is similar to as 465 | `p3.swift`, but specifies that each simulation app invocation should 466 | additionally return the log file which the application writes to 467 | `stderr`. 468 | 469 | //// 470 | 471 | FIXME: need to revise this figure: drop prog: 472 | 473 | , making the parallel portion of the script behave like this: 474 | 475 | image::part04.png[align="center"] 476 | 477 | .p4.swift 478 | ---- 479 | sys::[cat ../part04/p4.swift] 480 | ---- 481 | //// 482 | 483 | Now when you run `swift p4.swift` you'll see that two types output 484 | files will placed in the `output/` directory: `sim_N.out` and 485 | `sim_N.log`. The log files provide data on the runtime environment of 486 | each app invocation. For example: 487 | 488 | ----- 489 | $ cat output/sim_0.log 490 | 491 | Called as: simulate.sh: --timesteps 1 --range 100 --nvalues 5 492 | 493 | Start time: Tue Oct 22 14:54:11 CDT 2013 494 | Running as user: uid=5116(davidk) gid=311(collab) groups=311(collab),104(fuse),1349(swift),45053(swat) 495 | Running on node: stomp 496 | Node IP address: 140.221.9.237 497 | 498 | Simulation parameters: 499 | 500 | bias=0 501 | biasfile=none 502 | initseed=none 503 | log=yes 504 | paramfile=none 505 | range=100 506 | scale=1 507 | seedfile=none 508 | timesteps=1 509 | output width=8 510 | 511 | Environment: 512 | 513 | EDITOR=vim 514 | HOME=/homes/davidk 515 | JAVA_HOME=/nfs/proj-davidk/jdk1.7.0_01 516 | LANG=C 517 | .... 518 | ----- 519 | 520 | ///// 521 | To tell Swift to run the apps on compute nodes, we specify in the 522 | `apps` file that the apps should be executed on the `cloud` site 523 | (instead of the `localhost` site). We can specify the location of 524 | each app in the third field of the `apps` file, with either an 525 | absolute pathname or the name of an executable to be located in 526 | `PATH`). Here we use the latter form: 527 | 528 | ----- 529 | $ cat apps 530 | cloud simulate simulate.sh 531 | cloud stats stats.sh 532 | ----- 533 | 534 | You can experiment, for example, with an alternate version of stats.sh by specfying that app's location explicitly: 535 | 536 | ----- 537 | $ cat apps 538 | cloud simulate simulate.sh 539 | cloud stats /home/users/p01532/bin/my-alt-stats.sh 540 | ----- 541 | 542 | We can see that when we run many apps requesting a larger set of nodes (6), we are indeed running on the compute nodes: 543 | ----- 544 | $ swift p4.swift -nsim=1000 -steps=1 545 | Swift 0.94.1 RC2 swift-r6895 cog-r3765 546 | 547 | RunID: 20130827-1638-t23ax37a 548 | Progress: time: Tue, 27 Aug 2013 16:38:11 -0500 549 | Progress: time: Tue, 27 Aug 2013 16:38:12 -0500 Initializing:966 550 | Progress: time: Tue, 27 Aug 2013 16:38:13 -0500 Selecting site:499 Submitting:500 Submitted:1 551 | Progress: time: Tue, 27 Aug 2013 16:38:14 -0500 Selecting site:499 Stage in:1 Submitted:500 552 | Progress: time: Tue, 27 Aug 2013 16:38:16 -0500 Selecting site:499 Submitted:405 Active:95 Stage out:1 553 | Progress: time: Tue, 27 Aug 2013 16:38:17 -0500 Selecting site:430 Submitted:434 Active:66 Stage out:1 Finished successfully:69 554 | Progress: time: Tue, 27 Aug 2013 16:38:18 -0500 Selecting site:388 Submitted:405 Active:95 Stage out:1 Finished successfully:111 555 | ... 556 | Progress: time: Tue, 27 Aug 2013 16:38:30 -0500 Stage in:1 Submitted:93 Active:94 Finished successfully:812 557 | Progress: time: Tue, 27 Aug 2013 16:38:31 -0500 Submitted:55 Active:95 Stage out:1 Finished successfully:849 558 | Progress: time: Tue, 27 Aug 2013 16:38:32 -0500 Active:78 Stage out:1 Finished successfully:921 559 | Progress: time: Tue, 27 Aug 2013 16:38:34 -0500 Active:70 Stage out:1 Finished successfully:929 560 | Progress: time: Tue, 27 Aug 2013 16:38:37 -0500 Stage in:1 Finished successfully:1000 561 | Progress: time: Tue, 27 Aug 2013 16:38:38 -0500 Stage out:1 Finished successfully:1000 562 | Final status: Tue, 27 Aug 2013 16:38:38 -0500 Finished successfully:1001 563 | 564 | $ grep "on node:" output/*log | head 565 | output/sim_0.log:Running on node: nid00063 566 | output/sim_100.log:Running on node: nid00060 567 | output/sim_101.log:Running on node: nid00061 568 | output/sim_102.log:Running on node: nid00032 569 | output/sim_103.log:Running on node: nid00060 570 | output/sim_104.log:Running on node: nid00061 571 | output/sim_105.log:Running on node: nid00032 572 | output/sim_106.log:Running on node: nid00060 573 | output/sim_107.log:Running on node: nid00061 574 | output/sim_108.log:Running on node: nid00062 575 | 576 | $ grep "on node:" output/*log | awk '{print $4}' | sort | uniq -c 577 | 158 nid00032 578 | 156 nid00033 579 | 171 nid00060 580 | 178 nid00061 581 | 166 nid00062 582 | 171 nid00063 583 | $ hostname 584 | raven 585 | $ hostname -f 586 | nid00008 587 | ----- 588 | ///// 589 | Performing larger Swift runs 590 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 591 | 592 | To test with larger runs, there are two changes that are required. The first is a 593 | change to the command line arguments. The example below will run 1000 simulations 594 | with each simulation taking 5 seconds. 595 | 596 | ----- 597 | $ swift p6.swift -steps=5 -nsim=1000 598 | ----- 599 | 600 | Part 5: Controlling the compute-node pools where applications run 601 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 602 | 603 | This section is under development. 604 | 605 | Part 6: Specifying more complex workflow patterns 606 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 607 | 608 | p6.swift expands the workflow pattern of p4.swift to add additional 609 | stages to the workflow. Here, we generate a dynamic seed value that 610 | will be used by all of the simulations, and for each simulation, we 611 | run an pre-processing application to generate a unique "bias 612 | file". This pattern is shown below, followed by the Swift script. 613 | 614 | image::part06.png[align="center"] 615 | 616 | .p6.swift 617 | ---- 618 | sys::[cat ../part06/p6.swift] 619 | ---- 620 | 621 | Note that the workflow is based on data flow dependencies: each simulation depends on the seed value, calculated in this statement: 622 | ----- 623 | seedfile = genseed(1); 624 | ----- 625 | and on the bias file, computed and then consumed in these two dependent statements: 626 | ----- 627 | biasfile = genbias(1000, 20, simulate_script); 628 | (simout,simlog) = simulation(steps, range, biasfile, 1000000, values, simulate_script, seedfile); 629 | ----- 630 | 631 | To run: 632 | ---- 633 | $ cd ../part06 634 | $ swift p6.swift 635 | ---- 636 | 637 | The default parameters result in the following execution log: 638 | 639 | ----- 640 | $ swift p6.swift 641 | Swift 0.94.1 RC2 swift-r6895 cog-r3765 642 | 643 | RunID: 20130827-1917-jvs4gqm5 644 | Progress: time: Tue, 27 Aug 2013 19:17:56 -0500 645 | 646 | *** Script parameters: nsim=10 range=100 num values=10 647 | 648 | Progress: time: Tue, 27 Aug 2013 19:17:57 -0500 Stage in:1 Submitted:10 649 | Generated seed=382537 650 | Progress: time: Tue, 27 Aug 2013 19:17:59 -0500 Active:9 Stage out:1 Finished successfully:11 651 | Final status: Tue, 27 Aug 2013 19:18:00 -0500 Finished successfully:22 652 | ----- 653 | which produces the following output: 654 | ----- 655 | $ ls -lrt output 656 | total 264 657 | -rw-r--r-- 1 p01532 61532 9 Aug 27 19:17 seed.dat 658 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_9.dat 659 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_8.dat 660 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_7.dat 661 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_6.dat 662 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_5.dat 663 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_4.dat 664 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_3.dat 665 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_2.dat 666 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_1.dat 667 | -rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_0.dat 668 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_9.out 669 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_9.log 670 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_8.log 671 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_7.out 672 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_6.out 673 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_6.log 674 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_5.out 675 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_5.log 676 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_4.out 677 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_4.log 678 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_1.log 679 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_8.out 680 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_7.log 681 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_3.out 682 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_3.log 683 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_2.out 684 | -rw-r--r-- 1 p01532 61532 14898 Aug 27 19:18 sim_2.log 685 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_1.out 686 | -rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_0.out 687 | -rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_0.log 688 | -rw-r--r-- 1 p01532 61532 9 Aug 27 19:18 average.out 689 | -rw-r--r-- 1 p01532 61532 14675 Aug 27 19:18 average.log 690 | ----- 691 | 692 | Each sim_N.out file is the sum of its bias file plus newly "simulated" random output scaled by 1,000,000: 693 | 694 | ----- 695 | $ cat output/bias_0.dat 696 | 302 697 | 489 698 | 81 699 | 582 700 | 664 701 | 290 702 | 839 703 | 258 704 | 506 705 | 310 706 | 293 707 | 508 708 | 88 709 | 261 710 | 453 711 | 187 712 | 26 713 | 198 714 | 402 715 | 555 716 | 717 | $ cat output/sim_0.out 718 | 64000302 719 | 38000489 720 | 32000081 721 | 12000582 722 | 46000664 723 | 36000290 724 | 35000839 725 | 22000258 726 | 49000506 727 | 75000310 728 | ----- 729 | 730 | We produce 20 values in each bias file. Simulations of less than that 731 | number of values ignore the unneeded number, while simualtions of more 732 | than 20 will use the last bias number for all remoaining values past 733 | 20. As an exercise, adjust the code to produce the same number of 734 | bias values as is needed for each simulation. As a further exercise, 735 | modify the script to generate a unique seed value for each simulation, 736 | which is a common practice in ensemble computations. 737 | 738 | Tips for Specific Resources 739 | --------------------------- 740 | 741 | Open Science Data Cloud 742 | ~~~~~~~~~~~~~~~~~~~~~~~ 743 | 1. When you start instances on OSDC, use the standard Ubuntu image. 744 | 2. Ensure that your SSH key is added to the instance for password login. 745 | 3. Swift should run on the OSDC headnode. 746 | 4. You can use the following command within coaster-service.conf to automatically 747 | populate WORKER_HOSTS with the IP addresses of all active instances you have running. 748 | 749 | ----- 750 | export WORKER_HOSTS=$( nova list | grep ACTIVE | sed -e 's/^.*private=//' -e 's/ .*//' |sed ':a;N;$!ba;s/\n/ /g' ) 751 | ----- 752 | -------------------------------------------------------------------------------- /local.conf: -------------------------------------------------------------------------------- 1 | sites: [localhost] 2 | 3 | site.localhost { 4 | execution { 5 | type: "local" 6 | URL : "localhost" 7 | } 8 | staging : direct 9 | workDirectory : "/tmp/"${env.USER}"/swiftwork" 10 | maxParallelTasks : 20 11 | initialParallelTasks: 20 12 | app.ALL { executable: "*" } 13 | } 14 | 15 | TCPPortRange: "50000,51000" 16 | lazyErrors: false 17 | executionRetries: 0 18 | keepSiteDir: true 19 | providerStagingPinSwiftFiles: false 20 | alwaysTransferWrapperLog: true 21 | -------------------------------------------------------------------------------- /mandelbrot/README: -------------------------------------------------------------------------------- 1 | This is intended to be run from midway. 2 | 3 | Please get this version of swift-trunk, before mods to submit scripts were made. 4 | http://users.rcc.uchicago.edu/~yadunand/swift-trunk-Feb19.tar.gz 5 | 6 | This was only tested locally on midway. 7 | 8 | The assemble step depends on direct staging. 9 | 10 | The swift.conf is setup such that the assemble to gif movie, is done on localhost 11 | while the run_mandelbrot script is executed on the supported MPI sites. 12 | 13 | The run_mandelbrot script requires 3 env variables to be set, which are passed via 14 | the env. definition in the swift.conf, these variables are : 15 | 16 | env.MACHINE: "midway" 17 | env.MPI_APP_PATH: "/scratch/midway/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 18 | env.MPI_RANKS: "12" 19 | 20 | Since MPI build on different machines differ, make your own installation on the sites, 21 | and then point the MPI_APP_PATH at the right executable on the remote system, along 22 | with setting up modules in the if:elif ladder selected by env.MACHINE in run_mandelbrot script. 23 | 24 | Swan single rank runs works for me. THe mandelbrot code needs to be built by the default cc on the system. 25 | Multiple ranks fail with an apsched error. 26 | 27 | Stampede fails to run at all with MPI errors. 28 | -------------------------------------------------------------------------------- /mandelbrot/bin/Makefile: -------------------------------------------------------------------------------- 1 | CC=mpicc 2 | 3 | CFLAGS = -Wall -g 4 | 5 | SRC = $(wildcard *.c) 6 | 7 | DEPS = mandelbrot.h 8 | OBJS = $(SRC:.c=.o) 9 | 10 | EXEC=mandelbrot 11 | 12 | %.o: %.c $(DEPS) 13 | $(CC) -c -o $@ $< $(CFLAGS) 14 | 15 | $(EXEC) : $(OBJS) 16 | $(CC) -o $@ $^ $(CFLAGS) 17 | 18 | clean: 19 | rm -f $(OBJS) $(EXEC) *~ *out *err 20 | -------------------------------------------------------------------------------- /mandelbrot/bin/assemble: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "ARGS: $*" 4 | which convert 5 | if [[ "$?" != "0" ]] 6 | then 7 | echo "The application \"convert\" is necessary for making the movie" 1>&2 8 | fi 9 | 10 | which montage 11 | if [[ "$?" != "0" ]] 12 | then 13 | echo "The application \"montage\" is necessary for assembling montage" 1>&2 14 | fi 15 | 16 | TMP=$PWD/imglist.$RANDOM.txt 17 | 18 | MOVIE_FILE=$1; shift 19 | MONTAGE_FILE=$1; shift 20 | 21 | echo "MOVIE_FILE: $MOVIE_FILE" 22 | echo "MONTAGE_FILE: $MONTAGE_FILE" 23 | 24 | echo "convert -delay 20 -loop 0 $* $MOVIE_FILE" 25 | convert -delay 40 -loop 0 $* $MOVIE_FILE 26 | 27 | echo "montage -label %f -frame 5 -background '#336699' -geometry +4+4 $* $MONTAGE_FILE" 28 | montage -label %f -frame 5 -background '#336699' -geometry +4+4 $* $MONTAGE_FILE 29 | 30 | if [ -d $HOME/public_html ] 31 | then 32 | cp $MOVIE_FILE $HOME/public_html 33 | cp $MONTAGE_FILE $HOME/public_html 34 | fi 35 | 36 | -------------------------------------------------------------------------------- /mandelbrot/bin/driver.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "mandelbrot.h" 8 | 9 | #define DEBUG 1 10 | 11 | #define error(format, ...) fprintf(stderr, format , __VA_ARGS__) 12 | #define debug(format, ...) if (DEBUG == 1) fprintf(stdout, format , __VA_ARGS__) 13 | 14 | #define PREDEF 1 15 | #define MASTER_SLAVE 2 16 | 17 | int X_MAX = 10000; 18 | int Y_MAX = 10000; 19 | 20 | int dump_mandelbrot(char *buffer, char *filename) 21 | { 22 | char ppm_to_jpg[1000]; 23 | FILE * fp = fopen(filename, "w"); 24 | fprintf(fp, "P6\n# CREATOR: Yadu Nand / mandel program\n"); 25 | fprintf(fp, "%d %d\n255\n",X_MAX,Y_MAX); 26 | fwrite(buffer, sizeof(char), 3*X_MAX*Y_MAX, fp); 27 | //snprintf(ppm_to_jpg, 1000, "convert %s -resize 1000x1000 %s", filename, filename); 28 | //system(ppm_to_jpg); 29 | fclose(fp); 30 | return 0; 31 | } 32 | 33 | 34 | int predef_run_server(int myrank, int nprocs, int strategy, int itermax, char* output_filename) 35 | { 36 | int i; 37 | int slice_size; 38 | int slice_start; 39 | int slice_end; 40 | int rcv_buf_size; 41 | MPI_Status stat; 42 | //double start, stop; 43 | char * buffer = malloc(X_MAX * Y_MAX * 3 * sizeof(char)); 44 | slice_size = Y_MAX / nprocs; 45 | //printf("Nprocs : %d", nprocs); 46 | //printf("slice_size : %d", slice_size); 47 | slice_start = slice_size * myrank; 48 | if (myrank == (nprocs - 1)){ 49 | slice_end = Y_MAX - 1; 50 | }else{ 51 | slice_end = slice_start + slice_size ; 52 | } 53 | mandelbrot(slice_start, slice_end, buffer, itermax); 54 | // Collate results from all ranks 55 | for ( i = 1; i < nprocs ; i++ ){ 56 | //printf("[%d] Starting message to proc: %d\n", 0, i); 57 | if (i == (nprocs - 1)){ 58 | rcv_buf_size = Y_MAX - (slice_size*i); 59 | }else{ 60 | rcv_buf_size = slice_size; 61 | } 62 | MPI_Recv(buffer+(X_MAX*3*slice_size*i), rcv_buf_size*3*X_MAX, MPI_CHAR, i, 0, 63 | MPI_COMM_WORLD, &stat); 64 | } 65 | 66 | dump_mandelbrot(buffer, output_filename); 67 | free(buffer); 68 | return 0; 69 | } 70 | 71 | int predef_run_client(int myrank, int nprocs, int strategy, int itermax) 72 | { 73 | int slice_size; 74 | int slice_start; 75 | int slice_end; 76 | //double start, stop; 77 | char * buffer = malloc(X_MAX * Y_MAX * 3 * sizeof(char)); 78 | slice_size = Y_MAX / nprocs; 79 | printf("Nprocs : %d", nprocs); 80 | printf("slice_size : %d", slice_size); 81 | slice_start = slice_size * myrank; 82 | if (myrank == (nprocs - 1)){ 83 | slice_end = Y_MAX - 1; 84 | }else{ 85 | slice_end = slice_start + slice_size ; 86 | } 87 | // Do the actual work per rank 88 | mandelbrot(slice_start, slice_end, buffer+(X_MAX*3*slice_start), itermax); 89 | 90 | // Send results to server to collate 91 | MPI_Send(buffer+(X_MAX*3*slice_start), (slice_end-slice_start)*3*X_MAX, MPI_CHAR, 0, 0, MPI_COMM_WORLD); 92 | free(buffer); 93 | return 0; 94 | } 95 | 96 | #define TASK_REQ 1 97 | 98 | int master(int myrank, int nprocs, int strategy, int itermax, char *output_filename) 99 | { 100 | int i, offset, slice_size; 101 | MPI_Status stat; 102 | 103 | int task_descriptor[4] = {0,0,0,0}; 104 | char * buffer = malloc(X_MAX * Y_MAX * 3 * sizeof(char)); 105 | //char * request = malloc(10*sizeof(char)); 106 | // Slices are in multiples on rows. 107 | slice_size = SLICE_SIZE; 108 | 109 | // In this world slaves ask for work! and there is work to do 110 | for ( i = 0 ; i < Y_MAX - SLICE_SIZE ; i+=SLICE_SIZE ) { 111 | MPI_Recv(task_descriptor, 4, MPI_INT, MPI_ANY_SOURCE, TASK_REQ, MPI_COMM_WORLD, &stat); 112 | task_descriptor[0] = 1; // There's work! 113 | task_descriptor[1] = i; 114 | task_descriptor[2] = i+SLICE_SIZE; 115 | MPI_Send(task_descriptor, 4, MPI_INT, stat.MPI_SOURCE, TASK_REQ, MPI_COMM_WORLD); 116 | } 117 | 118 | debug("%s", "[MASTER] ALL TASKS DISPATCHED\n"); 119 | // At this point all work has been dispatched, and results are to be collected 120 | // Tell workers that their work is done 121 | for ( i = 1 ; i < nprocs; i++ ) { 122 | debug("[MASTER] Sending die to %d \n", i); 123 | MPI_Recv(task_descriptor, 4, MPI_INT, i, TASK_REQ, MPI_COMM_WORLD, &stat); 124 | task_descriptor[0] = 0; // There's no more work! 125 | task_descriptor[1] = 0; 126 | task_descriptor[2] = 0; 127 | MPI_Send(task_descriptor, 4, MPI_INT, i, TASK_REQ, MPI_COMM_WORLD); 128 | } 129 | 130 | debug("%s", "[MASTER] ALL WORKERS ASKED TO RETURN RESULTS\n"); 131 | // At this point slaves know that the master expect results back 132 | // Master listens to results sent from slaves, who send one slice 133 | // at a time 134 | 135 | // Slaves send buffer with offset in the mpi_tag 136 | char *recv_buffer = malloc( sizeof(char)*3 * X_MAX * SLICE_SIZE); 137 | 138 | for ( i = 0 ; i < Y_MAX - SLICE_SIZE; i+=SLICE_SIZE ) { 139 | MPI_Recv(recv_buffer, X_MAX* 3 *SLICE_SIZE, 140 | MPI_CHAR, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &stat); 141 | offset = stat.MPI_TAG; 142 | //printf("Offset : %d\n", offset); 143 | memcpy(buffer+(X_MAX*3*offset), recv_buffer, X_MAX*3*SLICE_SIZE); 144 | } 145 | 146 | // At this point all results have been copied back to the Master 147 | dump_mandelbrot(buffer, output_filename); 148 | free(recv_buffer); 149 | free(buffer); 150 | return 0; 151 | } 152 | 153 | 154 | int slave(int myrank, int nprocs, int strategy, int itermax) 155 | { 156 | int slice_size; 157 | MPI_Status stat; 158 | int offset_queue[Y_MAX]; 159 | int offset_head = -1; 160 | 161 | int task_descriptor[4] = {0,0,0,0}; 162 | char * buffer = malloc(X_MAX * Y_MAX * 3 * sizeof(char)); 163 | //char * request = malloc(10*sizeof(char)); 164 | // Slices are in multiples on rows. 165 | slice_size = SLICE_SIZE; 166 | 167 | // Worker asks for work in a loop 168 | while (offset_head < Y_MAX) { 169 | MPI_Send(task_descriptor, 4, MPI_INT, 0, TASK_REQ, MPI_COMM_WORLD); 170 | MPI_Recv(task_descriptor, 4, MPI_INT, 0, TASK_REQ, MPI_COMM_WORLD, &stat); 171 | 172 | printf("[SLAVE:%d] TASK: %d START: %d END: %d \n", myrank, task_descriptor[0], 173 | task_descriptor[1], task_descriptor[2]); 174 | // No more work 175 | if ( task_descriptor[0] == 0 ) break; 176 | 177 | // There's work to do 178 | mandelbrot(task_descriptor[1], task_descriptor[2], 179 | buffer+(X_MAX*3*task_descriptor[1]), itermax); 180 | offset_head++; 181 | offset_queue[offset_head] = task_descriptor[1]; 182 | } 183 | 184 | // At this point all work has been completed. 185 | // Slaves will send results back 186 | for ( ; offset_head >= 0; offset_head-=1 ) { 187 | MPI_Send(buffer+(X_MAX*3*offset_queue[offset_head]), 188 | slice_size*3*X_MAX, MPI_CHAR, 0, 189 | offset_queue[offset_head], // HACK :) 190 | MPI_COMM_WORLD); 191 | } 192 | // At this point all results have been copied back to the Master 193 | free(buffer); 194 | return 0; 195 | } 196 | 197 | void usage(void){ 198 | printf("./mandelbrot -r -s -i -f <\n"); 199 | return; 200 | } 201 | 202 | 203 | int main(int argc, char **argv) 204 | { 205 | int nprocs; 206 | int myrank; 207 | int stat; 208 | int namelen; 209 | char ch; 210 | char output_filename[1000]; 211 | int itermax; 212 | double start, stop; 213 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 214 | int strategy = MASTER_SLAVE; 215 | 216 | while ((ch=getopt(argc,argv,"s:i:f:r:")) != -1){ 217 | switch(ch){ 218 | case 's' : strategy = atoi(optarg) ; 219 | break; 220 | case 'i' : itermax = atoi(optarg) ; 221 | break; 222 | case 'r' : X_MAX = atoi(optarg) ; Y_MAX = X_MAX; 223 | break; 224 | case 'f' : snprintf(output_filename, 1000, "%s", optarg); 225 | break; 226 | case '?' : printf(" Error : Unknown argument list" ); 227 | usage(); 228 | exit(1) ; 229 | } 230 | } 231 | 232 | debug("[DEBUG]: filename : %s\n", output_filename); 233 | debug("[DEBUG]: itermax : %d\n", itermax); 234 | debug("[DEBUG]: strategy : %d\n", strategy); 235 | 236 | MPI_Init(&argc, &argv); 237 | stat = MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 238 | if ( stat != 0 ) error ("MPI_Comm_size returned an error code : %d", stat); 239 | 240 | stat = MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 241 | if ( stat != 0 ) error ("MPI_Comm_rank returned an error code : %d", stat); 242 | 243 | MPI_Get_processor_name(processor_name, &namelen); 244 | 245 | printf("Process %d on %s out of %d\n", myrank, processor_name, nprocs); 246 | //printf("Nprocs: %d, Nrank: %d \n", nprocs, myrank); 247 | start = MPI_Wtime(); 248 | if (strategy == PREDEF){ 249 | if ( myrank == 0 ){ 250 | predef_run_server(myrank, nprocs, strategy, itermax, output_filename); 251 | }else{ 252 | predef_run_client(myrank, nprocs, strategy, itermax); 253 | } 254 | }else if(strategy == MASTER_SLAVE){ 255 | if ( myrank == 0 ){ 256 | master(myrank, nprocs, strategy, itermax, output_filename); 257 | }else{ 258 | slave(myrank, nprocs, strategy, itermax); 259 | } 260 | } 261 | stop = MPI_Wtime(); 262 | if (myrank == 0){ 263 | fprintf(stdout, "NPROCS: %d Time to completion: %f seconds\n", nprocs, stop-start); 264 | } 265 | MPI_Finalize(); 266 | return 0; 267 | } 268 | -------------------------------------------------------------------------------- /mandelbrot/bin/makefile.edison: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | 3 | CFLAGS="-I/opt/cray/mpt/default/gni/mpich2-gnu/49/include" 4 | LDFLAGS="-L/opt/cray/mpt/default/gni/mpich2-gnu/49/lib -lmpich" 5 | SRC = $(wildcard *.c) 6 | 7 | DEPS = mandelbrot.h 8 | OBJS = $(SRC:.c=.o) 9 | 10 | EXEC=mandelbrot 11 | 12 | %.o: %.c $(DEPS) 13 | $(CC) -c -o $@ $< $(CFLAGS) $(LDFLAGS) 14 | 15 | $(EXEC) : $(OBJS) 16 | $(CC) -o $@ $^ $(CFLAGS) $(LDFLAGS) 17 | 18 | clean: 19 | rm -f $(OBJS) $(EXEC) *~ *out *err 20 | -------------------------------------------------------------------------------- /mandelbrot/bin/mandelbrot.c: -------------------------------------------------------------------------------- 1 | /** mandel.c by Eric R. Weeks written 9-28-96 2 | ** weeks@physics.emory.edu 3 | ** http://www.physics.emory.edu/~weeks/ 4 | ** 5 | ** This program is public domain, but this header must be left intact 6 | ** and unchanged. 7 | ** 8 | ** to compile: cc -o mand mandel.c 9 | ** 10 | ** -Yadu Nand B 11 | ** This code has been borrowed from http://www.physics.emory.edu/faculty/weeks//software/mandel.c 12 | ** as a base for HW2 problem 2. Code is modified to run in a parallel environment. 13 | **/ 14 | 15 | #include "mandelbrot.h" 16 | 17 | extern int X_MAX; 18 | extern int Y_MAX; 19 | 20 | void color(char *buffer, int red, int green, int blue) { 21 | buffer[0] = (char)red ; 22 | buffer[1] = (char)green ; 23 | buffer[2] = (char)blue ; 24 | } 25 | 26 | 27 | int mandelbrot(int YSLICE_START, int YSLICE_END, char * BUFFER, int itermax) 28 | { 29 | double x,xx,y,cx,cy; 30 | int iteration,hx,hy; 31 | double magnify=1.0;/* no magnification*/ 32 | int hxres = X_MAX;/* horizonal resolution*/ 33 | int hyres = Y_MAX;/* vertical resolution*/ 34 | //printf("YSLICE [%d] -> [%d] \n", YSLICE_START, YSLICE_END); 35 | for (hy=YSLICE_START; hy<=YSLICE_END; hy++) { 36 | for (hx=1; hx<=hxres; hx++) { 37 | cx = (((float)hx)/((float)hxres)-0.5)/magnify*3.0-0.7; 38 | cy = (((float)hy)/((float)hyres)-0.5)/magnify*3.0; 39 | x = 0.0; y = 0.0; 40 | for (iteration=1;iteration100.0) break; 45 | } 46 | if (iteration == itermax){ 47 | color(BUFFER, 0,0,0); 48 | } else { 49 | color(BUFFER, 255,(iteration*10)%255,(iteration*10)%255); 50 | } 51 | BUFFER += 3; 52 | } 53 | } 54 | return 0; 55 | } 56 | -------------------------------------------------------------------------------- /mandelbrot/bin/mandelbrot.h: -------------------------------------------------------------------------------- 1 | #ifndef MANDELBROT_H 2 | #define MANDELBROT_H 3 | #include 4 | #include 5 | #include 6 | 7 | extern int X_MAX; 8 | extern int Y_MAX; 9 | 10 | #define SLICE_SIZE 100 11 | 12 | // Returns 0 if nothing fails 13 | // buffer point to a buffer with 3*(sizeof) char * Num yslices * 10000 14 | int mandelbrot(int YSLICE_START, int YSLICE_END, char *buffer, int intermax); 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /mandelbrot/bin/run_mandelbrot: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$MPI_RANKS" == "" ]] 4 | then 5 | echo "MPI_RANKS not defined" 1>&2 6 | exit -1 7 | fi 8 | 9 | outputfile="${@: -1}" 10 | 11 | if [[ $MACHINE == "raven" ]] 12 | then 13 | $MPI_APP_PATH 14 | elif [[ $MACHINE == "swan" ]] 15 | then 16 | echo "aprun -n $MPI_RANKS -N 1 $MPI_APP_PATH $*" 17 | echo "hostname : $HOSTNAME" 18 | aprun -n $MPI_RANKS -N 1 $MPI_APP_PATH $* 19 | touch $outputfile 20 | 21 | elif [[ $MACHINE == "stampede" ]] 22 | then 23 | echo "$MPI_APP_PATH $*" 24 | $MPI_APP_PATH $* 25 | 26 | elif [[ $MACHINE == "blacklight" ]] 27 | then 28 | export PATH=/usr/local/packages/ImageMagick-6.7.3-10/bin:$PATH 29 | echo "mpirun -np $PBS_NCPUS $MPI_APP_PATH $*" 30 | mpirun -np $PBS_NCPUS $MPI_APP_PATH $* 31 | 32 | elif [[ $MACHINE == "midway" ]] 33 | then 34 | module load openmpi 35 | mpirun -n $MPI_RANKS $MPI_APP_PATH $* 36 | else 37 | echo "MACHINE : $MACHINE is not defined" 38 | fi 39 | 40 | convert $outputfile -resize 1000x1000 $outputfile 41 | -------------------------------------------------------------------------------- /mandelbrot/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -rf run0* *~ output/* 4 | -------------------------------------------------------------------------------- /mandelbrot/ensemble_mandel.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file image, file out, file err) mandelbrot (file mandel_sh, int iterations, int resolution) 4 | { 5 | bash @mandel_sh "-i" iterations "-s 1 -r" resolution "-f" @image stdout=@out stderr=@err; 6 | } 7 | 8 | app (file movie, file montage, file out, file err) assemble (file[] mandel_imgs) 9 | { 10 | assemble @movie @montage @mandel_imgs stdout=@out stderr=@err; 11 | } 12 | 13 | int itermax = toInt(arg("niter", "20")); # number of iterations for mandelbrot 14 | int step = toInt(arg("step", "5")); # number of iterations for mandelbrot 15 | int resolution = toInt(arg("res", "10000")); # Resolution of result 16 | 17 | // 5 -> 100 iterations stepping by 5 18 | file mandel_img[] ; 19 | file mandel_out[] ; 20 | file mandel_err[] ; 21 | file mandel_sh <"./bin/run_mandelbrot">; 22 | 23 | foreach i in [5:itermax:step]{ 24 | tracef("i = %i \n", i); 25 | (mandel_img[i], mandel_out[i], mandel_err[i]) = mandelbrot(mandel_sh, i, resolution); 26 | } 27 | 28 | file movie <"output/mandel.gif">; 29 | file montage <"output/montage.jpg">; 30 | file assemble_out <"output/assemble.out">; 31 | file assemble_err <"output/assemble.err">; 32 | (movie, montage, assemble_out, assemble_err) = assemble (mandel_img); 33 | -------------------------------------------------------------------------------- /mandelbrot/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$HOSTNAME" == *stampede* ]] 4 | then 5 | echo "XSEDE/Stampede" 6 | module load mvapich2 7 | cd bin/ 8 | make clean; make 9 | cd .. 10 | 11 | elif [[ "$HOSTNAME" == *blacklight* ]] 12 | then 13 | echo "XSEDE/Blacklight" 14 | module load ImageMagick/6.7.3-10 15 | 16 | else 17 | SWIFT=$PWD/swift-trunk/bin 18 | export PATH=$SWIFT:$PWD/bin:$PATH 19 | module load openmpi 20 | module load ffmpeg 21 | cd bin; make; cd .. 22 | fi 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /mandelbrot/swift.conf: -------------------------------------------------------------------------------- 1 | sites: [blacklight-mpi, localhost] 2 | 3 | # We have the mandelbrot generation app defined for all computational 4 | # sites, and the assemble app defined only on localhost. This ensures 5 | # that the compute heavy mandelbrot code runs on the compute resource 6 | # and the IO heavy assemble operation is handled on the local machine. 7 | 8 | site.localhost { 9 | execution { 10 | type : "local" # Execution is local 11 | URL : "localhost" # Point at localhost to run locally 12 | } 13 | staging : direct # Files are on the same machine, so can be accessed "directly" 14 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Directory where work is done 15 | maxParallelTasks : 101 # Maximum number of parallel tasks 16 | initialParallelTasks: 100 # Maximum number of tasks at start 17 | app.assemble { 18 | executable: ${env.PWD}"/bin/assemble" 19 | } 20 | } 21 | 22 | 23 | # Instructions for Blacklight 24 | # 1. If you are running on the blacklight login nodes, set jobManager: "local:pbs" 25 | site.blacklight-mpi { 26 | execution { 27 | type : "coaster" # Use coasters to run on remote sites 28 | URL : "blacklight.psc.xsede.org" # Blacklight login URL 29 | jobManager: "local:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 30 | options { 31 | maxJobs : 1 # Max jobs submitted to LRM 32 | nodeGranularity : 1 # Nodes per job 33 | maxNodesPerJob : 1 # Nodes per job 34 | tasksPerNode : 1 # Tasks per Node 35 | maxJobTime : "00:25:00" # Time requested per job 36 | jobQueue : "debug" 37 | workerLoggingLevel : "DEBUG" 38 | workerLoggingDirectory : ${env.PWD}"/workerlogs/" 39 | jobOptions { 40 | ppn : "16" # Virtual processors per node per Job 41 | } 42 | } 43 | } 44 | staging : "direct" # Stage files from "local" system to Blacklight 45 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 46 | maxParallelTasks : 101 # Maximum number of parallel tasks 47 | initialParallelTasks: 100 # Maximum number of tasks at start 48 | app.bash { 49 | executable : "/bin/bash" 50 | env.MACHINE : "blacklight" 51 | env.MPI_APP_PATH: "/usr/users/8/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 52 | env.MPI_RANKS : "8" 53 | } 54 | } 55 | 56 | # Instructions for Blacklight 57 | # 1. If you are running on the blacklight login nodes, set jobManager: "local:pbs" 58 | site.blacklight-mpi-no-coaster { 59 | execution { 60 | type : "pbs" # Use coasters to run on remote sites 61 | URL : "blacklight.psc.xsede.org" # Blacklight login URL 62 | jobManager: "local:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 63 | } 64 | filesystem { 65 | type: "local" 66 | URL: "localhost" 67 | } 68 | staging : "swift" 69 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 70 | maxParallelTasks : 101 # Maximum number of parallel tasks 71 | initialParallelTasks: 100 # Maximum number of tasks at start 72 | app.bash { 73 | executable : "/bin/bash" 74 | env.MACHINE : "blacklight" 75 | env.MPI_APP_PATH: "/usr/users/8/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 76 | env.MPI_RANKS : "8" 77 | options{ 78 | jobQueue : "debug" 79 | maxWallTime : "00:25:00" # Time requested per job 80 | #count : 2 81 | #ppn : 16 82 | } 83 | } 84 | } 85 | 86 | site.stampede-mpi { 87 | execution { 88 | type : "slurm" # Use coasters to run on remote sites 89 | URL : "stampede.tacc.utexas.edu" # Stampede login nodes login[1..4].stampede.tacc.utexas.edu 90 | jobManager: "local:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 91 | options { 92 | maxJobs : 1 # Max jobs submitted to LRM 93 | maxJobTime : "00:25:00" # Time requested per job 94 | } 95 | } 96 | filesystem { 97 | type: "local" 98 | URL: "localhost" 99 | } 100 | staging : "swift" # Staging through shared-fs 101 | workDirectory : ${env.HOME}"/swiftwork" # Location for intermediate files 102 | maxParallelTasks : 1 # Maximum number of parallel tasks 103 | initialParallelTasks: 1 # Maximum number of tasks at start 104 | app.bash { 105 | executable : "/bin/bash" 106 | env.MACHINE : "stampede" 107 | env.MPI_APP_PATH: "/home1/02551/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 108 | env.MPI_RANKS : "8" 109 | options{ 110 | jobQueue : "development" 111 | maxWallTime : "00:25:00" # Time requested per job 112 | #count : 2 113 | } 114 | } 115 | } 116 | 117 | 118 | site.stampede { 119 | execution { 120 | type : "coaster" # Use coasters to run on remote sites 121 | URL : "stampede.tacc.utexas.edu" # Stampede login nodes login[1..4].stampede.tacc.utexas.edu 122 | jobManager: "local:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 123 | options { 124 | maxJobs : 1 # Max jobs submitted to LRM 125 | nodeGranularity : 1 # Nodes per job 126 | maxNodesPerJob : 1 # Nodes per job 127 | #tasksPerNode : 8 # Tasks per Node 128 | #jobOptions{ 129 | # "tasks" : "8" 130 | #} 131 | jobQueue : "development" 132 | maxJobTime : "00:25:00" # Time requested per job 133 | } 134 | } 135 | staging : "local" # Stage files from "local" system to Midway 136 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 137 | maxParallelTasks : 101 # Maximum number of parallel tasks 138 | initialParallelTasks: 100 # Maximum number of tasks at start 139 | app.bash { 140 | executable: "/bin/bash" 141 | env.MACHINE: "stampede" 142 | env.MPI_APP_PATH: "/home1/02551/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 143 | env.MPI_RANKS: "8" 144 | } 145 | } 146 | 147 | # Instructions for Swan 148 | # 1. 149 | site.swan { 150 | execution { 151 | type: "coaster" 152 | URL: "swan.cray.com" 153 | jobManager: "ssh-cl:pbs" 154 | options { 155 | maxJobs: 10 156 | nodeGranularity: 1 157 | maxNodesPerJob: 1 158 | tasksPerNode: 8 159 | } 160 | } 161 | staging: local 162 | workDirectory: "/lus/scratch/p01953/swiftwork" 163 | maxParallelTasks: 101 164 | initialParallelTasks: 100 165 | app.bash { 166 | executable: "/bin/bash" 167 | env.MACHINE: "swan" 168 | env.MPI_APP_PATH: "/home/users/p01953/swift-tutorial/mandelbrot/bin/mandelbrot" 169 | env.MPI_RANKS: "1" 170 | } 171 | } 172 | 173 | site.midway-westmere { 174 | execution { 175 | type : "coaster" # Use coasters to run on remote sites 176 | URL : "swift.rcc.uchicago.edu" # Midway login node | Alternatively use midway.rcc.uchicago.edu 177 | jobManager: "local:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 178 | options { 179 | maxJobs : 10 # Max jobs submitted to LRM 180 | nodeGranularity : 1 # Nodes per job 181 | maxNodesPerJob : 1 # Nodes per job 182 | tasksPerNode : 8 # Tasks per Node 183 | jobQueue : "westmere" # Select queue from (sandyb, westmere, ...) 184 | maxJobTime : "00:25:00" # Time requested per job 185 | } 186 | } 187 | staging : "direct" # Stage files from "local" system to Midway 188 | workDirectory : "/tmp/"${env.USER} # Location for intermediate files 189 | maxParallelTasks : 101 # Maximum number of parallel tasks 190 | initialParallelTasks: 100 # Maximum number of tasks at start 191 | app.bash { 192 | executable: "/bin/bash" 193 | env.MACHINE: "midway" 194 | env.MPI_APP_PATH: "/scratch/midway/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 195 | env.MPI_RANKS: "12" 196 | } 197 | } 198 | 199 | # Instructions for Midway: 200 | # 1. If you are running on the midway login nodes set jobManager: "local:slurm" 201 | # 2. Set workDirectory to /tmp/your_username_on_midway 202 | site.midway-sandyb { 203 | execution { 204 | type : "coaster" # Use coasters to run on remote sites 205 | URL : "swift.rcc.uchicago.edu" # Midway login node | Alternatively use midway.rcc.uchicago.edu 206 | jobManager: "ssh-cl:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 207 | options { 208 | maxJobs : 10 # Max jobs submitted to LRM 209 | nodeGranularity : 1 # Nodes per job 210 | maxNodesPerJob : 1 # Nodes per job 211 | tasksPerNode : 1 # Tasks per Node 212 | jobQueue : "sandyb" # Select queue from (sandyb, westmere, ...) 213 | maxJobTime : "00:25:00" # Time requested per job 214 | } 215 | } 216 | staging : "direct" # Stage files from "local" system to Midway 217 | workDirectory : "/tmp/"${env.USER} # Location for intermediate files 218 | maxParallelTasks : 101 # Maximum number of parallel tasks 219 | initialParallelTasks: 100 # Maximum number of tasks at start 220 | app.bash { 221 | executable: "/bin/bash" 222 | env.MACHINE: "midway" 223 | env.MPI_APP_PATH: "/scratch/midway/yadunand/swift-tutorial/mandelbrot/bin/mandelbrot" 224 | env.MPI_RANKS: "12" 225 | } 226 | } 227 | 228 | TCPPortRange: "50000,51000" # TCP port range used by swift to communicate with remote sites 229 | lazyErrors: false # Swift fails immediately upon encountering an error 230 | executionRetries: 0 # Set number of retries upon task failures 231 | keepSiteDir: true # Keep Site Dir (useful for debug) 232 | providerStagingPinSwiftFiles: true # Pin staging files (useful for debug) 233 | alwaysTransferWrapperLog: true # Transfer wrapper logs (useful for debug) 234 | -------------------------------------------------------------------------------- /mpi/Makefile: -------------------------------------------------------------------------------- 1 | CC=mpicc 2 | 3 | CFLAGS = -Wall -g 4 | 5 | SRC = $(wildcard *.c) 6 | 7 | DEPS = timer.h 8 | OBJS = $(SRC:.c=.o) 9 | 10 | EXEC=mpi_hello 11 | 12 | %.o: %.c $(DEPS) 13 | $(CC) -c -o $@ $< $(CFLAGS) 14 | 15 | $(EXEC) : $(OBJS) 16 | $(CC) -o $@ $^ $(CFLAGS) 17 | 18 | clean: 19 | rm -f $(OBJS) $(EXEC) *~ 20 | -------------------------------------------------------------------------------- /mpi/buildcray.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gcc -I/opt/cray/mpt/default/gni/mpich2-gnu/49/include -L/opt/cray/mpt/default/gni/mpich2-gnu/49/lib -lmpich pi.c -o pi 4 | -------------------------------------------------------------------------------- /mpi/do_mpi.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file out, file err) do_mpi () 4 | { 5 | mpi_app stderr=filename(err) stdout=filename(out); 6 | } 7 | 8 | int nsim = toInt(arg("nsim", "10")); 9 | 10 | foreach i in [0:nsim-1] { 11 | file mpiout ; 12 | file mpierr ; 13 | (mpiout, mpierr) = do_mpi(); 14 | } 15 | -------------------------------------------------------------------------------- /mpi/mock.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "hi" 3 | module load openmpi 4 | mpirun -n 32 /scratch/midway/yadunand/swift-tutorial/mpi2/mpi_sum 5 | 6 | -------------------------------------------------------------------------------- /mpi/mpi_hello.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #define error(format, ...) fprintf(stderr, format , __VA_ARGS__) 7 | 8 | int main(int argc, char **argv) 9 | { 10 | int nprocs; 11 | int myrank; 12 | int stat; 13 | int namelen; 14 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 15 | 16 | MPI_Init(&argc, &argv); 17 | stat = MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 18 | if ( stat != 0 ) error ("MPI_Comm_size returned an error code : %d", stat); 19 | 20 | stat = MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 21 | if ( stat != 0 ) error ("MPI_Comm_rank returned an error code : %d", stat); 22 | 23 | MPI_Get_processor_name(processor_name, &namelen); 24 | 25 | fprintf(stderr, "Process %d on %s out of %d\n", myrank, processor_name, nprocs); 26 | fprintf(stdout, "[Rank:%d]Hello World!", myrank); 27 | 28 | MPI_Finalize(); 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /mpi/pi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | int main(int argc, char** argv) 6 | { 7 | int n, myid, numprocs, i; 8 | double PI25DT = 3.141592653589793238462643; 9 | double mypi, pi, h, sum, x; 10 | 11 | MPI_Init(&argc, &argv); 12 | MPI_Comm_size(MPI_COMM_WORLD, &numprocs); 13 | MPI_Comm_rank(MPI_COMM_WORLD, &myid); 14 | 15 | /* Number of intervals */ 16 | n = 25; 17 | 18 | MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD); 19 | h = 1.0 / (double) n; 20 | sum = 0.0; 21 | for ( i = myid + 1; i <= n; i += numprocs ) 22 | { 23 | x = h * ((double) i - 0.5); 24 | sum += ( 4.0 / (1.0 + x*x) ); 25 | } 26 | mypi = h * sum; 27 | MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 28 | 29 | if (myid == 0) 30 | { 31 | printf("pi = %.16f, Error is %.16f\n", 32 | pi, fabs(pi - PI25DT)); 33 | } 34 | 35 | MPI_Finalize(); 36 | return 0; 37 | } 38 | 39 | -------------------------------------------------------------------------------- /mpi/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | module swap PrgEnv-cray PrgEnv-gnu 5 | alias em='emacs' 6 | #export MPICH_CPUMASK_DISPLAY -------------------------------------------------------------------------------- /mpi/submit.sbatch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --job-name=mpi_hello 3 | #SBATCH --output=output%N.%j.out 4 | #SBATCH --error=error%N.%j.err 5 | #SBATCH --partition=development 6 | #SBATCH --time=00:10:00 7 | #SBATCH --nodes=1 8 | #SBATCH --ntasks-per-node=12 9 | #module load openmpi 10 | mpirun ./mpi_hello 11 | echo "Done " 12 | -------------------------------------------------------------------------------- /mpi/submit_scripts/pbs.submit: -------------------------------------------------------------------------------- 1 | #PBS -N mpi-test 2 | #PBS -m n 3 | #PBS -l mppwidth=1,mppnppn=1,mppdepth=4 4 | #PBS -l walltime=00:24:00 5 | #PBS -l advres=yadunandb.2763 6 | #PBS -q development 7 | #PBS -o /lustre/beagle2/yadunandb/swift-tutorial/mpi/output/mpu_sum.stdout 8 | #PBS -e /lustre/beagle2/yadunandb/swift-tutorial/mpi/output/mpu_sum.stderr 9 | export WORKER_LOGGING_LEVEL=NONE 10 | 11 | cd /lustre/beagle2/yadunandb/swift-tutorial/mpi 12 | aprun -n 24 /lustre/beagle2/yadunandb/swift-tutorial/mpi/mpi_sum 13 | 14 | -------------------------------------------------------------------------------- /mpi/swift.conf: -------------------------------------------------------------------------------- 1 | site.midway { 2 | execution { 3 | type: "coaster" 4 | URL: "swift.rcc.uchicago.edu" 5 | jobManager: "local:slurm" 6 | options { 7 | maxJobs: 1 8 | maxNodesPerJob: 2 9 | nodeGranularity: 2 10 | #tasksPerNode: 1 11 | jobQueue: "sandyb" 12 | jobProject: mpcs51087 13 | maxJobTime : "00:08:20" 14 | } 15 | } 16 | staging : "direct" 17 | workDirectory : "/scratch/midway/"${env.USER}"/swiftwork" 18 | maxParallelTasks : 65 19 | initialParallelTasks : 64 20 | app.mpi_app { 21 | executable: ${env.PWD}"/mock.sh" 22 | maxWallTime: "00:05:00" 23 | options{ 24 | jobType: single 25 | hostCount: "2" 26 | } 27 | } 28 | } 29 | 30 | sites: midway 31 | 32 | TCPPortRange: "50000,51000" 33 | lazyErrors: false 34 | executionRetries: 0 35 | keepSiteDir: true 36 | providerStagingPinSwiftFiles: false 37 | alwaysTransferWrapperLog: true 38 | 39 | -------------------------------------------------------------------------------- /nersc.conf: -------------------------------------------------------------------------------- 1 | sites: edison 2 | # sites can be set to a comma separated list of remote sites you have access to 3 | # eg.: sites: [beagle] or sites: [beagle, midway] 4 | # The sites definition in the config can be overridden by specifying -sites on the 5 | # swift command line as follows : swift -sites beagle p4.swift 6 | 7 | ############################# WARNING! ####################################### 8 | # NERSC is a government scientific computing center 9 | # Do *NOT* attempt to run on NERSC machines if you do not have access to them. 10 | ############################################################################## 11 | 12 | # Instructions for Edison 13 | # 1. If you are running remotely on edison , set jobManager: "ssh-cl:pbs" 14 | # Ensure that you have passwordless ssh access to edison from your machine. 15 | site.edison { 16 | execution { 17 | type : "coaster" # Use coasters to run on remote sites 18 | URL : "edison.nersc.gov" # Edison login URL 19 | jobManager: "local:slurm" # Connect locally to slurm Local Resource manager(LRM) 20 | options { 21 | maxJobs : 1 # Max jobs submitted to LRM 22 | nodeGranularity : 1 # Integer multiple of this will be requested per job 23 | maxNodesPerJob : 1 # Max Nodes that can be requested per job 24 | tasksPerNode : 24 # Tasks per Node, This is not used here 25 | maxJobTime : "00:25:00" # Time requested per job 26 | jobQueue : "debug" 27 | } 28 | } 29 | staging : "local" # Stage files from "local" system to Edison 30 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 31 | maxParallelTasks : 101 # Maximum number of parallel tasks 32 | initialParallelTasks: 100 # Maximum number of tasks at start 33 | app.ALL { executable: "*" } # All tasks to be found from commandline 34 | } 35 | 36 | # Instructions for Edison 37 | # 1. If you are running remotely on edison , set jobManager: "ssh-cl:pbs" 38 | # Ensure that you have passwordless ssh access to edison from your machine. 39 | site.edison_multinode { 40 | execution { 41 | type : "coaster" # Use coasters to run on remote sites 42 | URL : "edison.nersc.gov" # Edison login URL 43 | jobManager: "local:slurm" # Connect locally to slurm Local Resource manager(LRM) 44 | options { 45 | maxJobs : 1 # Max jobs submitted to LRM 46 | nodeGranularity : 2 # Nodes per job 47 | maxNodesPerJob : 2 # Nodes per job 48 | maxJobTime : "00:25:00" # Time requested per job 49 | jobQueue : "debug" 50 | jobOptions { 51 | jobType : "single" # Submit jobs via aprun mechanism 52 | #depth : "24" # 4 cores per task 53 | } 54 | } 55 | } 56 | staging : "local" # Stage files from "local" system to Edison 57 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 58 | maxParallelTasks : 101 # Maximum number of parallel tasks 59 | initialParallelTasks: 100 # Maximum number of tasks at start 60 | app.ALL { executable: "*" } # All tasks to be found from commandline 61 | } 62 | 63 | 64 | 65 | # Instructions for Hopper 66 | # 1. If you are running on the hopper login nodes, set jobManager: "local:pbs" 67 | # 2. Find your project name/allocation and set jobProject : "YOUR_PROJECT_ON_HOPPER" 68 | # 3. Set userHomeOverride : "/lustre/hopper2/YOUR_USERNAME_ON_HOPPER/swiftwork" 69 | # 4. Set workDirectory : "/tmp/YOUR_USERNAME_ON_HOPPER/swiftwork" 70 | site.hopper { 71 | execution { 72 | type : "coaster" # Use coasters to run on remote sites 73 | URL : "hopper.nersc.gov" # Hopper login URL 74 | jobManager: "local:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 75 | options { 76 | maxJobs : 1 # Max jobs submitted to LRM 77 | nodeGranularity : 1 # Nodes per job 78 | maxNodesPerJob : 1 # Nodes per job 79 | tasksPerNode : 4 # Tasks per Node 80 | maxJobTime : "00:25:00" # Time requested per job 81 | workerLoggingLevel : DEBUG 82 | workerLoggingDirectory : ${env.PWD}/"workerlogs" 83 | jobOptions { 84 | pbs.aprun: true # Submit jobs via aprun mechanism 85 | pbs.mpp : true # Mpp enabled 86 | depth : "4" # 4 cores per task 87 | } 88 | } 89 | } 90 | staging : "local" # Stage files from "local" system to Hopper 91 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 92 | maxParallelTasks : 101 # Maximum number of parallel tasks 93 | initialParallelTasks: 100 # Maximum number of tasks at start 94 | app.ALL { executable: "*" } # All tasks to be found from commandline 95 | } 96 | 97 | 98 | 99 | 100 | TCPPortRange: "50000,51000" # TCP port range used by swift to communicate with remote sites 101 | lazyErrors: false # Swift fails immediately upon encountering an error 102 | executionRetries: 0 # Set number of retries upon task failures 103 | keepSiteDir: true # Keep Site Dir (useful for debug) 104 | providerStagingPinSwiftFiles: false # Pin staging files (useful for debug) 105 | alwaysTransferWrapperLog: true # Transfer wrapper logs (useful for debug) 106 | -------------------------------------------------------------------------------- /part01/p1.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file o) simulation () 4 | { 5 | simulate stdout=filename(o); 6 | } 7 | 8 | file f <"sim.out">; 9 | f = simulation(); 10 | -------------------------------------------------------------------------------- /part01/swift.conf: -------------------------------------------------------------------------------- 1 | ../local.conf -------------------------------------------------------------------------------- /part02/p2.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file o) simulation () 4 | { 5 | simulate stdout=filename(o); 6 | } 7 | 8 | foreach i in [0:9] { 9 | file f ; 10 | f = simulation(); 11 | } 12 | -------------------------------------------------------------------------------- /part02/swift.conf: -------------------------------------------------------------------------------- 1 | ../local.conf -------------------------------------------------------------------------------- /part03/p3.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file o) simulation (int sim_steps, int sim_range, int sim_values) 4 | { 5 | simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=filename(o); 6 | } 7 | 8 | app (file o) analyze (file s[]) 9 | { 10 | stats filenames(s) stdout=filename(o); 11 | } 12 | 13 | int nsim = toInt(arg("nsim","10")); 14 | int steps = toInt(arg("steps","1")); 15 | int range = toInt(arg("range","100")); 16 | int values = toInt(arg("values","5")); 17 | 18 | file sims[]; 19 | 20 | foreach i in [0:nsim-1] { 21 | file simout ; 22 | simout = simulation(steps,range,values); 23 | sims[i] = simout; 24 | } 25 | 26 | file stats<"output/average.out">; 27 | stats = analyze(sims); 28 | -------------------------------------------------------------------------------- /part03/swift.conf: -------------------------------------------------------------------------------- 1 | ../local.conf -------------------------------------------------------------------------------- /part04/p4.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file out) sortdata (file unsorted) 4 | { 5 | sort "-n" filename(unsorted) stdout=filename(out); 6 | } 7 | 8 | file unsorted <"unsorted.txt">; 9 | file sorted <"sorted.txt">; 10 | 11 | sorted = sortdata(unsorted); 12 | -------------------------------------------------------------------------------- /part04/swift.conf: -------------------------------------------------------------------------------- 1 | ../swift.conf -------------------------------------------------------------------------------- /part04/unsorted.txt: -------------------------------------------------------------------------------- 1 | 7 2 | 49 3 | 73 4 | 58 5 | 30 6 | 72 7 | 44 8 | 78 9 | 23 10 | 9 11 | 40 12 | 65 13 | 92 14 | 42 15 | 87 16 | 3 17 | 27 18 | 29 19 | 12 20 | 69 21 | 57 22 | 60 23 | 33 24 | 99 25 | 16 26 | 35 27 | 97 28 | 26 29 | 67 30 | 10 31 | 79 32 | 21 33 | 93 34 | 36 35 | 85 36 | 45 37 | 28 38 | 91 39 | 94 40 | 1 41 | 53 42 | 8 43 | 68 44 | 90 45 | 24 46 | 96 47 | 22 48 | 66 49 | 77 50 | 98 51 | 81 52 | 13 53 | 14 54 | 63 55 | 25 56 | 15 57 | 17 58 | 95 59 | 5 60 | 4 61 | 51 62 | 88 63 | 82 64 | 52 65 | 37 66 | 38 67 | 71 68 | 31 69 | 75 70 | 6 71 | 62 72 | 19 73 | 54 74 | 89 75 | 70 76 | 20 77 | 34 78 | 50 79 | 59 80 | 47 81 | 39 82 | 11 83 | 18 84 | 61 85 | 76 86 | 74 87 | 56 88 | 84 89 | 55 90 | 80 91 | 2 92 | 0 93 | 64 94 | 48 95 | 83 96 | 46 97 | 43 98 | 32 99 | 41 100 | 86 101 | -------------------------------------------------------------------------------- /part04/xsede.conf: -------------------------------------------------------------------------------- 1 | ../xsede.conf -------------------------------------------------------------------------------- /part05/p5.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values, file sim_script) 4 | { 5 | bash @sim_script "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log; 6 | } 7 | 8 | app (file out, file log) analyze (file s[], file stat_script) 9 | { 10 | bash @stat_script filenames(s) stdout=@out stderr=@log; 11 | } 12 | 13 | int nsim = toInt(arg("nsim", "10")); 14 | int steps = toInt(arg("steps", "1")); 15 | int range = toInt(arg("range", "100")); 16 | int values = toInt(arg("values", "5")); 17 | 18 | file sims[]; 19 | file simulate_script <"simulate.sh">; 20 | file stats_script <"stats.sh">; 21 | 22 | foreach i in [0:nsim-1] { 23 | file simout ; 24 | file simlog ; 25 | (simout,simlog) = simulation(steps,range,values,simulate_script); 26 | sims[i] = simout; 27 | } 28 | 29 | file stats_out<"output/average.out">; 30 | file stats_log<"output/average.log">; 31 | (stats_out, stats_log) = analyze(sims,stats_script); 32 | -------------------------------------------------------------------------------- /part05/simulate.sh: -------------------------------------------------------------------------------- 1 | ../app/simulate.sh -------------------------------------------------------------------------------- /part05/stats.sh: -------------------------------------------------------------------------------- 1 | ../app/stats.sh -------------------------------------------------------------------------------- /part05/swift.conf: -------------------------------------------------------------------------------- 1 | ../swift.conf -------------------------------------------------------------------------------- /part05/xsede.conf: -------------------------------------------------------------------------------- 1 | ../xsede.conf -------------------------------------------------------------------------------- /part06/p6.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | # app() functions for application programs to be called: 4 | 5 | app (file out) genseed (int nseeds, file seed_script) 6 | { 7 | bash @seed_script "-r" 2000000 "-n" nseeds stdout=@out; 8 | } 9 | 10 | app (file out) genbias (int bias_range, int nvalues, file bias_script) 11 | { 12 | bash @bias_script "-r" bias_range "-n" nvalues stdout=@out; 13 | } 14 | 15 | app (file out, file log) simulation (int timesteps, int sim_range, 16 | file bias_file, int scale, int sim_count, 17 | file sim_script, file seed_file) 18 | { 19 | bash @sim_script "-t" timesteps "-r" sim_range "-B" @bias_file "-x" scale 20 | "-n" sim_count "-S" @seed_file stdout=@out stderr=@log; 21 | } 22 | 23 | app (file out, file log) analyze (file s[], file stat_script) 24 | { 25 | bash @stat_script filenames(s) stdout=@out stderr=@log; 26 | } 27 | 28 | # Command line arguments 29 | 30 | int nsim = toInt(arg("nsim", "10")); # number of simulation programs to run 31 | int steps = toInt(arg("steps", "1")); # number of timesteps (seconds) per simulation 32 | int range = toInt(arg("range", "100")); # range of the generated random numbers 33 | int values = toInt(arg("values", "10")); # number of values generated per simulation 34 | 35 | # Main script and data 36 | 37 | file simulate_script <"simulate.sh">; 38 | file stats_script <"stats.sh">; 39 | file seedfile <"output/seed.dat">; # Dynamically generated bias for simulation ensemble 40 | 41 | tracef("\n*** Script parameters: nsim=%i range=%i num values=%i\n\n", nsim, range, values); 42 | seedfile = genseed(1,simulate_script); 43 | 44 | file sims[]; # Array of files to hold each simulation output 45 | 46 | foreach i in [0:nsim-1] { 47 | file biasfile ; 48 | file simout ; 49 | file simlog ; 50 | biasfile = genbias(1000, 20, simulate_script); 51 | (simout,simlog) = simulation(steps, range, biasfile, 1000000, values, simulate_script, seedfile); 52 | sims[i] = simout; 53 | } 54 | 55 | file stats_out<"output/average.out">; 56 | file stats_log<"output/average.log">; 57 | (stats_out,stats_log) = analyze(sims, stats_script); 58 | -------------------------------------------------------------------------------- /part06/simulate.sh: -------------------------------------------------------------------------------- 1 | ../app/simulate.sh -------------------------------------------------------------------------------- /part06/stats.sh: -------------------------------------------------------------------------------- 1 | ../app/stats.sh -------------------------------------------------------------------------------- /part06/swift.conf: -------------------------------------------------------------------------------- 1 | ../swift.conf -------------------------------------------------------------------------------- /part06/xsede.conf: -------------------------------------------------------------------------------- 1 | ../xsede.conf -------------------------------------------------------------------------------- /part07/Makefile: -------------------------------------------------------------------------------- 1 | CC?=mpicc 2 | 3 | CFLAGS = -Wall -g 4 | 5 | SRC = $(wildcard *.c) 6 | 7 | DEPS = 8 | OBJS = $(SRC:.c=.o) 9 | 10 | EXEC=mpi_hello 11 | 12 | %.o: %.c $(DEPS) 13 | $(CC) -c -o $@ $< $(CFLAGS) 14 | 15 | $(EXEC) : $(OBJS) 16 | $(CC) -o $@ $^ $(CFLAGS) 17 | 18 | clean: 19 | rm -f $(OBJS) $(EXEC) *~ 20 | -------------------------------------------------------------------------------- /part07/mpi_hello.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #define error(format, ...) fprintf(stderr, format , __VA_ARGS__) 8 | 9 | int main(int argc, char **argv) 10 | { 11 | int nprocs; 12 | int myrank; 13 | int stat; 14 | int namelen; 15 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 16 | 17 | MPI_Init(&argc, &argv); 18 | if(argc==2) { 19 | sleep(atoi(argv[1])); 20 | } 21 | 22 | stat = MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 23 | if ( stat != 0 ) error ("MPI_Comm_size returned an error code : %d", stat); 24 | 25 | stat = MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 26 | if ( stat != 0 ) error ("MPI_Comm_rank returned an error code : %d", stat); 27 | 28 | MPI_Get_processor_name(processor_name, &namelen); 29 | 30 | fprintf(stderr, "Process %d on %s out of %d\n", myrank, processor_name, nprocs); 31 | fprintf(stdout, "[Rank:%d]Hello World!\n", myrank); 32 | 33 | MPI_Finalize(); 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /part07/nersc.conf: -------------------------------------------------------------------------------- 1 | ../nersc.conf -------------------------------------------------------------------------------- /part07/p7.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | string curdir = java("java.lang.System","getProperty","user.dir"); 4 | 5 | app (file out, file err) mpi_hello (int time, int nproc) 6 | { 7 | mpiwrap nproc mpiapp time stdout=@out stderr=@err; 8 | } 9 | 10 | int nsim = toInt(arg("nsim", "10")); 11 | int time = toInt(arg("time", "1")); 12 | int nproc = toInt(arg("nproc", "56")); 13 | 14 | global string mpiapp = arg("mpiapp", curdir+"/mpi_hello"); 15 | 16 | foreach i in [0:nsim-1] { 17 | file mpiout ; 18 | file mpierr ; 19 | (mpiout, mpierr) = mpi_hello(time, nproc); 20 | } 21 | -------------------------------------------------------------------------------- /part07/swift.conf: -------------------------------------------------------------------------------- 1 | ../swift.conf -------------------------------------------------------------------------------- /part08/Makefile: -------------------------------------------------------------------------------- 1 | CC?=mpicc 2 | 3 | CFLAGS = -Wall -g 4 | 5 | SRC = $(wildcard *.c) 6 | 7 | DEPS = 8 | OBJS = $(SRC:.c=.o) 9 | 10 | EXEC = hipi 11 | 12 | %.o: %.c $(DEPS) 13 | $(CC) -c -o $@ $< $(CFLAGS) 14 | 15 | $(EXEC) : $(OBJS) 16 | $(CC) -o $@ $^ $(CFLAGS) 17 | 18 | clean: 19 | rm -f $(OBJS) $(EXEC) *~ 20 | -------------------------------------------------------------------------------- /part08/hipi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | int nintervals; 11 | 12 | void pi() 13 | { 14 | int myid, numprocs, i, n; 15 | double PI25DT = 3.141592653589793238462643; 16 | double mypi, pi, h, sum, x; 17 | 18 | MPI_Comm_size(MPI_COMM_WORLD,&numprocs); 19 | MPI_Comm_rank(MPI_COMM_WORLD,&myid); 20 | 21 | if (myid == 0) n = nintervals; 22 | MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD); 23 | 24 | h = 1.0 / (double) n; 25 | sum = 0.0; 26 | for (i = myid + 1; i <= n; i += numprocs) { 27 | x = h * ((double)i - 0.5); 28 | sum += 4.0 / (1.0 + x*x); 29 | } 30 | mypi = h * sum; 31 | 32 | MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, 33 | MPI_COMM_WORLD); 34 | 35 | if (myid == 0) printf("Pi: %.16f Error: %.16f\n", 36 | pi, fabs(pi - PI25DT)); 37 | } 38 | 39 | #define error(format, ...) fprintf(stderr, format , __VA_ARGS__) 40 | 41 | int main(int argc, char **argv) 42 | { 43 | int nprocs; 44 | int myrank; 45 | int stat; 46 | int namelen; 47 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 48 | double t1; 49 | 50 | MPI_Init(&argc, &argv); 51 | stat = MPI_Comm_size(MPI_COMM_WORLD, &nprocs); 52 | if ( stat != 0 ) error ("MPI_Comm_size returned an error code : %d", stat); 53 | 54 | stat = MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 55 | if ( stat != 0 ) error ("MPI_Comm_rank returned an error code : %d", stat); 56 | 57 | if (myrank == 0) { 58 | t1 = MPI_Wtime(); 59 | if(argc == 3) { 60 | nintervals = atoi(argv[1]); 61 | sleep(atoi(argv[2])); 62 | printf("Args: argc=%d n=%d sleep=%d\n", argc, atoi(argv[1]), atoi(argv[2])); 63 | } 64 | else { 65 | fprintf(stderr,"usage: hipi nintervals sleeptime\n"); 66 | MPI_Abort(MPI_COMM_WORLD,1); 67 | } 68 | } 69 | 70 | MPI_Get_processor_name(processor_name, &namelen); 71 | 72 | printf("Process %d out of %d on host %s\n", myrank, nprocs, processor_name); 73 | 74 | pi(); 75 | 76 | if (myrank == 0) printf("Elapsed time: %f\n", MPI_Wtime() - t1); 77 | 78 | MPI_Finalize(); 79 | return 0; 80 | } 81 | 82 | // Time functions for later use (needs debugging, causes pi to return NaN). 83 | 84 | #define TIMEBUFLEN 26 85 | 86 | char buffer[TIMEBUFLEN]; 87 | int millisec; 88 | struct tm* tm_info; 89 | struct timeval tv; 90 | 91 | void prtime_msec() { 92 | 93 | fprintf(stdout, "Time: 12:34:56\n"); 94 | gettimeofday(&tv, NULL); 95 | return; 96 | 97 | millisec = lrint(tv.tv_usec/1000.0); // Round to nearest millisec 98 | if (millisec>=1000) { // Allow for rounding up to nearest second 99 | millisec -=1000; 100 | tv.tv_sec++; 101 | } 102 | 103 | tm_info = localtime(&(tv.tv_sec)); 104 | 105 | strftime(buffer, TIMEBUFLEN, "%Y:%m:%d %H:%M:%S", tm_info); 106 | fprintf(stdout, "Time: %s.%03d\n", buffer, millisec); 107 | } 108 | 109 | void prtime() 110 | { 111 | time_t timer; 112 | char buffer[26]; 113 | struct tm* tm_info; 114 | 115 | time(&timer); 116 | tm_info = localtime(&timer); 117 | 118 | strftime(buffer, 26, "%Y:%m:%d %H:%M:%S", tm_info); 119 | puts(buffer); 120 | } 121 | 122 | -------------------------------------------------------------------------------- /part08/hipiwrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is a wrapper that launches the mpi binary using the appropriate mpi launcher. 4 | # On edison/cori the launch will be done using srun 5 | 6 | MPI_RANKS=$1 7 | MPI_APP_PATH=$2 8 | shift 2 9 | 10 | if which srun &> /dev/null ; then 11 | 12 | echo "Executing with srun :" 13 | srun -n $MPI_RANKS $MPI_APP_PATH $* 14 | 15 | elif which mpirun &> /dev/null ; then 16 | 17 | echo "Executing with mpirun :" 18 | mpirun -np $MPI_RANKS $MPI_APP_PATH $* 19 | 20 | fi 21 | 22 | 23 | -------------------------------------------------------------------------------- /part08/nersc.conf: -------------------------------------------------------------------------------- 1 | ../nersc.conf -------------------------------------------------------------------------------- /part08/p8.swift: -------------------------------------------------------------------------------- 1 | type file; 2 | 3 | string curdir = java("java.lang.System","getProperty","user.dir"); 4 | 5 | app (file out, file err) pi (int intervals, int duration ) 6 | { 7 | mpiwrap 48 pibinary intervals duration stdout=@out stderr=@err; 8 | } 9 | 10 | app (file out) summarize (file pi_runs[] ) 11 | { 12 | grep "^Pi:" filenames(pi_runs) stdout=@out; 13 | } 14 | 15 | int duration = toInt(arg("duration", "0")); # Min duration of pi run (artificial) 16 | 17 | file pi_out[] ; 18 | file pi_err[] ; 19 | 20 | global string pibinary = arg("executable", curdir+"/hipi"); 21 | 22 | foreach interval in [10,100,1000,10000,100000] { 23 | (pi_out[interval],pi_err[interval]) = pi(interval, duration); 24 | } 25 | 26 | file summary <"output/summary.out">; 27 | 28 | summary = summarize(pi_out); 29 | -------------------------------------------------------------------------------- /part08/swift.conf: -------------------------------------------------------------------------------- 1 | ../swift.conf -------------------------------------------------------------------------------- /part08/xsede.conf: -------------------------------------------------------------------------------- 1 | ../xsede.conf -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | ######################### CONFIGS ########################### 2 | 3 | export BEAGLE_USERNAME="" 4 | export BEAGLE_PROJECT="" 5 | export MIDWAY_USERNAME="" 6 | export AWS_CREDENTIALS_FILE="" 7 | export URL_OF_AD_HOC_MACHINE_1="" 8 | export AD_HOC_1_USERNAME="" 9 | export AD_HOC_N_USERNAME="" 10 | export OSG_USERNAME="" 11 | export OSG_PROJECT="" 12 | export BLUES_USERNAME="" 13 | export CLOUD_HEADNODE="http://HEADNODE_ADDRESS:50010" 14 | ############################################################# 15 | 16 | 17 | # ensure that this script is being sourced 18 | if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then 19 | echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]} 20 | exit 1 21 | fi 22 | 23 | 24 | # Setting scripts folder to the PATH env var. 25 | TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 26 | 27 | if [ _$(which cleanup 2>/dev/null) != _$TUTDIR/bin/cleanup ]; then 28 | echo Adding $TUTDIR/bin:$TUTDIR/app: to front of PATH 29 | PATH=$TUTDIR/bin:$TUTDIR/app:$PATH 30 | else 31 | echo Assuming $TUTDIR/bin:$TUTDIR/app: is already at front of PATH 32 | fi 33 | 34 | 35 | # Set PATH for specific hosts 36 | 37 | if hostname | grep bridges; then 38 | 39 | module load java # Get Oracle Java 1.8.x 40 | SWIFT=/home/wilde/swift/rev/swift-0.96.2 41 | PATH=$SWIFT/bin:$JAVA:$PATH 42 | 43 | module unload mpi/intel_mpi 44 | module load mpi/gcc_mvapich 45 | export CC="mpicc" 46 | echo "modules adjusted: unloaded mpi/intel_mpi; loaded mpi/gcc_mvapich" 47 | echo "This list should should show mpi/gcc_mvapich as only MPI:" 48 | module list 49 | 50 | elif hostname -f | grep "edison" ; then 51 | echo "Configuring for Edison" 52 | module swap PrgEnv-intel PrgEnv-gnu 53 | export PATH=$PWD/part07:$PATH 54 | export CC="cc" 55 | 56 | pushd . 57 | cd part07/ ; 58 | make clean ; make 59 | popd 60 | 61 | pushd . 62 | cd part08/ ; rm hipi 63 | make clean ; make 64 | popd 65 | 66 | 67 | elif hostname -f | grep "ncsa.illinois.edu" ; then 68 | # BW does not have a clearly identifiable hostname on the login nodes 69 | 70 | module load java # Oracle java java/jdk1.8.0_51 71 | module unload PrgEnv-cray/5.2.82 72 | module load PrgEnv-gnu/5.2.82 73 | SWIFT=~wilde/scratch/swift/rev/swift-0.96.2 74 | PATH=$SWIFT/bin:$PATH 75 | export CC="cc" 76 | 77 | elif hostname | grep comet; then 78 | 79 | JAVA=/oasis/scratch/comet/xdtr1/temp_project/jdk1.8.0_91/bin 80 | SWIFT=/oasis/scratch/comet/xdtr1/temp_project/swift/swift-0.96.2/bin 81 | PATH=$SWIFT:$JAVA:$PATH 82 | 83 | elif hostname | grep workflow.iu; then 84 | 85 | SWIFT=/opt/swift/swift-0.96.2/bin 86 | JAVA=$(echo /opt/swift/jdk1.*/bin) 87 | PATH=$SWIFT:$JAVA:$PATH 88 | export X509_USER_PROXY=/tmp/x509.$USER.$RANDOM 89 | 90 | elif [ -d /usr/local/bin/swift-trunk ] && [ -d /usr/local/bin/jdk1.7.0_51 ]; then 91 | 92 | JAVA=/usr/local/bin/jdk1.7.0_51/bin 93 | SWIFT=/usr/local/bin/swift-trunk/bin 94 | PATH=$JAVA:$SWIFT:$PATH 95 | 96 | elif [ -d /opt/swift/swift-0.96.1 ] && [ -d /opt/swift/jdk1.7.0_51 ] 97 | then 98 | export SWIFT=/opt/swift/swift-0.96.1/bin 99 | export JAVA=/opt/swift/jdk1.7.0_51/bin 100 | export PATH=$SWIFT:$JAVA:$PATH 101 | export X509_USER_PROXY=/tmp/x509.$USER.$RANDOM 102 | fi 103 | 104 | echo "Swift version is $(swift -version)" 105 | 106 | return 107 | -------------------------------------------------------------------------------- /swift.conf: -------------------------------------------------------------------------------- 1 | sites: [localhost] 2 | # sites can be set to a comma separated list of remote sites you have access to 3 | # eg.: sites: [beagle] or sites: [beagle, midway] 4 | # The sites definition in the config can be overridden by specifying -sites on the 5 | # swift command line as follows : swift -sites beagle p4.swift 6 | 7 | # Default site for examples 1-3 8 | # This site runs tasks on the local machine 9 | site.localhost { 10 | execution { 11 | type : "local" # Execution is local 12 | URL : "localhost" # Point at localhost to run locally 13 | } 14 | staging : direct # Files are on the same machine, so can be accessed "directly" 15 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Directory where work is done 16 | maxParallelTasks : 101 # Maximum number of parallel tasks 17 | initialParallelTasks: 100 # Maximum number of tasks at start 18 | app.ALL { executable: "*" } # All tasks to be found from commandline 19 | } 20 | 21 | # Instructions for Beagle 22 | # 1. If you are running on the beagle login nodes, set jobManager: "local:pbs" 23 | # 2. Find your project name/allocation and set jobProject : "YOUR_PROJECT_ON_BEAGLE" 24 | # 3. Set userHomeOverride : "/lustre/beagle2/YOUR_USERNAME_ON_BEAGLE/swiftwork" 25 | # 4. Set workDirectory : "/tmp/YOUR_USERNAME_ON_BEAGLE/swiftwork" 26 | site.beagle { 27 | execution { 28 | type : "coaster" # Use coasters to run on remote sites 29 | URL : "login4.beagle.ci.uchicago.edu" # Beagle login URL 30 | jobManager: "ssh-cl:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 31 | options { 32 | maxJobs : 1 # Max jobs submitted to LRM 33 | nodeGranularity : 1 # Nodes per job 34 | maxNodesPerJob : 1 # Nodes per job 35 | tasksPerNode : 4 # Tasks per Node 36 | jobQueue : "development" # Select queue 37 | jobProject : ${env.BEAGLE_PROJECT} # Project|Allocation on Beagle 38 | userHomeOverride: "/lustre/beagle2/"${env.BEAGLE_USERNAME}"/swiftwork" # Point to lustre shared-filesystem 39 | maxJobTime : "00:25:00" # Time requested per job 40 | jobOptions { 41 | pbs.aprun: true # Submit jobs via aprun mechanism 42 | pbs.mpp : true # Mpp enabled 43 | depth : "4" # 4 cores per task 44 | } 45 | } 46 | } 47 | staging : "local" # Stage files from "local" system to Beagle 48 | workDirectory : "/tmp/"${env.BEAGLE_USERNAME}"/swiftwork" # Location for intermediate files 49 | maxParallelTasks : 101 # Maximum number of parallel tasks 50 | initialParallelTasks: 100 # Maximum number of tasks at start 51 | app.ALL { executable: "*" } # All tasks to be found from commandline 52 | } 53 | 54 | # Instructions for Midway: 55 | # 1. If you are running on the midway login nodes set jobManager: "local:slurm" 56 | # 2. Set workDirectory to /tmp/your_username_on_midway 57 | site.midway { 58 | execution { 59 | type : "coaster" # Use coasters to run on remote sites 60 | URL : "swift.rcc.uchicago.edu" # Midway login node | Alternatively use midway.rcc.uchicago.edu 61 | jobManager: "ssh-cl:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 62 | options { 63 | maxJobs : 1 # Max jobs submitted to LRM 64 | nodeGranularity : 1 # Nodes per job 65 | maxNodesPerJob : 1 # Nodes per job 66 | tasksPerNode : 4 # Tasks per Node 67 | jobQueue : "sandyb" # Select queue from (sandyb, westmere, ...) 68 | maxJobTime : "00:25:00" # Time requested per job 69 | } 70 | } 71 | staging : "local" # Stage files from "local" system to Midway 72 | workDirectory : "/tmp/"${env.MIDWAY_USERNAME} # Location for intermediate files 73 | maxParallelTasks : 101 # Maximum number of parallel tasks 74 | initialParallelTasks: 100 # Maximum number of tasks at start 75 | app.ALL { executable: "*" } # All tasks to be found from commandline 76 | } 77 | 78 | # Instruction for OSGConnect 79 | # 1. If you are running on the OSGConnect login node set jobManager: "local:condor" 80 | # 2. Set projectname : "YOUR_PROJECTNAME_ON_OSG" 81 | # 3. Set workDirectory : "/tmp/YOUR_USERNAME_ON_OSG" 82 | 83 | site.osgc { 84 | execution { 85 | type : "coaster" # Use coasters to run on remote sites 86 | URL : "login.osgconnect.net" # OSGConnect login node 87 | jobManager : "ssh-cl:condor" # Use ssh-cl to connect, Condor is the Local resource manager 88 | options { 89 | maxJobs : 1 # Max jobs submitted to LRM 90 | nodeGranularity : 1 # Nodes per job 91 | maxNodesPerJob : 1 # Nodes per job 92 | tasksPerNode : 1 # Tasks per Node 93 | maxJobTime : "00:25:00" # Time requested per job 94 | jobOptions.condor { 95 | projectname : ${env.OSG_PROJECT} # Set the Project name of your allocation on OSG 96 | # Using requirements directive, you can add any requirement attributes for condor. Eg: 97 | # requirements : "(HAS_CVMFS_oasis_opensciencegrid_org =?= TRUE)" 98 | } 99 | } 100 | } 101 | staging : "local" # Stage files from "local" system to OSGConnect 102 | workDirectory : "/tmp/"${env.OSG_USERNAME} # Location for intermediate files 103 | maxParallelTasks : 101 # Maximum number of parallel tasks 104 | initialParallelTasks: 100 # Maximum number of tasks at start 105 | app.ALL { executable: "*" } # All tasks to be found from commandline 106 | } 107 | 108 | # Instructions for Amazon EC2 109 | # 1. Set ec2CredentialsFile: "ABSOLUTE_PATH_TO_YOUR_AWS_CREDENTIALS_FILE" 110 | # 2. Ensure that ec2KeypairFile points to a valid path 111 | # 3. If you update ec2WorkerImage, ensure that it is a ubuntu image 112 | 113 | site.aws { 114 | execution { 115 | type : "coaster" # Use coasters to run on remote sites 116 | URL : "127.0.0.1" # We use a local service to connect to AWS 117 | jobManager : "local:ec2-cloud" # ec2-cloud provider manages the cloud resources 118 | options { 119 | maxJobs : 1 # Max jobs requested from AWS 120 | tasksPerNode : 2 # Tasks per Node 121 | maxJobTime : "00:25:00" # Time requested per job 122 | jobOptions { 123 | ec2CredentialsFile: ${env.AWS_CREDENTIALS_FILE} # The credentials required to authenticate with AWS 124 | ec2SecurityGroup : swift_security_group # This security group will be generated 125 | ec2KeypairName : swift-test-pair # Key-pair to connect to nodes 126 | ec2KeypairFile : ${env.HOME}/.ssh/swift-test-pair.pem # Path to create the key-pair 127 | ec2WorkerImage : ami-23700813 # Amazon image id 128 | ec2WorkerType : t1.micro # Instance type 129 | } 130 | } 131 | } 132 | staging : "local" # Stage files from "local" system to AWS 133 | workDirectory : "/tmp/"${env.USER}"/work" # Location for intermediate files 134 | maxParallelTasks : 20 # Maximum number of parallel tasks 135 | initialParallelTasks: 20 # Maximum number of tasks at start 136 | app.ALL { executable: "*" } # All tasks to be found from commandline 137 | } 138 | 139 | 140 | # Instructions for a pool of Ad-hoc machines 141 | # Assume that you have access to N machines ( alpha.foo.net, beta.foo.net ... omega.foo.net ) 142 | # Replicate the site.ad-hoc-1 site definition block for each site you have access to. 143 | # Set the site. with say site.alpha, site.beta etc.. 144 | # Set the site definition for each site with the correct URL 145 | # Set the first line of this file to a comma separated list of sites, eg: 146 | # sites: [ alpha, beta, omega ] 147 | 148 | # Instructions for Ad-hoc-1 149 | # 1. Set URL : URL_OF_AD_HOC_MACHINE_1 150 | # 2. Set workDirectory : "/tmp/USERNAME_ON_AD-HOC-1/work" 151 | site.ad-hoc-1 { 152 | execution { 153 | type : "coaster" # Use coasters to run on remote sites 154 | URL : ${env.URL_OF_AD_HOC_MACHINE_1} # URL of the remote machine to connect to 155 | jobManager : "ssh-cl:local" # ssh-cl to connect to machine, run worker locally 156 | options { 157 | maxJobs : 2 # Max jobs sent to remote node 158 | tasksPerNode : 10 # Tasks per Node 159 | maxJobTime : "00:25:00" # Time requested per job 160 | } 161 | } 162 | staging : "local" # Stage files from "local" system to ad-hoc-1 163 | workDirectory : "/tmp/"${env.USER}"/work" # Location for intermediate files 164 | maxParallelTasks : 20 # Maximum number of parallel tasks 165 | initialParallelTasks: 20 # Maximum number of tasks at start 166 | app.ALL { executable: "*" } # All tasks to be found from commandline 167 | } 168 | 169 | # Site definition for use at Legion UCL 170 | # parallel environments have been disabled in config 171 | # Use http://users.rcc.uchicago.edu/~yadunand/swift-trunk-sge-mod.tar.gz 172 | site.Legion { 173 | execution { 174 | type: "coaster" 175 | jobManager: "local:sge" 176 | URL : "localhost" 177 | options { 178 | maxJobs: 5 179 | nodeGranularity: 1 180 | maxNodesPerJob: 1 181 | tasksPerNode: 1 182 | jobProject: "Training" 183 | jobQueue: "Tarvek" 184 | maxJobTime: "00:08:20" 185 | } 186 | } 187 | initialParallelTasks : 5 188 | staging: local 189 | workDirectory: "/tmp/"${env.USER} 190 | app.ALL { 191 | executable: "*" 192 | maxWallTime: "00:05:00" 193 | } 194 | } 195 | 196 | # Instructions for running work on a remote cloud setup 197 | # 1. Follow instructions here to setup a cluster : 198 | # https://github.com/swift-lang/swift-on-cloud/tree/master/aws 199 | # 2. Set the CLOUD_HEADNODE to point at persistent service 200 | # running on the headnode of the cluster. 201 | # The coaster service is usually running on port 50010. 202 | site.remote-cloud { 203 | execution { 204 | type : "coaster-persistent" 205 | URL : ${env.CLOUD_HEADNODE} # URL of the remote machine to connect to 206 | jobManager : "local:local" # ssh-cl to connect to machine, run worker locally 207 | options { 208 | maxJobs : 2 # Max jobs sent to remote node 209 | tasksPerNode : 10 # Tasks per Node 210 | maxJobTime : "00:25:00" # Time requested per job 211 | } 212 | } 213 | staging : "local" # Stage files from "local" system to ad-hoc-1 214 | workDirectory : "/tmp/"${env.USER}"/work" # Location for intermediate files 215 | maxParallelTasks : 20 # Maximum number of parallel tasks 216 | initialParallelTasks: 20 # Maximum number of tasks at start 217 | app.ALL { executable: "*" } # All tasks to be found from commandline 218 | } 219 | 220 | # Instructions for Blues 221 | # 1. If you are running on the blues login, set jobManager: "local:pbs" 222 | # 2. Set workDirectory : "/home/YOUR_USERNAME_ON_BLUES/swiftwork" 223 | site.blues { 224 | execution { 225 | type : "coaster" # Use coasters to run on remote sites 226 | URL : "blues.lcrc.anl.gov" # Blues login URL 227 | jobManager: "ssh-cl:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 228 | options { 229 | maxJobs : 4 # Max jobs submitted to LRM 230 | nodeGranularity : 1 # Nodes per job 231 | maxNodesPerJob : 1 # Nodes per job 232 | tasksPerNode : 4 # Tasks per Node 233 | jobQueue : "route" # Select queue 234 | maxJobTime : "00:25:00" # Time requested per job 235 | } 236 | } 237 | staging : "local" # Stage files from "local" system to Beagle 238 | workDirectory : "/home/"${env.BLUES_USERNAME}"/swiftwork" # Location for intermediate files 239 | maxParallelTasks : 101 # Maximum number of parallel tasks 240 | initialParallelTasks: 100 # Maximum number of tasks at start 241 | app.ALL { executable: "*" } # All tasks to be found from commandline 242 | } 243 | 244 | # Instructions for swan: 245 | # Note: Do not use mpp on swan 246 | site.swan { 247 | execution { 248 | type: "coaster" 249 | URL: "swan.cray.com" 250 | jobManager: "local:pbs" 251 | options { 252 | nodeGranularity: 4 253 | maxNodesPerJob: 4 254 | maxJobs: 1 255 | tasksPerNode: 24 256 | jobOptions { 257 | pbs.aprun: true # Submit jobs via aprun mechanism 258 | pbs.mpp : false # Mpp enabled 259 | ppn: 12 260 | } 261 | } 262 | } 263 | staging: direct 264 | workDirectory: "/tmp/"${env.USER}"/swiftwork" 265 | maxParallelTasks: 101 266 | initialParallelTasks: 100 267 | app.ALL { executable: "*" } 268 | 269 | } 270 | 271 | 272 | TCPPortRange: "50000,51000" # TCP port range used by swift to communicate with remote sites 273 | lazyErrors: false # Swift fails immediately upon encountering an error 274 | executionRetries: 0 # Set number of retries upon task failures 275 | keepSiteDir: true # Keep Site Dir (useful for debug) 276 | providerStagingPinSwiftFiles: false # Pin staging files (useful for debug) 277 | alwaysTransferWrapperLog: true # Transfer wrapper logs (useful for debug) 278 | 279 | -------------------------------------------------------------------------------- /test_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script seems to be customized for user 'yadunandb', intentional? 4 | 5 | source setup.sh 6 | 7 | export BEAGLE_USERNAME="yadunandb" 8 | export BEAGLE_PROJECT="CI-CCR000013" 9 | export MIDWAY_USERNAME="yadunand" 10 | export AWS_CREDENTIALS_FILE="/home/yadu/.ssh/swift-grant-credentials.csv" 11 | export URL_OF_AD_HOC_MACHINE_1="crank" 12 | export AD_HOC_1_USERNAME="yadunand" 13 | export OSG_USERNAME="yadunand" 14 | export OSG_PROJECT="swift" 15 | export GORDON_USERNAME="yadunand" 16 | export TRESTLES_USERNAME="yadunand" 17 | export HOPPER_USERNAME="yadunand" 18 | export BLACKLIGHT_USERNAME="yadunand" 19 | 20 | 21 | if 0 ;then 22 | for i in $(seq 1 1 3) 23 | do 24 | pushd . 25 | echo "============================TESTING part0$i===========================" 26 | cd part0$i 27 | swift p$i.swift -site=localhost 28 | if [[ $? == 0 ]] 29 | then 30 | echo "Cleaning up!" 31 | cleanup 32 | fi 33 | echo -e "\n\n" 34 | popd 35 | done 36 | fi 37 | 38 | #SITES=('beagle' 'midway' 'osgc' 'ad-hoc-1') 39 | #SITES=('stampede' 'gordon' 'blacklight') 40 | SITES=('edison') 41 | CONFIG="-config nersc.conf" 42 | for i in $(seq 4 1 6) 43 | do 44 | pushd . 45 | echo "============================TESTING part0$i===========================" 46 | cd part0$i 47 | 48 | for SITE in ${SITES[*]} 49 | do 50 | echo "Running on SITE : $SITE" 51 | swift $CONFIG -sites $SITE p$i.swift 52 | if [[ $? == 0 ]] 53 | then 54 | echo "Cleaning up!" 55 | cleanup 56 | fi 57 | done 58 | echo -e "\n\n" 59 | popd 60 | done 61 | 62 | 63 | SITES=('edison_multinode') 64 | CONFIG="-config nersc.conf" 65 | for i in $(seq 7 1 8) 66 | do 67 | pushd . 68 | echo "============================TESTING part0$i===========================" 69 | cd part0$i 70 | 71 | for SITE in ${SITES[*]} 72 | do 73 | echo "Running on SITE : $SITE" 74 | swift $CONFIG -sites $SITE p$i.swift 75 | if [[ $? == 0 ]] 76 | then 77 | echo "Cleaning up!" 78 | cleanup 79 | fi 80 | done 81 | echo -e "\n\n" 82 | popd 83 | done 84 | -------------------------------------------------------------------------------- /xsede.conf: -------------------------------------------------------------------------------- 1 | sites: [stampede] 2 | 3 | # Instructions for Stampede 4 | # 1. If you are running on the midway login nodes set jobManager: "local:slurm" 5 | # 2. Set workDirectory to /tmp/your_username_on_stampede 6 | site.stampede { 7 | execution { 8 | type : "coaster" # Use coasters to run on remote sites 9 | URL : "login4.stampede.tacc.utexas.edu" # Stampede login nodes login[1..4].stampede.tacc.utexas.edu 10 | jobManager: "ssh-cl:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 11 | options { 12 | maxJobs : 1 # Max jobs submitted to LRM 13 | nodeGranularity : 1 # Nodes per job 14 | maxNodesPerJob : 1 # Nodes per job 15 | tasksPerNode : 4 # Tasks per Node 16 | jobQueue : "development" # Select queue from (development, normal, large) 17 | maxJobTime : "00:25:00" # Time requested per job 18 | } 19 | } 20 | staging : "local" # Stage files from "local" system to Midway 21 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 22 | maxParallelTasks : 101 # Maximum number of parallel tasks 23 | initialParallelTasks: 100 # Maximum number of tasks at start 24 | app.ALL { executable: "*" } # All tasks to be found from commandline 25 | } 26 | 27 | # Instructions for Blacklight 28 | # 1. If you are running on the blacklight login nodes, set jobManager: "local:pbs" 29 | # 2. If you are running Set userHomeOverride : "/lustre/blacklight2/YOUR_USERNAME_ON_BLACKLIGHT/swiftwork" 30 | # 4. Set workDirectory : "/tmp/YOUR_USERNAME_ON_BLACKLIGHT/swiftwork" 31 | site.blacklight { 32 | execution { 33 | type : "coaster" # Use coasters to run on remote sites 34 | URL : "blacklight.psc.xsede.org" # Blacklight login URL 35 | jobManager: "ssh-cl:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 36 | options { 37 | maxJobs : 1 # Max jobs submitted to LRM 38 | nodeGranularity : 1 # Nodes per job 39 | maxNodesPerJob : 1 # Nodes per job 40 | tasksPerNode : 4 # Tasks per Node 41 | maxJobTime : "00:25:00" # Time requested per job 42 | jobQueue : debug 43 | jobOptions { 44 | ppn : "16" # Virtual processors per node per Job 45 | } 46 | } 47 | } 48 | staging : "local" # Stage files from "local" system to Blacklight 49 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 50 | maxParallelTasks : 101 # Maximum number of parallel tasks 51 | initialParallelTasks: 100 # Maximum number of tasks at start 52 | app.ALL { executable: "*" } # All tasks to be found from commandline 53 | } 54 | 55 | # Instructions for Gordon 56 | # 1. Do *NOT* run on the Gordon login nodes. There are memory limits which prevent swift from running 57 | # properly on these machines. 58 | site.gordon { 59 | execution { 60 | type : "coaster" # Use coasters to run on remote sites 61 | URL : "gordon.sdsc.edu" # Gordon login URL 62 | jobManager: "ssh-cl:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 63 | options { 64 | maxJobs : 1 # Max jobs submitted to LRM 65 | nodeGranularity : 1 # Nodes per job 66 | maxNodesPerJob : 1 # Nodes per job 67 | tasksPerNode : 4 # Tasks per Node 68 | maxJobTime : "00:25:00" # Time requested per job 69 | jobOptions { 70 | ppn : "16" # Virtual processors per node per Job 71 | } 72 | } 73 | } 74 | staging : "local" # Stage files from "local" system to Gordon 75 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 76 | maxParallelTasks : 101 # Maximum number of parallel tasks 77 | initialParallelTasks: 100 # Maximum number of tasks at start 78 | app.ALL { executable: "*" } # All tasks to be found from commandline 79 | } 80 | 81 | # Instructions for Trestles 82 | # 1. Do *NOT* run on the Trestles login nodes. There are memory limits which prevent swift from running 83 | # properly on these machines. 84 | site.trestles { 85 | execution { 86 | type : "coaster" # Use coasters to run on remote sites 87 | URL : "trestles.sdsc.edu" # Trestles login URL 88 | jobManager: "ssh-cl:pbs" # use ssh-cl to connect, pbs is the Local Resource manager(LRM) 89 | options { 90 | maxJobs : 1 # Max jobs submitted to LRM 91 | nodeGranularity : 1 # Nodes per job 92 | maxNodesPerJob : 1 # Nodes per job 93 | tasksPerNode : 4 # Tasks per Node 94 | maxJobTime : "00:25:00" # Time requested per job 95 | jobOptions { 96 | ppn : "16" # Virtual processors per node per Job 97 | } 98 | } 99 | } 100 | staging : "local" # Stage files from "local" system to Trestles 101 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 102 | maxParallelTasks : 101 # Maximum number of parallel tasks 103 | initialParallelTasks: 100 # Maximum number of tasks at start 104 | app.ALL { executable: "*" } # All tasks to be found from commandline 105 | } 106 | 107 | site.stampede-mpi { 108 | execution { 109 | type : "coaster" # Use coasters to run on remote sites 110 | URL : "login4.stampede.tacc.utexas.edu" # Stampede login nodes login[1..4].stampede.tacc.utexas.edu 111 | jobManager: "local:slurm" # Use ssh-cl to connect, slurm is the Local resource manager 112 | options { 113 | maxJobs : 2 # Max jobs submitted to LRM 114 | nodeGranularity : 1 # Nodes per job 115 | maxNodesPerJob : 1 # Nodes per job 116 | tasksPerNode : 4 # Tasks per Node 117 | jobQueue : "normal" # Select queue from (development, normal, large) 118 | maxJobTime : "00:25:00" # Time requested per job 119 | } 120 | } 121 | staging : "direct" # Stage files from "local" system to Midway 122 | workDirectory : "/tmp/"${env.USER}"/swiftwork" # Location for intermediate files 123 | maxParallelTasks : 101 # Maximum number of parallel tasks 124 | initialParallelTasks: 100 # Maximum number of tasks at start 125 | app.ALL { executable: "*" } # All tasks to be found from commandline 126 | } 127 | 128 | TCPPortRange: "50000,51000" # TCP port range used by swift to communicate with remote sites 129 | lazyErrors: false # Swift fails immediately upon encountering an error 130 | executionRetries: 0 # Set number of retries upon task failures 131 | keepSiteDir: true # Keep Site Dir (useful for debug) 132 | providerStagingPinSwiftFiles: false # Pin staging files (useful for debug) 133 | alwaysTransferWrapperLog: true # Transfer wrapper logs (useful for debug) 134 | --------------------------------------------------------------------------------