├── .travis.yml ├── LICENSE.rst ├── Makefile ├── Options.mk.example ├── README.rst ├── activate.sh ├── bcast-tar.c ├── bcast.c ├── benchmarks ├── anaconda │ ├── dobench.sh │ ├── plot.py │ └── python-file-ops.png ├── cray-xc30 │ ├── bench-1.out │ ├── bench-1024.out │ ├── bench-128.out │ ├── bench-16.out │ ├── bench-2.out │ ├── bench-2048.out │ ├── bench-256.out │ ├── bench-32.out │ ├── bench-4.out │ ├── bench-4096.out │ ├── bench-512.out │ ├── bench-5310.out │ ├── bench-64.out │ ├── bench-8.out │ ├── bench.template │ ├── collect.sh │ ├── cray-xc30-startup-time.png │ ├── plot.py │ ├── run-bench.sh │ └── tar-cray-xc30.slurm ├── cray-xc40 │ └── tar-cray-xc40.slurm ├── cray-xt │ ├── bench-1.out │ ├── bench-1024.out │ ├── bench-128.out │ ├── bench-16.out │ ├── bench-2.out │ ├── bench-2048.out │ ├── bench-256.out │ ├── bench-32.out │ ├── bench-4.out │ ├── bench-4096.out │ ├── bench-512.out │ ├── bench-64.out │ ├── bench-8.out │ ├── bench-8192.out │ ├── bench.template │ ├── collect.sh │ ├── cray-xt-startup-time.png │ ├── plot.py │ ├── run-bench.sh │ └── tar-cray-xt.job ├── startup-time.png └── startup-time.txt ├── nersc-hpcp ├── activate.sh ├── bundle-hpcp.sh └── easy-install.pth ├── nersc ├── activate.sh ├── build.sh ├── deploy.sh └── run-test.slurm ├── plot-startup.py ├── runtest.sh ├── tar-anaconda.sh ├── tar-dir.sh └── tar-pip.sh /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | 3 | # Disabled openmpi because it conflicts with anaconda's mpich. 4 | # 5 | #addons: 6 | # apt: 7 | # packages: 8 | # - libopenmpi-dev 9 | # - openmpi-bin 10 | 11 | env : 12 | - TRAVIS_PYTHON_VERSION=2.7 13 | - TRAVIS_PYTHON_VERSION=3.6 14 | 15 | before_install: 16 | - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh 17 | - chmod +x miniconda.sh 18 | - ./miniconda.sh -b -p $HOME/miniconda 19 | - export PATH=$HOME/miniconda/bin:$PATH 20 | - conda update --yes conda 21 | - conda create --yes -n test python=$TRAVIS_PYTHON_VERSION gcc_linux-64 bzip2 22 | - export CONDA_BUILD=1 23 | - source activate test 24 | - conda install --yes numpy nose 25 | - conda install --yes mpi4py mpich 26 | 27 | install: 28 | - which gcc 29 | - which mpicc 30 | - cp Options.mk.example Options.mk 31 | - make bcast 32 | - mpirun -n 2 python -c 'from mpi4py import MPI; print(MPI);' 33 | - bash tar-anaconda.sh anaconda.tar.gz $HOME/miniconda/envs/test 34 | 35 | script: 36 | - bash runtest.sh 37 | -------------------------------------------------------------------------------- /LICENSE.rst: -------------------------------------------------------------------------------- 1 | LICENSE: Launching Python Applications with MPI Broadcast 2 | ========================================================= 3 | 4 | :Author: Yu Feng 5 | :Contact: rainwoodman@gmail.com 6 | :Web Site: http://github.com/rainwoodman/python-mpi-bcast 7 | :Organization: Berkeley Center for Cosmological Physics (http://bccp.berkeley.edu) 8 | Berkeley Institute for Data Science (http://bids.berkeley.edu) 9 | 10 | 11 | Copyright (c) 2015. Yu Feng 12 | All rights reserved. 13 | 14 | Redistribution and use in source and binary forms, with or without 15 | modification, are permitted provided that the following conditions 16 | are met: 17 | 18 | * Redistributions of source code must retain the above copyright 19 | notice, this list of conditions and the following disclaimer. 20 | 21 | * Redistributions in binary form must reproduce the above 22 | copyright notice, this list of conditions and the following 23 | disclaimer in the documentation and/or other materials provided 24 | with the distribution. 25 | 26 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS 27 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include Options.mk 2 | 3 | VERSION = 0.1.3 4 | SCRIPTS = activate.sh tar-anaconda.sh tar-dir.sh tar-pip.sh 5 | BCASTSRC = bcast.c bcast-tar.c 6 | DOC = README.rst LICENSE.rst Options.mk.example 7 | .PHONY: build clean sdist install 8 | 9 | all: bcast 10 | 11 | Options.mk: 12 | @echo "Need Options.mk; produce the file by copying and modifying Options.mk.example." 13 | @exit 1 14 | 15 | bcast : bcast.c bcast-tar.c depends/lib/libarchive.a 16 | $(MPICC) -Idepends/include -Ldepends/lib -g -O0 -o bcast \ 17 | bcast.c bcast-tar.c \ 18 | -larchive -lz -lbz2 19 | 20 | libarchive-3.1.2.tar.gz: 21 | curl -o $@ http://www.libarchive.org/downloads/libarchive-3.1.2.tar.gz 22 | 23 | libarchive-3.1.2/configure: libarchive-3.1.2.tar.gz 24 | tar -xzvf libarchive-3.1.2.tar.gz && touch $@ 25 | 26 | depends/lib/libarchive.a: libarchive-3.1.2/configure 27 | (cd libarchive-3.1.2; \ 28 | ./configure --prefix=/ "CC=$(CC)" \ 29 | --without-iconv \ 30 | --without-xml2 \ 31 | --without-nettle \ 32 | --without-openssl \ 33 | --without-lzma \ 34 | --without-expat \ 35 | --disable-shared --enable-static \ 36 | --disable-bsdtar --disable-bsdcpio; \ 37 | make install DESTDIR=$(PWD)/depends) 38 | 39 | 40 | install: bcast 41 | mkdir -p $(PREFIX)/libexec/python-mpi-bcast 42 | cp bcast $(SCRIPTS) $(PREFIX)/libexec/python-mpi-bcast/ 43 | 44 | sdist: 45 | mkdir -p python-mpi-bcast-$(VERSION) 46 | cp $(SCRIPTS) $(BCASTSRC) $(DOC) Makefile python-mpi-bcast-$(VERSION) 47 | tar -czvf python-mpi-bcast-$(VERSION).tar.gz python-mpi-bcast-$(VERSION) 48 | rm -rf python-mpi-bcast-$(VERSION) 49 | 50 | clean: 51 | rm -f bcast 52 | 53 | 54 | -------------------------------------------------------------------------------- /Options.mk.example: -------------------------------------------------------------------------------- 1 | # on BlueWaters (cray) 2 | # MPICC='cc -static' 3 | 4 | # on COMA (CMU) 5 | # MPICC=mpiicc LDSHARED="mpiicc -shared" 6 | 7 | # on Fedora 19 with openmpi 8 | # MPICC=mpicc LDSHARED="mpicc -shared" 9 | # 10 | # on Ubuntu with openmpi 11 | # MPICC=mpicc LDSHARED="mpicc -shared" 12 | 13 | # on Edison (gcc) 14 | # MPICC='cc -static' 15 | 16 | MPICC=mpicc 17 | 18 | # install location 19 | PREFIX=$(PWD)/usr 20 | 21 | CC=$(MPICC) 22 | CCDYNAMIC=$(MPICC) -dynamic 23 | LDSHARED=$(MPICC) -shared 24 | 25 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | python-mpi-bcast 2 | ================ 3 | 4 | An HPC friendly python environment that deploys packages to computing nodes via MPI. 5 | 6 | .. image:: https://api.travis-ci.org/rainwoodman/python-mpi-bcast.svg 7 | :alt: Build Status 8 | :target: https://travis-ci.org/rainwoodman/python-mpi-bcast/ 9 | 10 | Benchmark on Cray systems 11 | ------------------------- 12 | 13 | We performed benchmarks on the Cray XC-30 system Edison at NERSC and the Cray XT system BlueWaters 14 | at NCSA. 15 | 16 | See the following figure. python-mpi-bcast demonstrates excellent scaling properties. 17 | We can start 12,288 ranks, each import scipy, and doing all of these in 50 seconds. 18 | 19 | The overhead of python-mpi-bcast at full machine capability is comparable to the ALPS applicationg launching 20 | overhead. 21 | 22 | ==================== ===================== 23 | Cray XC30 Edison Cray XT BlueWaters 24 | -------------------- --------------------- 25 | |xc30| |xt| 26 | ==================== ===================== 27 | 28 | .. |xc30| image:: https://raw.githubusercontent.com/rainwoodman/python-mpi-bcast/master/cray-xc30/cray-xc30-startup-time.png 29 | .. |xt| image:: https://raw.githubusercontent.com/rainwoodman/python-mpi-bcast/master/cray-xt/cray-xt-startup-time.png 30 | 31 | Why is Python slow on a supercomputer? 32 | --------------------------------------- 33 | 34 | One problem with large scale parallel application written in Python is the slow startup time. 35 | The Python interpreter may spend half an hour before even start processing any useful user logic. 36 | 37 | Python does a lot of file operations upon startup. 38 | This is not an issue for small scale applications -- but on 39 | applications at a massive scale (10K+ MPI ranks), these file 40 | operations become a burden to the shared file system, just like the 41 | shared library burden, described in [Hopper-UG] 42 | 43 | For example, on a typical python installation with numpy the number of 44 | file operations to :: 45 | 46 | $ strace -ff -e file python -c '' 2>&1 |wc -l 47 | 917 48 | 49 | $ strace -ff -e file python -c 'import numpy.fft' 2>&1 |wc -l 50 | 51 | 4557 52 | $ strace -ff -e file python -c 'import numpy.fft; import scipy.interpolate' 2>&1|wc -l 53 | 8089 54 | 55 | Now multiply this number by the number of ranks, 1024, for example. 56 | 57 | Keep in mind that in a massively parallel application, the payload may 58 | in fact only access a few very large files. The overhead here is a 59 | headache. 60 | 61 | What do we do about this? 62 | ------------------------- 63 | 64 | People have thought that python just can never work well on HPC systems. 65 | This is not true. 66 | We can start 1024 Python ranks on edison.nersc.gov in 40 seconds, consistently as long as we 67 | follow the principles in this page. We will need the help of a tool 'bcast' that is provided here. 68 | 69 | The idea is simple: 70 | 71 | - Avoid meta-data requests from slow filesystems (e.g. home directories); 72 | - Avoid as much as possible meta-data requests on (even fast) shared filesystems; 73 | 74 | If these two are done, spinning up thousands of python ranks is no slower than 75 | spinning up the same number of C ranks; and no modifications on the user programs 76 | needs to be done. 77 | 78 | The biggest part is from :code:`bcast` provided here, which deploys selected packages 79 | to the computing node, and properly set up the python environment to avoid 80 | most of the meta-data requests on the shared filesystem. 81 | 82 | 83 | Here is the TODO list that enables the full benefits of the 84 | python-mpi implementation provided here. These steps can be implemented 85 | either by the computing faciliaties, or by a user. 86 | 87 | 1. Install Conda/Anaconda, and create a tar ball of the entire installation with 88 | the supplied 'tar-anaconda.sh' 89 | 90 | .. code:: bash 91 | 92 | wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh 93 | chmod +x miniconda.sh 94 | ./miniconda.sh -b -p $HOME/miniconda 95 | export PATH=$HOME/miniconda/bin:$PATH 96 | conda update --yes conda 97 | conda create --yes -n test python=2.7 98 | source activate test 99 | conda install --yes numpy=1.9 nose mpi4py # install other packages as well 100 | bash tar-anaconda.sh anaconda.tar.gz $HOME/miniconda/envs/test 101 | 102 | .. note:: 103 | 104 | On some systems, an anaconda based installation is already supplied by the vendor. 105 | (e.g. Edison and Cori). In that case, find the location of that installation 106 | via the module file, and directly use tar-anaconda.sh to generate a tar ball. 107 | 108 | .. attention:: 109 | 110 | copy the tar ball file to a fast file system, e.g. scratch or project directory. 111 | 112 | We will assume the location is $SCRATCH/2.7-anaconda.tar.gz 113 | 114 | 2. Alternatively, prepackage individual python packages to .tar.gz files. On some systems 115 | where the conda prebuilt packages are not an option, this will be the only feasible way. 116 | We provide a script tar-pip.sh for this: 117 | 118 | .. code:: bash 119 | 120 | # build a fitsio bundle 121 | 122 | bash tar-pip.sh fitsio-0.9.8rc2.tar.gz https://github.com/esheldon/fitsio/archive/v0.9.8rc2.zip 123 | 124 | # build a bundle for locally checked out code with a setup.py 125 | 126 | bash tar-pip.sh my-package.tar.gz . 127 | 128 | # you get the idea 129 | 130 | .. note:: 131 | 132 | Still, the installation of some packages may not be this trivial. 133 | Luckily, usually the vendor must have compiled most python packages, and it is worthwhile 134 | to inspect the module files and directly run the tar command there, skipping the installation 135 | part. 136 | 137 | 3. Reset :code:`PYTHONHOME` :code:`PYTHONBASE`, :code:`PYTHONUSERBASE`, and :code:`PATH`, 138 | :code:`LD_LIBRARY_PATH` to /dev/shm/local. 139 | 140 | This can be done by sourcing 'activate.sh'. activate.sh takes 2 arguments, the prefix of the new python 141 | environment, and the command prefix to launch 'bcast'. activate.sh also provide a 'bcast' function 142 | to the shell script, which will simply run bcast with the provided prefix. A good choice of the prefix 143 | is /dev/shm/local. If the computing nodes contain private scratch hardrives, that would be a good location as well. 144 | 145 | .. warning:: 146 | 147 | All packages install in :code:`~/.local` is unavailable during the session. 148 | 149 | 4. Copy the relevant python scripts to a fast filesystem. 150 | 151 | Especially be aware of starting a python script in HOME directory. It can be very 152 | slow. (recall sometimes ls on home directory takes for ever?) 153 | 154 | 155 | Here is a full job script example on Edison following all of the guidelines. 156 | Notice that on Edison, I have already created the tar ball of the 157 | 2.7 and 3.4 version of anaconda installation at /project/projectdirs/m779/python-mpi 158 | 159 | .. code:: bash 160 | 161 | #PBS -j eo 162 | #PBS -l mppwidth=1024 163 | #PBS -q debug 164 | 165 | set -x 166 | export OMP_NUM_THREADS=1 167 | 168 | source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local "aprun -n 1024 -d 1" 169 | 170 | cd $PBS_O_WORKDIR 171 | 172 | # send the anaconda packages 173 | bcast -v /project/projectdirs/m779/python-mpi/2.7-anaconda.tar.gz 174 | 175 | # testpkg contains the tar-ed version of the script; 176 | # if the script is sufficiently complicated, it helps to treat it like 177 | # another package. 178 | 179 | bcast -v testpkg.tar.gz 180 | 181 | time aprun -n 1024 -d 1 python-mpi /dev/shm/local/testpkg/main.py 182 | 183 | Yu Feng - BCCP / BIDS. 184 | 185 | -------------------------------------------------------------------------------- /activate.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | _PYTHONMPIBCASTAPRUN=$2 4 | _PYTHONMPIBCASTBCASTROOT=$1 5 | 6 | if [[ -n $BASH_VERSION ]]; then 7 | _SCRIPT_LOCATION=${BASH_SOURCE[0]} 8 | elif [[ -n $ZSH_VERSION ]]; then 9 | _SCRIPT_LOCATION=${funcstack[1]} 10 | else 11 | echo "Only bash and zsh are supported" 12 | return 1 13 | fi 14 | 15 | _PYTHONMPIBCASTDIRNAME=`dirname ${_SCRIPT_LOCATION}` 16 | _PYTHONMPIBCASTDIRNAME=`readlink -f $_PYTHONMPIBCASTDIRNAME` 17 | _PYTHONMPI_OLD_PYTHONUSERBASE=$PYTHONUSERBASE 18 | 19 | function bundle-pip { 20 | $_PYTHONMPIBCASTDIRNAME/tar-pip.sh $* 21 | } 22 | 23 | function bundle-anaconda { 24 | $_PYTHONMPIBCASTDIRNAME/tar-anaconda.sh $* 25 | } 26 | 27 | if [[ -n $_PYTHONMPIBCASTBCASTROOT ]]; then 28 | trap "$_PYTHONMPIBCASTAPRUN rm -rf $_PYTHONMPIBCASTBCASTROOT" EXIT TERM KILL 29 | 30 | # use the correct Python 31 | 32 | export PATH=$_PYTHONMPIBCASTBCASTROOT/bin:$PATH 33 | # since we use anaconda's relocated python there is no need 34 | # to worry about other PATH variables. 35 | # especially LD_LIBRARY_PATH -- anaconda packages shall use the correct rpath anyways. 36 | 37 | # reset PYTHONUSERBASE to avoid looking up home 38 | export PYTHONUSERBASE=$_PYTHONMPIBCASTBCASTROOT 39 | # important to make the non-cyclic object/dict deconstruction collective 40 | export PYTHONHASHSEED=0 41 | 42 | function bcast { 43 | $_PYTHONMPIBCASTAPRUN $_PYTHONMPIBCASTDIRNAME/bcast -p $_PYTHONMPIBCASTBCASTROOT $* || return 1 44 | } 45 | 46 | function bcast-userbase { 47 | local USERBASE= 48 | 49 | if [ -n $_PYTHONMPI_OLD_PYTHONUSERBASE ]; then 50 | if [ -d $_PYTHONMPI_OLD_PYTHONUSERBASE ]; then 51 | USERBASE=`mktemp --tmpdir XXXXXXX.tar.gz` 52 | bundle-anaconda $USERBASE $_PYTHONMPI_OLD_PYTHONUSERBASE 53 | bcast $USERBASE 54 | rm $USERBASE 55 | fi 56 | fi 57 | } 58 | 59 | function bcast-pip { 60 | local FILE=`mktemp --tmpdir XXXXXXX.tar.gz` 61 | $_PYTHONMPIBCASTDIRNAME/tar-pip.sh $FILE $* 62 | bcast $FILE 63 | rm $FILE 64 | } 65 | 66 | function bcast-dir-pip { 67 | # workaround https://github.com/pypa/pip/issues/2195 68 | local DIR=`mktemp -d --tmpdir XXXXXXX` 69 | (cd $1; python setup.py sdist -d $DIR) 70 | bcast-pip $DIR/*.tar.gz 71 | rm -rf $DIR 72 | } 73 | 74 | function mirror { 75 | # BASH gimmicks: local always return 0 76 | # http://unix.stackexchange.com/a/146900 77 | local TMPFILE 78 | TMPFILE=`$_PYTHONMPIBCASTDIRNAME/tar-dir.sh $*` 79 | if ! [ $? -eq 0 ]; then 80 | # tar-dir must have failed 81 | exit 1 82 | fi 83 | bcast $TMPFILE 84 | rm $TMPFILE 85 | } 86 | 87 | if [[ -n $BASH_VERSION ]]; then 88 | hash -r 89 | elif [[ -n $ZSH_VERSION ]]; then 90 | rehash 91 | else 92 | echo "Only bash and zsh are supported" 93 | return 1 94 | fi 95 | fi 96 | 97 | -------------------------------------------------------------------------------- /bcast-tar.c: -------------------------------------------------------------------------------- 1 | /* 2 | * * This file is in the public domain. 3 | * * Use it as you wish. 4 | * */ 5 | 6 | /* Modified from the untar.c example of libarchive. 7 | * 8 | * Yu Feng, Jan. 2016 9 | * */ 10 | 11 | #include 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | static void errmsg(const char *); 24 | void extract(const char *filename, int do_extract); 25 | static void fail(const char *, const char *, int); 26 | static int copy_data(struct archive *, struct archive *); 27 | static void msg(const char *); 28 | static void warn(const char *, const char *); 29 | 30 | 31 | void 32 | extract(const char *filename, int do_extract) 33 | { 34 | int flags = ARCHIVE_EXTRACT_UNLINK; 35 | struct archive *a; 36 | struct archive *ext; 37 | struct archive_entry *entry; 38 | int r; 39 | 40 | a = archive_read_new(); 41 | ext = archive_write_disk_new(); 42 | archive_write_disk_set_options(ext, flags); 43 | /* 44 | * * Note: archive_write_disk_set_standard_lookup() is useful 45 | * * here, but it requires library routines that can add 500k or 46 | * * more to a static executable. 47 | * */ 48 | archive_read_support_format_tar(a); 49 | archive_read_support_filter_gzip(a); 50 | archive_read_support_filter_bzip2(a); 51 | /* 52 | * * On my system, enabling other archive formats adds 20k-30k 53 | * * each. Enabling gzip decompression adds about 20k. 54 | * * Enabling bzip2 is more expensive because the libbz2 library 55 | * * isn't very well factored. 56 | * */ 57 | if (filename != NULL && strcmp(filename, "-") == 0) 58 | filename = NULL; 59 | if ((r = archive_read_open_filename(a, filename, 10240))) 60 | fail("archive_read_open_filename()", 61 | archive_error_string(a), r); 62 | for (;;) { 63 | r = archive_read_next_header(a, &entry); 64 | if (r == ARCHIVE_EOF) 65 | break; 66 | if (r != ARCHIVE_OK) 67 | fail("archive_read_next_header()", 68 | archive_error_string(a), 1); 69 | if (do_extract) { 70 | archive_entry_set_perm(entry, 0777); 71 | r = archive_write_header(ext, entry); 72 | if (r != ARCHIVE_OK) 73 | warn("archive_write_header()", 74 | archive_error_string(ext)); 75 | else { 76 | copy_data(a, ext); 77 | r = archive_write_finish_entry(ext); 78 | if (r != ARCHIVE_OK) 79 | fail("archive_write_finish_entry()", 80 | archive_error_string(ext), 1); 81 | } 82 | 83 | } 84 | } 85 | archive_read_close(a); 86 | archive_read_free(a); 87 | archive_write_close(ext); 88 | archive_write_free(ext); 89 | } 90 | 91 | static int 92 | copy_data(struct archive *ar, struct archive *aw) 93 | { 94 | int r; 95 | const void *buff; 96 | size_t size; 97 | #if ARCHIVE_VERSION >= 3000000 98 | int64_t offset; 99 | #else 100 | off_t offset; 101 | #endif 102 | 103 | for (;;) { 104 | r = archive_read_data_block(ar, &buff, &size, &offset); 105 | if (r == ARCHIVE_EOF) 106 | return (ARCHIVE_OK); 107 | if (r != ARCHIVE_OK) 108 | return (r); 109 | r = archive_write_data_block(aw, buff, size, offset); 110 | if (r != ARCHIVE_OK) { 111 | warn("archive_write_data_block()", 112 | archive_error_string(aw)); 113 | return (r); 114 | } 115 | } 116 | } 117 | 118 | /* 119 | * * These reporting functions use low-level I/O; on some systems, this 120 | * * is a significant code reduction. Of course, on many server and 121 | * * desktop operating systems, malloc() and even crt rely on printf(), 122 | * * which in turn pulls in most of the rest of stdio, so this is not an 123 | * * optimization at all there. (If you're going to pay 100k or more 124 | * * for printf() anyway, you may as well use it!) 125 | * */ 126 | static void 127 | msg(const char *m) 128 | { 129 | write(1, m, strlen(m)); 130 | } 131 | 132 | static void 133 | errmsg(const char *m) 134 | { 135 | write(2, m, strlen(m)); 136 | } 137 | 138 | static void 139 | warn(const char *f, const char *m) 140 | { 141 | errmsg(f); 142 | errmsg(" failed: "); 143 | errmsg(m); 144 | errmsg("\n"); 145 | } 146 | 147 | static void 148 | fail(const char *f, const char *m, int r) 149 | { 150 | warn(f, m); 151 | exit(r); 152 | } 153 | 154 | static void 155 | usage(void) 156 | { 157 | const char *m = "Usage: untar [-tvx] [-f file] [file]\n"; 158 | errmsg(m); 159 | exit(1); 160 | } 161 | -------------------------------------------------------------------------------- /bcast.c: -------------------------------------------------------------------------------- 1 | /********************************************* 2 | * broadcasting pre-built packages to computing nodes 3 | * Author: Yu Feng 4 | * Contact: rainwoodman@gmail.com 5 | * 6 | * This file is part of python-mpi-bcast 7 | * 8 | *******************************************/ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | static int VERBOSE = 0; 19 | static int TIME = 0; 20 | 21 | struct fnlist { 22 | char * path; 23 | struct fnlist * next; 24 | } fnlist; 25 | 26 | static void 27 | _mkdir(const char *dir); 28 | 29 | static char * 30 | basename(const char * path); 31 | 32 | static void 33 | untar(char * dest, char * PREFIX); 34 | 35 | static int getnid(); 36 | 37 | static int ThisTask = 0; 38 | static int NodeRank = -1; 39 | MPI_Comm NODE_GROUPS; 40 | MPI_Comm NODE_LEADERS; 41 | 42 | static double t_bcast; 43 | static double t_tar; 44 | static double t_init; 45 | 46 | static void initialize(int nid) { 47 | MPI_Comm_rank(MPI_COMM_WORLD, &ThisTask); 48 | 49 | /* First split into ranks on the same node */ 50 | MPI_Comm_split(MPI_COMM_WORLD, nid, ThisTask, &NODE_GROUPS); 51 | 52 | MPI_Comm_rank(NODE_GROUPS, &NodeRank); 53 | 54 | /* Next split by Node Rank */ 55 | MPI_Comm_split(MPI_COMM_WORLD, NodeRank, ThisTask, &NODE_LEADERS); 56 | 57 | } 58 | 59 | static void bcast(char * src, char * PREFIX) { 60 | if(NodeRank != 0) { 61 | MPI_Barrier(MPI_COMM_WORLD); 62 | return; 63 | } 64 | 65 | double t1; 66 | long fsize; 67 | char *fcontent; 68 | char * dest = alloca(strlen(PREFIX) + 100); 69 | char * filename = basename(src); 70 | 71 | sprintf(dest, "%s/%s-%d", PREFIX, filename, ThisTask); 72 | 73 | free(filename); 74 | 75 | t1 = MPI_Wtime(); 76 | 77 | if(ThisTask == 0) { 78 | FILE * fp = fopen(src, "r"); 79 | if(fp == NULL) { 80 | fprintf(stderr, "package file %s not found\n", src); 81 | MPI_Abort(MPI_COMM_WORLD, 1); 82 | } 83 | fseek(fp, 0, SEEK_END); 84 | fsize = ftell(fp); 85 | fseek(fp, 0, SEEK_SET); 86 | 87 | fcontent = malloc(fsize + 1); 88 | fread(fcontent, 1, fsize, fp); 89 | fclose(fp); 90 | MPI_Bcast(&fsize, 1, MPI_LONG, 0, NODE_LEADERS); 91 | MPI_Bcast(fcontent, fsize, MPI_BYTE, 0, NODE_LEADERS); 92 | if(VERBOSE) { 93 | printf("Bcasting %s: %ld bytes\n", src, fsize); 94 | fflush(stdout); 95 | } 96 | } else { 97 | MPI_Bcast(&fsize, 1, MPI_LONG, 0, NODE_LEADERS); 98 | fcontent = malloc(fsize + 1); 99 | MPI_Bcast(fcontent, fsize, MPI_BYTE, 0, NODE_LEADERS); 100 | } 101 | 102 | MPI_Barrier(NODE_LEADERS); 103 | FILE * fp = fopen(dest, "w"); 104 | if(fp == NULL) { 105 | fprintf(stderr, "Cannot write to %s\n", dest); 106 | MPI_Abort(MPI_COMM_WORLD, 1); 107 | } 108 | fwrite(fcontent, 1, fsize, fp); 109 | fclose(fp); 110 | free(fcontent); 111 | 112 | t_bcast += MPI_Wtime() - t1; 113 | 114 | t1 = MPI_Wtime(); 115 | untar(dest, PREFIX); 116 | unlink(dest); 117 | 118 | MPI_Barrier(NODE_LEADERS); 119 | 120 | MPI_Barrier(MPI_COMM_WORLD); 121 | t_tar += MPI_Wtime() - t1; 122 | 123 | if(ThisTask == 0) { 124 | if(VERBOSE) { 125 | printf("Packages delivered. \n"); 126 | fflush(stdout); 127 | } 128 | } 129 | } 130 | 131 | static void 132 | untar(char * dest, char * PREFIX) 133 | { 134 | extern void 135 | extract(const char *filename, int do_extract); 136 | 137 | char cwd[1024]; 138 | getcwd(cwd, 1024); 139 | chdir(PREFIX); 140 | extract(dest, 1); 141 | chdir(cwd); 142 | 143 | /* 144 | char * cmd = alloca(strlen(dest) + strlen(PREFIX) + 100); 145 | if(!strcmp(dest + strlen(dest) - 3, "bz2")) { 146 | sprintf(cmd, "tar --overwrite -xjf \"%s\" -C \"%s\"", dest, PREFIX); 147 | } else 148 | if(!strcmp(dest + strlen(dest) - 3, "tar")) { 149 | sprintf(cmd, "tar --overwrite -xf \"%s\" -C \"%s\"", dest, PREFIX); 150 | } else 151 | if(!strcmp(dest + strlen(dest) - 2, "gz")) { 152 | sprintf(cmd, "tar --overwrite -xzf \"%s\" -C \"%s\"", dest, PREFIX); 153 | } else { 154 | sprintf(cmd, "cp \"%s\" \"%s\"", dest, PREFIX); 155 | } 156 | 157 | if(VERBOSE) { 158 | char hostname[1024]; 159 | gethostname(hostname, 1024); 160 | printf("%s: Running command: %s\n", hostname, cmd); 161 | fflush(stdout); 162 | } 163 | system(cmd); 164 | */ 165 | } 166 | 167 | static void 168 | process_file(char * filename, char * PREFIX) 169 | { 170 | char path[1024]; 171 | int nc = 0; 172 | if(ThisTask == 0) { 173 | FILE * fp = fopen(filename, "r"); 174 | if(fp == NULL) { 175 | fprintf(stderr, "failed to open package list %s.\n", filename); 176 | MPI_Abort(MPI_COMM_WORLD, 1); 177 | } 178 | while(!feof(fp)) { 179 | fgets(path, 1020, fp); 180 | nc = strlen(path); 181 | char * p = path + nc; 182 | for(p = path + nc - 1; p >= path; p --) { 183 | if(isspace(*p)) *p = 0; 184 | else break; 185 | } 186 | for(p = path; *p; p ++) { 187 | if(!isspace(*p)) break; 188 | } 189 | nc = strlen(p) + 1; 190 | if(nc == 1) continue; 191 | if(p[0] == '#') continue; 192 | /* fix \n */ 193 | MPI_Bcast(&nc, 1, MPI_INT, 0, MPI_COMM_WORLD); 194 | MPI_Bcast(p, nc, MPI_BYTE, 0, MPI_COMM_WORLD); 195 | bcast(p, PREFIX); 196 | } 197 | fclose(fp); 198 | nc = 0; 199 | MPI_Bcast(&nc, 1, MPI_INT, 0, MPI_COMM_WORLD); 200 | } else { 201 | while(1) { 202 | MPI_Bcast(&nc, 1, MPI_INT, 0, MPI_COMM_WORLD); 203 | if(nc == 0) break; 204 | MPI_Bcast(path, nc, MPI_BYTE, 0, MPI_COMM_WORLD); 205 | bcast(path, PREFIX); 206 | } 207 | } 208 | } 209 | 210 | int 211 | main(int argc, char **argv) 212 | { 213 | /* first allow everyone to purge files created by me */ 214 | umask(0); 215 | 216 | MPI_Init(&argc, &argv); 217 | double t0 = MPI_Wtime(); 218 | 219 | t_bcast = 0; 220 | t_tar = 0; 221 | t_init = 0; 222 | 223 | int nid = getnid(); 224 | initialize(nid); 225 | 226 | t_init = MPI_Wtime() - t0; 227 | 228 | t0 = MPI_Wtime(); 229 | 230 | int ch; 231 | extern char * optarg; 232 | extern int optind; 233 | char * PREFIX = "/dev/shm/python"; 234 | fnlist.next = NULL; 235 | 236 | while((ch = getopt(argc, argv, "vtf:p:")) != -1) { 237 | switch(ch) { 238 | case 'v': 239 | VERBOSE = 1; 240 | break; 241 | case 't': 242 | TIME = 1; 243 | break; 244 | case 'p': 245 | PREFIX = optarg; 246 | break; 247 | case 'f': 248 | { 249 | struct fnlist * p = malloc(sizeof(fnlist)); 250 | p->next = fnlist.next; 251 | fnlist.next = p; 252 | p->path = strdup(optarg); 253 | } 254 | break; 255 | case '?': 256 | if(ThisTask == 0) { 257 | fprintf(stderr, "usage: bcast [-v] [-f filelist] [-p /dev/shm/python] [packages ...]\n"); 258 | } 259 | goto quit; 260 | } 261 | } 262 | 263 | if(ThisTask == 0) { 264 | if(VERBOSE) { 265 | printf("tmpdir:%s\n", PREFIX); 266 | fflush(stdout); 267 | } 268 | } 269 | 270 | if(NodeRank == 0) { 271 | _mkdir(PREFIX); 272 | } 273 | 274 | MPI_Barrier(MPI_COMM_WORLD); 275 | 276 | struct fnlist * p; 277 | for(p = fnlist.next; p ; p = p->next) { 278 | process_file(p->path, PREFIX); 279 | } 280 | int i; 281 | for(i = optind; i < argc; i ++) { 282 | bcast(argv[i], PREFIX); 283 | } 284 | 285 | if(ThisTask == 0) 286 | if(TIME) { 287 | printf("Time : %g in init\n", t_init); 288 | printf("Time : %g in bcast\n", t_bcast); 289 | printf("Time : %g in tar\n", t_tar); 290 | printf("Time : %g in total \n", MPI_Wtime() - t0); 291 | } 292 | quit: 293 | MPI_Barrier(MPI_COMM_WORLD); 294 | 295 | MPI_Finalize(); 296 | return 0; 297 | } 298 | 299 | static void 300 | _mkdir(const char *dir) 301 | { 302 | char * tmp = strdup(dir); 303 | char *p = NULL; 304 | size_t len; 305 | 306 | len = strlen(tmp); 307 | if(tmp[len - 1] == '/') 308 | tmp[len - 1] = 0; 309 | for(p = tmp + 1; *p; p++) 310 | if(*p == '/') { 311 | *p = 0; 312 | mkdir(tmp, 0777); 313 | *p = '/'; 314 | } 315 | mkdir(tmp, 0777); 316 | free(tmp); 317 | } 318 | 319 | static char * 320 | basename(const char * path) { 321 | const char * p = path + strlen(path); 322 | while(p >= path) { 323 | if(*p == '/') { 324 | break; 325 | } 326 | p--; 327 | } 328 | return strdup(p+1); 329 | } 330 | 331 | static int getnid() { 332 | char hostname[1024]; 333 | int i; 334 | gethostname(hostname, 1024); 335 | 336 | MPI_Barrier(MPI_COMM_WORLD); 337 | 338 | int l = strlen(hostname) + 4; 339 | int ml = 0; 340 | int NTask; 341 | int ThisTask; 342 | char * buffer; 343 | int * nid; 344 | MPI_Comm_size(MPI_COMM_WORLD, &NTask); 345 | MPI_Comm_rank(MPI_COMM_WORLD, &ThisTask); 346 | MPI_Allreduce(&l, &ml, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); 347 | 348 | buffer = malloc(ml * NTask); 349 | nid = malloc(sizeof(int) * NTask); 350 | MPI_Allgather(hostname, ml, MPI_BYTE, buffer, ml, MPI_BYTE, MPI_COMM_WORLD); 351 | 352 | typedef int(*compar_fn)(const void *, const void *); 353 | qsort(buffer, NTask, ml, (compar_fn) strcmp); 354 | 355 | nid[0] = 0; 356 | for(i = 1; i < NTask; i ++) { 357 | if(strcmp(buffer + i * ml, buffer + (i - 1) *ml)) { 358 | nid[i] = nid[i - 1] + 1; 359 | } else { 360 | nid[i] = nid[i - 1]; 361 | } 362 | } 363 | for(i = 0; i < NTask; i ++) { 364 | if(!strcmp(hostname, buffer + i * ml)) { 365 | break; 366 | } 367 | } 368 | int rt = nid[i]; 369 | free(buffer); 370 | free(nid); 371 | MPI_Barrier(MPI_COMM_WORLD); 372 | return rt; 373 | } 374 | 375 | -------------------------------------------------------------------------------- /benchmarks/anaconda/dobench.sh: -------------------------------------------------------------------------------- 1 | export PYTHONPATH= 2 | export PYTHONUSERBASE=/tmp 3 | 4 | PYTHON=$1 5 | for i in `seq 20`; do 6 | mkdir -p /tmp/tmpdir-$i 7 | export PYTHONPATH=/tmp/tmpdir-$i:$PYTHONPATH 8 | $PYTHON -c 'import sys;print(len(sys.path))' 9 | strace -ff -e file $PYTHON -c 'print(0)' |& wc -l 10 | strace -ff -e file $PYTHON -c 'import numpy' |& wc -l 11 | strace -ff -e file $PYTHON -c 'import scipy' |& wc -l 12 | strace -ff -e file $PYTHON -c 'import numba' |& wc -l 13 | strace -ff -e file $PYTHON -c 'import matplotlib' |& wc -l 14 | done 15 | -------------------------------------------------------------------------------- /benchmarks/anaconda/plot.py: -------------------------------------------------------------------------------- 1 | from matplotlib.backends.backend_agg import FigureCanvasAgg 2 | from matplotlib.figure import Figure 3 | import numpy 4 | 5 | data2 = numpy.loadtxt('bench-log2.txt').reshape(-1, 6).T 6 | data3 = numpy.loadtxt('bench-log3.txt').reshape(-1, 6).T 7 | print data2.shape, data3.shape 8 | f = Figure(figsize=(5, 4)) 9 | ax = f.add_subplot(111) 10 | l = [] 11 | for i in range(5): 12 | ll, = ax.plot(data2[0]- data2[0][0] + 1, data2[i+1]) 13 | ax.plot(data3[0] - data3[0][0] + 1, data3[i+1], color=ll.get_color(), ls='--') 14 | l.append(ll) 15 | ax.legend( 16 | l, 17 | ['Bare Python', 'import numpy', 'import scipy', 'import numba', 'import matplotlib'], 18 | fontsize='small', frameon=False, loc='upper left') 19 | 20 | ax.set_ylabel("File-System Operations") 21 | ax.set_xlabel("Length of PYTHONPATH") 22 | canvas = FigureCanvasAgg(f) 23 | f.tight_layout() 24 | f.savefig('python-file-ops.png', dpi=200) 25 | f.savefig('python-file-ops.pdf', dpi=200) 26 | -------------------------------------------------------------------------------- /benchmarks/anaconda/python-file-ops.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainwoodman/python-mpi-bcast/962c724b7b22bd914c5552991fdd3d8c8caddf08/benchmarks/anaconda/python-file-ops.png -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-1.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 1' 6 | ++ APRUN='srun -N 1' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 1 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 0.433365 in bcast 32 | Time : 7.58505 in tar 33 | Time : 0.0590911 in chmod 34 | 35 | real 0m11.083s 36 | user 0m0.168s 37 | sys 0m0.020s 38 | + cd /dev/shm/local 39 | + srun -N 1 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m8.235s 42 | user 0m0.180s 43 | sys 0m0.020s 44 | + finish 45 | + srun -N 1 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-1024.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 1024' 6 | ++ APRUN='srun -N 1024' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 1024 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 1.92552 in bcast 32 | Time : 8.9971 in tar 33 | Time : 0.267918 in chmod 34 | 35 | real 0m28.967s 36 | user 0m0.496s 37 | sys 0m0.288s 38 | + cd /dev/shm/local 39 | + srun -N 1024 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m26.954s 42 | user 0m1.804s 43 | sys 0m0.360s 44 | + finish 45 | + srun -N 1024 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-128.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 128' 6 | ++ APRUN='srun -N 128' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 128 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 1.3127 in bcast 32 | Time : 8.05013 in tar 33 | Time : 0.0731061 in chmod 34 | 35 | real 0m17.752s 36 | user 0m0.192s 37 | sys 0m0.056s 38 | + cd /dev/shm/local 39 | + srun -N 128 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m13.238s 42 | user 0m0.296s 43 | sys 0m0.084s 44 | + finish 45 | + srun -N 128 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-16.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 16' 6 | ++ APRUN='srun -N 16' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 16 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 0.87579 in bcast 32 | Time : 7.66913 in tar 33 | Time : 0.058095 in chmod 34 | 35 | real 0m14.416s 36 | user 0m0.176s 37 | sys 0m0.028s 38 | + cd /dev/shm/local 39 | + srun -N 16 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m11.632s 42 | user 0m0.180s 43 | sys 0m0.056s 44 | + finish 45 | + srun -N 16 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-2.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 2' 6 | ++ APRUN='srun -N 2' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 2 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 0.478364 in bcast 32 | Time : 7.6122 in tar 33 | Time : 0.0615981 in chmod 34 | 35 | real 0m12.015s 36 | user 0m0.172s 37 | sys 0m0.016s 38 | + cd /dev/shm/local 39 | + srun -N 2 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m8.891s 42 | user 0m0.176s 43 | sys 0m0.036s 44 | + finish 45 | + srun -N 2 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-2048.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 2048' 6 | ++ APRUN='srun -N 2048' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 2048 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 3.43116 in bcast 32 | Time : 31.4195 in tar 33 | Time : 6.36747 in chmod 34 | 35 | real 1m10.367s 36 | user 0m0.640s 37 | sys 0m0.524s 38 | + cd /dev/shm/local 39 | + srun -N 2048 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 1m55.430s 42 | user 0m7.148s 43 | sys 0m1.908s 44 | + finish 45 | + srun -N 2048 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-256.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 256' 6 | ++ APRUN='srun -N 256' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 256 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 1.4895 in bcast 32 | Time : 9.56686 in tar 33 | Time : 0.0968668 in chmod 34 | 35 | real 0m18.966s 36 | user 0m0.236s 37 | sys 0m0.096s 38 | + cd /dev/shm/local 39 | + srun -N 256 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m15.174s 42 | user 0m0.440s 43 | sys 0m0.112s 44 | + finish 45 | + srun -N 256 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-32.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 32' 6 | ++ APRUN='srun -N 32' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 32 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 1.06822 in bcast 32 | Time : 7.70731 in tar 33 | Time : 0.0579579 in chmod 34 | 35 | real 0m14.794s 36 | user 0m0.156s 37 | sys 0m0.052s 38 | + cd /dev/shm/local 39 | + srun -N 32 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m9.379s 42 | user 0m0.192s 43 | sys 0m0.072s 44 | + finish 45 | + srun -N 32 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-4.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 4' 6 | ++ APRUN='srun -N 4' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 4 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 0.617725 in bcast 32 | Time : 7.68573 in tar 33 | Time : 0.0608499 in chmod 34 | 35 | real 0m14.111s 36 | user 0m0.172s 37 | sys 0m0.016s 38 | + cd /dev/shm/local 39 | + srun -N 4 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m11.408s 42 | user 0m0.180s 43 | sys 0m0.024s 44 | + finish 45 | + srun -N 4 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-4096.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 4096' 6 | ++ APRUN='srun -N 4096' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 4096 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 2.167 in bcast 32 | Time : 11.1573 in tar 33 | Time : 0.808127 in chmod 34 | 35 | real 0m48.583s 36 | user 0m1.184s 37 | sys 0m0.984s 38 | + cd /dev/shm/local 39 | + srun -N 4096 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 1m25.281s 42 | user 0m19.817s 43 | sys 0m0.936s 44 | + finish 45 | + srun -N 4096 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-512.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 512' 6 | ++ APRUN='srun -N 512' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 512 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 1.69665 in bcast 32 | Time : 7.95301 in tar 33 | Time : 0.152223 in chmod 34 | 35 | real 0m20.977s 36 | user 0m0.324s 37 | sys 0m0.164s 38 | + cd /dev/shm/local 39 | + srun -N 512 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m19.202s 42 | user 0m0.724s 43 | sys 0m0.164s 44 | + finish 45 | + srun -N 512 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-5310.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 5310' 6 | ++ APRUN='srun -N 5310' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 5310 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 2.15751 in bcast 32 | Time : 43.1168 in tar 33 | Time : 1.19226 in chmod 34 | 35 | real 1m27.233s 36 | user 0m1.300s 37 | sys 0m1.316s 38 | + cd /dev/shm/local 39 | + srun -N 5310 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 2m8.020s 42 | user 0m33.514s 43 | sys 0m1.392s 44 | + finish 45 | + srun -N 5310 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-64.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 64' 6 | ++ APRUN='srun -N 64' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 64 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 1.19269 in bcast 32 | Time : 7.7162 in tar 33 | Time : 0.0620041 in chmod 34 | 35 | real 0m17.016s 36 | user 0m0.188s 37 | sys 0m0.032s 38 | + cd /dev/shm/local 39 | + srun -N 64 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m12.196s 42 | user 0m0.220s 43 | sys 0m0.076s 44 | + finish 45 | + srun -N 64 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench-8.out: -------------------------------------------------------------------------------- 1 | + export OMP_NUM_THREADS=1 2 | + OMP_NUM_THREADS=1 3 | + export ATP_ENABLED=0 4 | + ATP_ENABLED=0 5 | + source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local 'srun -N 8' 6 | ++ APRUN='srun -N 8' 7 | ++ BCASTROOT=/dev/shm/local 8 | ++ [[ -n 3.2.51(1)-release ]] 9 | ++ _SCRIPT_LOCATION=/project/projectdirs/m779/python-mpi/activate.sh 10 | +++ dirname /project/projectdirs/m779/python-mpi/activate.sh 11 | ++ DIRNAME=/project/projectdirs/m779/python-mpi 12 | ++ trap finish EXIT 13 | ++ trap finish TERM 14 | ++ trap finish KILL 15 | ++ export PYTHONPATH=/dev/shm/local/lib/python 16 | ++ PYTHONPATH=/dev/shm/local/lib/python 17 | ++ export PYTHONHOME=/dev/shm/local 18 | ++ PYTHONHOME=/dev/shm/local 19 | ++ export PYTHONUSERBASE=/dev/shm/local 20 | ++ PYTHONUSERBASE=/dev/shm/local 21 | ++ OLD_LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 22 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 23 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/opt/gcc/5.2.0/snos/lib64:/usr/syscom/nsg/lib 24 | ++ export PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 25 | ++ PATH=/dev/shm/local/bin:/project/projectdirs/m779/yfeng1/local-edison/bin:/project/projectdirs/m779/yfeng1/envs/python-2.7-edison/bin:/usr/common/usg/python/2.7-anaconda/lib/python2.7/site-packages/mpi4py/bin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/sbin:/opt/cray/alps/5.2.3-2.0502.9295.14.14.ari/bin:/opt/cray/dvs/2.5_0.9.0-1.0502.1958.2.55.ari/bin:/opt/cray/xpmem/0.1-2.0502.57015.1.15.ari/bin:/opt/cray/pmi/5.0.10-1.0000.11050.0.0.ari/bin:/opt/cray/ugni/6.0-1.0502.10245.9.9.ari/bin:/opt/cray/udreg/2.3.2-1.0502.9889.2.20.ari/bin:/opt/gcc/5.2.0/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/nsg/bin:/opt/slurm/default/bin:/opt/cray/mpt/7.3.0/gni/bin:/opt/cray/craype/2.5.0/bin:/opt/cray/switch/1.0-1.0502.57058.1.58.ari/bin:/opt/cray/eslogin/eswrap/1.1.0-1.020200.1130.0/bin:/usr/syscom/nsg/sbin:/usr/syscom/nsg/bin:/opt/modules/3.2.10.3/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:/usr/lib/qt3/bin:/opt/cray/bin 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ hash -r 28 | + bcast -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 29 | + LD_LIBRARY_PATH= 30 | + srun -N 8 /project/projectdirs/m779/python-mpi/bcast -p /dev/shm/local -t /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 31 | Time : 0.768337 in bcast 32 | Time : 7.62392 in tar 33 | Time : 0.0553541 in chmod 34 | 35 | real 0m14.159s 36 | user 0m0.172s 37 | sys 0m0.028s 38 | + cd /dev/shm/local 39 | + srun -N 8 --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 40 | 41 | real 0m11.536s 42 | user 0m0.184s 43 | sys 0m0.028s 44 | + finish 45 | + srun -N 8 rm -rf /dev/shm/local 46 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/bench.template: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #SBATCH -N @N@ 4 | #SBATCH -o bench-@N@.out 5 | #SBATCH -t 00:10:00 6 | 7 | set -x 8 | export OMP_NUM_THREADS=1 9 | export ATP_ENABLED=0 10 | 11 | source /project/projectdirs/m779/python-mpi/activate.sh /dev/shm/local "srun -N @N@" 12 | 13 | # send the anaconda packages 14 | time bcast -t \ 15 | /project/projectdirs/m779/python-mpi/cray-xc30/cray-xc30.tar.gz \ 16 | /project/projectdirs/m779/yfeng1/python-2.7-edison.tar.gz 17 | 18 | cd /dev/shm/local 19 | time srun -N @N@ --ntasks-per-node=24 python-mpi -c 'import scipy;from mpi4py import MPI;' 20 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/collect.sh: -------------------------------------------------------------------------------- 1 | cat >process.awk < bench-$1.slurm 3 | sbatch -p regular bench-$1.slurm 4 | -------------------------------------------------------------------------------- /benchmarks/cray-xc30/tar-cray-xc30.slurm: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #SBATCH -N 1 4 | #SBATCH -p regular 5 | #SBATCH -t 00:10:00 6 | 7 | srun -n 1 strace python-mpi -c 'from mpi4py import MPI' &> xc30.trace 8 | 9 | grep "= 3$" xc30.trace | grep so | sed -s 's;open(";;' | sed -s 's;".*;;' > filelist 10 | 11 | rm -rf lib 12 | mkdir -p lib 13 | srun -n 1 cp `cat filelist` lib/ 14 | tar -czvf cray-xc30.tar.gz lib/ 15 | -------------------------------------------------------------------------------- /benchmarks/cray-xc40/tar-cray-xc40.slurm: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #SBATCH -N 1 4 | #SBATCH -p debug 5 | #SBATCH -r 00:10:00 6 | 7 | srun -n 1 strace python-mpi -c 'from mpi4py import MPI' &> xc40.trace 8 | 9 | grep "= 3$" xc40.trace | grep so | sed -s 's;open(";;' | sed -s 's;".*;;' > filelist 10 | 11 | rm -rf lib 12 | mkdir -p lib 13 | 14 | srun -n 1 cp `cat filelist` lib/ 15 | tar -czvf cray-xc40.tar.gz lib/ 16 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-1.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25339 3 | at Thu Jan 14 20:29:18 CST 2016 4 | Job Id: 3415204.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-1.job 8 | Requested resources: neednodes=1:ppn=32,nodes=1:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.027 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 32' 24 | ++ APRUN='aprun -n 32' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 32 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 1.03708 in bcast 50 | Time : 7.72704 in tar 51 | Time : 8.76577 in total 52 | Application 31661310 resources: utime ~276s, stime ~52s, Rss ~261064, inblocks ~1058339, outblocks ~2368031 53 | 54 | real 0m14.524s 55 | user 0m0.444s 56 | sys 0m0.124s 57 | + cd /dev/shm 58 | + aprun -n 32 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661344 resources: utime ~28s, stime ~49s, Rss ~27576, inblocks ~196586, outblocks ~135062 60 | 61 | real 0m6.863s 62 | user 0m0.064s 63 | sys 0m0.080s 64 | + finish 65 | + aprun -n 32 rm -rf /dev/shm/local 66 | Application 31661346 resources: utime ~0s, stime ~48s, Rss ~21480, inblocks ~156, outblocks ~109 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-1024.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25329 3 | at Thu Jan 14 22:28:05 CST 2016 4 | Job Id: 3415335.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-1024.job 8 | Requested resources: neednodes=1024:ppn=32,nodes=1024:ppn=32,walltime=00:10:00 9 | Queue: low 10 | Account: jp6 11 | End Torque Prologue: 0.030 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 32768' 24 | ++ APRUN='aprun -n 32768' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 32768 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 4.35499 in bcast 50 | Time : 7.75117 in tar 51 | Time : 18.7247 in total 52 | Application 31664114 resources: utime ~517575s, stime ~223292s, Rss ~330504, inblocks ~545254006, outblocks ~2424971077 53 | 54 | real 0m38.965s 55 | user 0m0.460s 56 | sys 0m0.148s 57 | + cd /dev/shm 58 | + aprun -n 32768 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31664209 resources: utime ~134491s, stime ~170528s, Rss ~91256, inblocks ~206142566, outblocks ~136698486 60 | 61 | real 0m27.830s 62 | user 0m0.084s 63 | sys 0m0.044s 64 | + finish 65 | + aprun -n 32768 rm -rf /dev/shm/local 66 | Application 31664273 resources: utime ~0s, stime ~38023s, Rss ~21480, inblocks ~159744, outblocks ~111616 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-128.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25350 3 | at Thu Jan 14 20:28:02 CST 2016 4 | Job Id: 3415215.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-128.job 8 | Requested resources: neednodes=128:ppn=32,nodes=128:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.028 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 4096' 24 | ++ APRUN='aprun -n 4096' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 4096 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 3.33282 in bcast 50 | Time : 7.67622 in tar 51 | Time : 17.6183 in total 52 | Application 31661215 resources: utime ~57316s, stime ~28170s, Rss ~274268, inblocks ~68476412, outblocks ~303121340 53 | 54 | real 0m36.475s 55 | user 0m0.468s 56 | sys 0m0.112s 57 | + cd /dev/shm 58 | + aprun -n 4096 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661287 resources: utime ~2520s, stime ~15172s, Rss ~39560, inblocks ~25727355, outblocks ~17621485 60 | 61 | real 0m20.528s 62 | user 0m0.072s 63 | sys 0m0.104s 64 | + finish 65 | + aprun -n 4096 rm -rf /dev/shm/local 66 | Application 31661299 resources: utime ~0s, stime ~5537s, Rss ~21480, inblocks ~19968, outblocks ~13952 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-16.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25430 3 | at Thu Jan 14 20:28:11 CST 2016 4 | Job Id: 3415210.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-16.job 8 | Requested resources: neednodes=16:ppn=32,nodes=16:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.052 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 512' 24 | ++ APRUN='aprun -n 512' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 512 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 2.37899 in bcast 50 | Time : 7.62238 in tar 51 | Time : 10.0052 in total 52 | Application 31661224 resources: utime ~3600s, stime ~3167s, Rss ~275004, inblocks ~9027326, outblocks ~37890065 53 | 54 | real 0m26.157s 55 | user 0m0.440s 56 | sys 0m0.152s 57 | + cd /dev/shm 58 | + aprun -n 512 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661285 resources: utime ~268s, stime ~1878s, Rss ~33184, inblocks ~3216181, outblocks ~2203300 60 | 61 | real 0m19.862s 62 | user 0m0.064s 63 | sys 0m0.072s 64 | + finish 65 | + aprun -n 512 rm -rf /dev/shm/local 66 | Application 31661298 resources: utime ~0s, stime ~737s, Rss ~21480, inblocks ~2496, outblocks ~1744 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-2.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25435 3 | at Thu Jan 14 20:28:22 CST 2016 4 | Job Id: 3415206.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-2.job 8 | Requested resources: neednodes=2:ppn=32,nodes=2:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.028 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 64' 24 | ++ APRUN='aprun -n 64' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 64 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 1.33602 in bcast 50 | Time : 7.79275 in tar 51 | Time : 9.1316 in total 52 | Application 31661235 resources: utime ~389s, stime ~369s, Rss ~273100, inblocks ~1588713, outblocks ~4736189 53 | 54 | real 0m24.511s 55 | user 0m0.448s 56 | sys 0m0.144s 57 | + cd /dev/shm 58 | + aprun -n 64 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661290 resources: utime ~28s, stime ~212s, Rss ~32356, inblocks ~397489, outblocks ~270635 60 | 61 | real 0m15.198s 62 | user 0m0.052s 63 | sys 0m0.060s 64 | + finish 65 | + aprun -n 64 rm -rf /dev/shm/local 66 | Application 31661300 resources: utime ~0s, stime ~81s, Rss ~21480, inblocks ~312, outblocks ~218 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-2048.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25328 3 | at Thu Jan 14 22:28:03 CST 2016 4 | Job Id: 3415336.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-2048.job 8 | Requested resources: neednodes=2048:ppn=32,nodes=2048:ppn=32,walltime=00:10:00 9 | Queue: low 10 | Account: jp6 11 | End Torque Prologue: 0.029 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 65536' 24 | ++ APRUN='aprun -n 65536' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 65536 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 4.90292 in bcast 50 | Time : 7.80896 in tar 51 | Time : 19.2645 in total 52 | Application 31664112 resources: utime ~1082787s, stime ~445650s, Rss ~386568, inblocks ~1088974607, outblocks ~4849942267 53 | 54 | real 0m42.552s 55 | user 0m0.484s 56 | sys 0m0.116s 57 | + cd /dev/shm 58 | + aprun -n 65536 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31664214 resources: utime ~264288s, stime ~333971s, Rss ~153988, inblocks ~412149243, outblocks ~269376767 60 | 61 | real 0m29.488s 62 | user 0m0.088s 63 | sys 0m0.092s 64 | + finish 65 | + aprun -n 65536 rm -rf /dev/shm/local 66 | Application 31664282 resources: utime ~0s, stime ~52345s, Rss ~21480, inblocks ~319488, outblocks ~223232 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-256.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25348 3 | at Thu Jan 14 20:27:54 CST 2016 4 | Job Id: 3415217.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-256.job 8 | Requested resources: neednodes=256:ppn=32,nodes=256:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.029 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 8192' 24 | ++ APRUN='aprun -n 8192' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 8192 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 3.67547 in bcast 50 | Time : 8.27857 in tar 51 | Time : 11.9836 in total 52 | Application 31661212 resources: utime ~73275s, stime ~51573s, Rss ~288516, inblocks ~136738345, outblocks ~606242713 53 | 54 | real 0m30.532s 55 | user 0m0.460s 56 | sys 0m0.168s 57 | + cd /dev/shm 58 | + aprun -n 8192 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661249 resources: utime ~6053s, stime ~30850s, Rss ~46536, inblocks ~51476819, outblocks ~35246588 60 | 61 | real 0m21.315s 62 | user 0m0.084s 63 | sys 0m0.028s 64 | + finish 65 | + aprun -n 8192 rm -rf /dev/shm/local 66 | Application 31661292 resources: utime ~0s, stime ~10511s, Rss ~21480, inblocks ~39936, outblocks ~27904 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-32.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25429 3 | at Thu Jan 14 20:28:09 CST 2016 4 | Job Id: 3415212.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-32.job 8 | Requested resources: neednodes=32:ppn=32,nodes=32:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.028 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 1024' 24 | ++ APRUN='aprun -n 1024' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 1024 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 2.87929 in bcast 50 | Time : 7.60851 in tar 51 | Time : 10.4983 in total 52 | Application 31661222 resources: utime ~7671s, stime ~6259s, Rss ~268772, inblocks ~17528257, outblocks ~75780273 53 | 54 | real 0m25.840s 55 | user 0m0.456s 56 | sys 0m0.112s 57 | + cd /dev/shm 58 | + aprun -n 1024 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661268 resources: utime ~504s, stime ~3544s, Rss ~34216, inblocks ~6429190, outblocks ~4398802 60 | 61 | real 0m20.630s 62 | user 0m0.064s 63 | sys 0m0.052s 64 | + finish 65 | + aprun -n 1024 rm -rf /dev/shm/local 66 | Application 31661296 resources: utime ~0s, stime ~1358s, Rss ~21480, inblocks ~4992, outblocks ~3488 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-4.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25433 3 | at Thu Jan 14 20:28:22 CST 2016 4 | Job Id: 3415207.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-4.job 8 | Requested resources: neednodes=4:ppn=32,nodes=4:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.028 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 128' 24 | ++ APRUN='aprun -n 128' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 128 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 1.6674 in bcast 50 | Time : 7.77316 in tar 51 | Time : 9.44329 in total 52 | Application 31661234 resources: utime ~910s, stime ~779s, Rss ~273952, inblocks ~2651529, outblocks ~9472440 53 | 54 | real 0m26.005s 55 | user 0m0.464s 56 | sys 0m0.124s 57 | + cd /dev/shm 58 | + aprun -n 128 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661293 resources: utime ~62s, stime ~445s, Rss ~32476, inblocks ~802977, outblocks ~537850 60 | 61 | real 0m15.611s 62 | user 0m0.072s 63 | sys 0m0.044s 64 | + finish 65 | + aprun -n 128 rm -rf /dev/shm/local 66 | Application 31661302 resources: utime ~0s, stime ~176s, Rss ~21480, inblocks ~624, outblocks ~436 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-4096.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25262 3 | at Thu Jan 14 21:39:15 CST 2016 4 | Job Id: 3415337.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-4096.job 8 | Requested resources: neednodes=4096:ppn=32,nodes=4096:ppn=32,walltime=00:10:00 9 | Queue: low 10 | Account: jp6 11 | End Torque Prologue: 0.030 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 131072' 24 | ++ APRUN='aprun -n 131072' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 131072 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 5.20081 in bcast 50 | Time : 8.08294 in tar 51 | Time : 19.4329 in total 52 | Application 31663097 resources: utime ~2446987s, stime ~930707s, Rss ~496128, inblocks ~2175466023, outblocks ~9699887967 53 | 54 | real 0m44.914s 55 | user 0m0.540s 56 | sys 0m0.120s 57 | + cd /dev/shm 58 | + aprun -n 131072 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31663168 resources: utime ~405269s, stime ~628876s, Rss ~278076, inblocks ~829013944, outblocks ~413120989 60 | 61 | real 0m42.680s 62 | user 0m0.112s 63 | sys 0m0.056s 64 | + finish 65 | + aprun -n 131072 rm -rf /dev/shm/local 66 | Application 31663241 resources: utime ~0s, stime ~95126s, Rss ~21480, inblocks ~638976, outblocks ~446464 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-512.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25339 3 | at Thu Jan 14 20:27:52 CST 2016 4 | Job Id: 3415219.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-512.job 8 | Requested resources: neednodes=512:ppn=32,nodes=512:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.029 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 16384' 24 | ++ APRUN='aprun -n 16384' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 16384 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 3.961 in bcast 50 | Time : 7.66121 in tar 51 | Time : 18.0315 in total 52 | Application 31661209 resources: utime ~240492s, stime ~113425s, Rss ~302148, inblocks ~271944873, outblocks ~1212485590 53 | 54 | real 0m34.767s 55 | user 0m0.472s 56 | sys 0m0.100s 57 | + cd /dev/shm 58 | + aprun -n 16384 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661255 resources: utime ~15391s, stime ~65681s, Rss ~60840, inblocks ~103007312, outblocks ~70657725 60 | 61 | real 0m20.853s 62 | user 0m0.080s 63 | sys 0m0.040s 64 | + finish 65 | + aprun -n 16384 rm -rf /dev/shm/local 66 | Application 31661294 resources: utime ~0s, stime ~22393s, Rss ~21480, inblocks ~79872, outblocks ~55808 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-64.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25427 3 | at Thu Jan 14 20:28:05 CST 2016 4 | Job Id: 3415214.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-64.job 8 | Requested resources: neednodes=64:ppn=32,nodes=64:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.051 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 2048' 24 | ++ APRUN='aprun -n 2048' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 2048 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 6.31153 in bcast 50 | Time : 7.62201 in tar 51 | Time : 13.9463 in total 52 | Application 31661217 resources: utime ~21885s, stime ~12710s, Rss ~270672, inblocks ~34400321, outblocks ~151560606 53 | 54 | real 0m30.062s 55 | user 0m0.440s 56 | sys 0m0.128s 57 | + cd /dev/shm 58 | + aprun -n 2048 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661274 resources: utime ~1123s, stime ~6832s, Rss ~36016, inblocks ~12858570, outblocks ~8848645 60 | 61 | real 0m21.155s 62 | user 0m0.060s 63 | sys 0m0.056s 64 | + finish 65 | + aprun -n 2048 rm -rf /dev/shm/local 66 | Application 31661297 resources: utime ~0s, stime ~2336s, Rss ~21480, inblocks ~9984, outblocks ~6976 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-8.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25431 3 | at Thu Jan 14 20:28:13 CST 2016 4 | Job Id: 3415208.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-8.job 8 | Requested resources: neednodes=8:ppn=32,nodes=8:ppn=32,walltime=00:10:00 9 | Queue: normal 10 | Account: jp6 11 | End Torque Prologue: 0.039 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 256' 24 | ++ APRUN='aprun -n 256' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 256 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 1.99504 in bcast 50 | Time : 12.1929 in tar 51 | Time : 14.1914 in total 52 | Application 31661226 resources: utime ~2692s, stime ~1568s, Rss ~274264, inblocks ~4770987, outblocks ~18945003 53 | 54 | real 0m34.624s 55 | user 0m0.460s 56 | sys 0m0.132s 57 | + cd /dev/shm 58 | + aprun -n 256 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31661291 resources: utime ~125s, stime ~910s, Rss ~32884, inblocks ~1602099, outblocks ~1136251 60 | 61 | real 0m15.697s 62 | user 0m0.060s 63 | sys 0m0.056s 64 | + finish 65 | + aprun -n 256 rm -rf /dev/shm/local 66 | Application 31661301 resources: utime ~0s, stime ~343s, Rss ~21480, inblocks ~1248, outblocks ~872 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench-8192.out: -------------------------------------------------------------------------------- 1 | ---------------------------------------- 2 | Begin Torque Prologue on nid25258 3 | at Sat Jan 16 21:09:37 CST 2016 4 | Job Id: 3415338.nid11293 5 | Username: yfeng1 6 | Group: PRAC_jp6 7 | Job name: bench-8192.job 8 | Requested resources: neednodes=8192:ppn=32,nodes=8192:ppn=32,walltime=00:10:00 9 | Queue: low 10 | Account: jp6 11 | End Torque Prologue: 0.035 elapsed 12 | ---------------------------------------- 13 | 14 | 15 | + export OMP_NUM_THREADS=1 16 | + OMP_NUM_THREADS=1 17 | + export ATP_ENABLED=0 18 | + ATP_ENABLED=0 19 | + export PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 20 | + PATH=/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 21 | + export LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 22 | + LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 23 | + source /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh /dev/shm/local 'aprun -n 262144' 24 | ++ APRUN='aprun -n 262144' 25 | ++ BCASTROOT=/dev/shm/local 26 | ++ [[ -n 3.2.51(1)-release ]] 27 | ++ _SCRIPT_LOCATION=/u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 28 | +++ dirname /u/sciteam/yfeng1/source/python-mpi-bcast/activate.sh 29 | ++ DIRNAME=/u/sciteam/yfeng1/source/python-mpi-bcast 30 | ++ trap finish EXIT 31 | ++ trap finish TERM 32 | ++ trap finish KILL 33 | ++ export PYTHONPATH=/dev/shm/local/lib/python 34 | ++ PYTHONPATH=/dev/shm/local/lib/python 35 | ++ export PYTHONHOME=/dev/shm/local 36 | ++ PYTHONHOME=/dev/shm/local 37 | ++ export PYTHONUSERBASE=/dev/shm/local 38 | ++ PYTHONUSERBASE=/dev/shm/local 39 | ++ OLD_LD_LIBRARY_PATH=/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 40 | ++ export LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 41 | ++ LD_LIBRARY_PATH=/dev/shm/local/lib:/u/sciteam/yfeng1/anaconda2/lib:/u/sciteam/yfeng1/anaconda2/lib:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/lib 42 | ++ export PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 43 | ++ PATH=/dev/shm/local/bin:/u/sciteam/yfeng1/anaconda2/bin:/u/sciteam/yfeng1/anaconda2/bin:/sw/xe/darshan/2.3.0/darshan-2.3.0_cle52/bin:/sw/xe/altd/bin:/sw/admin/scripts:/sw/user/scripts:/opt/moab/8.1/bin:/opt/moab/8.1/sbin:/opt/torque/5.0.2-bwpatch/sbin:/opt/torque/5.0.2-bwpatch/bin:/opt/cray/mpt/7.2.0/gni/bin:/opt/cray/rca/1.0.0-2.0502.53711.3.125.gem/bin:/opt/cray/pmi/5.0.6-1.0000.10439.140.3.gem/bin:/opt/cray/cce/8.3.10/cray-binutils/x86_64-unknown-linux-gnu/bin:/opt/cray/cce/8.3.10/craylibs/x86-64/bin:/opt/cray/cce/8.3.10/cftn/bin:/opt/cray/cce/8.3.10/CC/bin:/opt/cray/craype/2.3.0/bin:/opt/cray/llm/default/bin:/opt/cray/llm/default/etc:/opt/cray/xpmem/0.1-2.0502.55507.3.2.gem/bin:/opt/cray/dmapp/7.0.1-1.0502.9501.5.211.gem/bin:/opt/cray/ugni/5.0-1.0502.9685.4.24.gem/bin:/opt/cray/udreg/2.3.2-1.0502.9275.1.25.gem/bin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/sbin:/opt/cray/lustre-cray_gem_s/2.5_3.0.101_0.31.1_1.0502.8394.15.1-1.0502.19859.16.1/bin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/sbin:/opt/cray/alps/5.2.1-2.0502.9710.28.1.gem/bin:/opt/cray/sdb/1.0-1.0502.55976.5.27.gem/bin:/opt/cray/nodestat/2.2-1.0502.53712.3.109.gem/bin:/opt/modules/3.2.10.3/bin:/u/sciteam/yfeng1/bin:/usr/local/bin:/usr/bin:/bin:/usr/bin/X11:/usr/X11R6/bin:/usr/games:/usr/lib/mit/bin:/usr/lib/mit/sbin:.:/usr/lib/qt3/bin:/opt/cray/bin 44 | ++ [[ -n 3.2.51(1)-release ]] 45 | ++ hash -r 46 | + bcast -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 47 | + LD_LIBRARY_PATH= 48 | + aprun -n 262144 /u/sciteam/yfeng1/source/python-mpi-bcast/bcast -p /dev/shm/local -t /u/sciteam/yfeng1/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz /u/sciteam/yfeng1/anaconda2/anaconda-2.7.tar.gz 49 | Time : 5.58864 in bcast 50 | Time : 8.57061 in tar 51 | Time : 14.1837 in total 52 | Application 31747241 resources: utime ~6530393s, stime ~1995310s, Rss ~720740, inblocks ~4350097178, outblocks ~19399774806 53 | 54 | real 0m56.772s 55 | user 0m0.552s 56 | sys 0m0.140s 57 | + cd /dev/shm 58 | + aprun -n 262144 python-mpi -c 'import scipy;from mpi4py import MPI;' 59 | Application 31747399 resources: utime ~2787557s, stime ~1665768s, Rss ~526992, inblocks ~1645849609, outblocks ~956702614 60 | 61 | real 1m49.533s 62 | user 0m0.164s 63 | sys 0m0.048s 64 | + finish 65 | + aprun -n 262144 rm -rf /dev/shm/local 66 | Application 31747464 resources: utime ~0s, stime ~180347s, Rss ~21480, inblocks ~1277952, outblocks ~892928 67 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/bench.template: -------------------------------------------------------------------------------- 1 | #PBS -l nodes=@N@:ppn=32 2 | #PBS -j eo 3 | #PBS -o bench-@N@.out 4 | #PBS -l walltime=00:10:00 5 | 6 | set -x 7 | export OMP_NUM_THREADS=1 8 | export ATP_ENABLED=0 9 | 10 | export PATH=~/anaconda2/bin:$PATH 11 | export LD_LIBRARY_PATH=~/anaconda2/lib:$LD_LIBRARY_PATH 12 | source ~/source/python-mpi-bcast/activate.sh /dev/shm/local "aprun -n $((@N@ * 32))" 13 | 14 | # send the anaconda packages 15 | time bcast -t \ 16 | ~/source/python-mpi-bcast/cray-xt/cray-xt.tar.gz \ 17 | ~/anaconda2/anaconda-2.7.tar.gz 18 | 19 | cd /dev/shm 20 | time aprun -n $((@N@ * 32)) python-mpi -c 'import scipy;from mpi4py import MPI;' 21 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/collect.sh: -------------------------------------------------------------------------------- 1 | cat >process.awk < bench-$1.job 3 | qsub -q low bench-$1.job 4 | -------------------------------------------------------------------------------- /benchmarks/cray-xt/tar-cray-xt.job: -------------------------------------------------------------------------------- 1 | #PBS -j eo 2 | #PBS -N xt-tar-sys 3 | #PBS -l walltime=00:10:00 4 | #PBS -q debug 5 | #PBS -l nodes=1:ppn=32 6 | 7 | cd $PBS_O_WORKDIR 8 | 9 | module load bwpy 10 | module load bwpy-mpi 11 | 12 | aprun -n 1 strace python-mpi -c 'from mpi4py import MPI' &> xt.trace 13 | 14 | grep "= 3$" xt.trace | grep so | sed -s 's;open(";;' | sed -s 's;".*;;' > filelist 15 | 16 | rm -rf lib 17 | mkdir -p lib 18 | aprun -n 1 cp `cat filelist` lib/ 19 | tar -czvf cray-xt.tar.gz lib/ 20 | -------------------------------------------------------------------------------- /benchmarks/startup-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainwoodman/python-mpi-bcast/962c724b7b22bd914c5552991fdd3d8c8caddf08/benchmarks/startup-time.png -------------------------------------------------------------------------------- /benchmarks/startup-time.txt: -------------------------------------------------------------------------------- 1 | # mppwidth time (sec) 2 | 48 15 3 | 3072 22 4 | 6144 32 5 | 12288 54 6 | -------------------------------------------------------------------------------- /nersc-hpcp/activate.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | if [[ -n $BASH_VERSION ]]; then 4 | _SCRIPT_LOCATION=${BASH_SOURCE[0]} 5 | elif [[ -n $ZSH_VERSION ]]; then 6 | _SCRIPT_LOCATION=${funcstack[1]} 7 | else 8 | echo "Only bash and zsh are supported" 9 | return 1 10 | fi 11 | 12 | if [ x"$SLURM_JOB_NUM_NODES" == x ]; then 13 | echo "The script is avalaible from a job script only." 14 | echo "Use with sbatch (for batch scripts) or salloc (for interactive)." 15 | return 1 16 | fi 17 | 18 | DIRNAME=`dirname ${_SCRIPT_LOCATION}` 19 | 20 | source /usr/common/contrib/bccp/python-mpi-bcast/activate.sh /dev/shm/local "srun -n $SLURM_JOB_NUM_NODES" 21 | FILES=`find $DIRNAME -name '*.tar.gz'` 22 | 23 | #HPCPMODULES=`find_hpcp_modules` 24 | HPCPMODULES=$FILES 25 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/composer_xe_2015.1.133/mkl/lib/intel64 26 | 27 | echo "found hpcport packages: $HPCPMODULES" 28 | bcast -v $HPCPMODULES 29 | -------------------------------------------------------------------------------- /nersc-hpcp/bundle-hpcp.sh: -------------------------------------------------------------------------------- 1 | packages=`find /usr/common/contrib/hpcosmo/hpcports_gnu-9.0 -maxdepth 1 -type d` 2 | 3 | source /usr/common/contrib/bccp/python-mpi-bcast/activate.sh 4 | 5 | for package in $packages; do 6 | echo $package to `basename $package`.tar.gz 7 | bundle-anaconda `basename $package`.tar.gz $package 8 | done 9 | -------------------------------------------------------------------------------- /nersc-hpcp/easy-install.pth: -------------------------------------------------------------------------------- 1 | import sys; sys.__plen = len(sys.path) 2 | ./pydl-0.5.2-py2.7.egg 3 | ./astropy-1.2.1-py2.7-linux-x86_64.egg 4 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 5 | import sys; sys.__plen = len(sys.path) 6 | ./matplotlib-1.4.3-py2.7-linux-x86_64.egg 7 | ./mock-2.0.0-py2.7.egg 8 | ./pytz-2016.6.1-py2.7.egg 9 | ./pbr-1.10.0-py2.7.egg 10 | ./funcsigs-1.0.2-py2.7.egg 11 | ./python_dateutil-2.4.2-py2.7.egg 12 | ./six-1.10.0-py2.7.egg 13 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 14 | import sys; sys.__plen = len(sys.path) 15 | ./hpsspy-0.2.1-py2.7.egg 16 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 17 | import sys; sys.__plen = len(sys.path) 18 | ./quaternionarray-0.6.2.dev0-py2.7.egg 19 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 20 | import sys; sys.__plen = len(sys.path) 21 | ./MySQL_python-1.2.3-py2.7-linux-x86_64.egg 22 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 23 | import sys; sys.__plen = len(sys.path) 24 | ./PyInstaller-3.2-py2.7.egg 25 | ./setuptools-20.0-py2.7.egg 26 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 27 | import sys; sys.__plen = len(sys.path) 28 | ./photutils-0.1-py2.7-linux-x86_64.egg 29 | ./astropy-1.2.1-py2.7-linux-x86_64.egg 30 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 31 | import sys; sys.__plen = len(sys.path) 32 | ./rpy2-2.7.6-py2.7-linux-x86_64.egg 33 | ./singledispatch-3.4.0.3-py2.7.egg 34 | ./six-1.10.0-py2.7.egg 35 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 36 | import sys; sys.__plen = len(sys.path) 37 | ./pytz-2015.2-py2.7.egg 38 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 39 | import sys; sys.__plen = len(sys.path) 40 | ./requests-2.8.1-py2.7.egg 41 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 42 | import sys; sys.__plen = len(sys.path) 43 | ./astropy_helpers-1.1.1-py2.7.egg 44 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 45 | import sys; sys.__plen = len(sys.path) 46 | ./setuptools-20.0-py2.7.egg 47 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 48 | import sys; sys.__plen = len(sys.path) 49 | ./astropy-1.2.1-py2.7-linux-x86_64.egg 50 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 51 | import sys; sys.__plen = len(sys.path) 52 | ./python_dateutil-2.4.2-py2.7.egg 53 | ./six-1.10.0-py2.7.egg 54 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 55 | import sys; sys.__plen = len(sys.path) 56 | ./six-1.10.0-py2.7.egg 57 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 58 | import sys; sys.__plen = len(sys.path) 59 | ./healpy-1.9.0-py2.7-linux-x86_64.egg 60 | ./astropy-1.2.1-py2.7-linux-x86_64.egg 61 | ./six-1.10.0-py2.7.egg 62 | ./matplotlib-1.4.3-py2.7-linux-x86_64.egg 63 | ./mock-2.0.0-py2.7.egg 64 | ./pytz-2016.6.1-py2.7.egg 65 | ./python_dateutil-2.4.2-py2.7.egg 66 | ./pbr-1.10.0-py2.7.egg 67 | ./funcsigs-1.0.2-py2.7.egg 68 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 69 | import sys; sys.__plen = len(sys.path) 70 | ./paramiko-1.15.2-py2.7.egg 71 | ./ecdsa-0.13-py2.7.egg 72 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 73 | import sys; sys.__plen = len(sys.path) 74 | ./lmfit-0.8.3-py2.7.egg 75 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 76 | import sys; sys.__plen = len(sys.path) 77 | ./sncosmo-1.0.0-py2.7-linux-x86_64.egg 78 | ./astropy-1.2.1-py2.7-linux-x86_64.egg 79 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 80 | import sys; sys.__plen = len(sys.path) 81 | ./pymongo-3.0.2-py2.7-linux-x86_64.egg 82 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 83 | import sys; sys.__plen = len(sys.path) 84 | ./pkgconfig-1.1.0-py2.7.egg 85 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 86 | import sys; sys.__plen = len(sys.path) 87 | ./h5py-2.6.0-py2.7-linux-x86_64.egg 88 | ./six-1.10.0-py2.7.egg 89 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 90 | import sys; sys.__plen = len(sys.path) 91 | ./speclite-0.4-py2.7.egg 92 | ./astropy-1.2.1-py2.7-linux-x86_64.egg 93 | import sys; new = sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p = getattr(sys, '__egginsert', 0); sys.path[p:p] = new; sys.__egginsert = p + len(new) 94 | -------------------------------------------------------------------------------- /nersc/activate.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | if [[ -n $BASH_VERSION ]]; then 4 | _SCRIPT_LOCATION=${BASH_SOURCE[0]} 5 | elif [[ -n $ZSH_VERSION ]]; then 6 | _SCRIPT_LOCATION=${funcstack[1]} 7 | else 8 | echo "Only bash and zsh are supported" 9 | return 1 10 | fi 11 | 12 | if [ x"$SLURM_JOB_NUM_NODES" == x ]; then 13 | echo "The script is avalaible from a job script only." 14 | echo "Use with sbatch (for batch scripts) or salloc (for interactive)." 15 | return 1 16 | fi 17 | 18 | DIRNAME=`dirname ${_SCRIPT_LOCATION}` 19 | 20 | source $DIRNAME/../activate.sh /dev/shm/local "srun -n $SLURM_JOB_NUM_NODES" 21 | 22 | function __init__ { 23 | local SHOWTIME= 24 | if [ "x$1" == "x-t" ]; then 25 | SHOWTIME=-t 26 | fi 27 | 28 | local ANACONDA= 29 | local NERSCROOT=$DIRNAME/${NERSC_HOST} 30 | 31 | case "$LOADEDMODULES" in 32 | *2.7-anaconda* ) 33 | ANACONDA=$NERSCROOT/2.7-anaconda 34 | ;; 35 | *3.4-anaconda* ) 36 | ANACONDA=$NERSCROOT/3.4-anaconda 37 | ;; 38 | *3.5-anaconda* ) 39 | ANACONDA=$NERSCROOT/3.5-anaconda 40 | ;; 41 | * ) 42 | echo "Run module load python/2.7-anaconda first" 43 | return 1 44 | ;; 45 | esac; 46 | 47 | bcast $SHOWTIME $NERSCROOT/system-libraries.tar.gz \ 48 | $ANACONDA/python.tar.gz \ 49 | $ANACONDA/mpi4py.tar.gz \ 50 | $ANACONDA/fitsio.tar.gz 51 | 52 | bcast-userbase 53 | } 54 | 55 | __init__ $* 56 | -------------------------------------------------------------------------------- /nersc/build.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #SBATCH -N 1 4 | #SBATCH -p debug 5 | #SBATCH -t 00:20:00 6 | #SBATCH -o build.log 7 | 8 | # kill children as we die 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT 10 | 11 | source ../activate.sh 12 | 13 | PREFIX=${NERSC_HOST} 14 | mkdir -p $PREFIX 15 | 16 | function rotate { 17 | if [ -f $1 ]; then 18 | local postfix=`stat -c %y $1 | cut -d ' ' --output-delimiter='_' -f1,2` 19 | mv $1 $1-$postfix 20 | fi 21 | mkdir -p `dirname $1` 22 | mv $2 $1 23 | echo "Created file at $1" 24 | } 25 | 26 | function build { 27 | if module swap python python/$1 2>&1 | grep ERROR ; then 28 | return 1 29 | fi 30 | # the first call won't set the env because it is in a child process. 31 | module swap python python/$1 32 | 33 | local python=`which python` 34 | local anaconda=`dirname $python`/.. 35 | anaconda=`readlink -f $anaconda` 36 | mkdir -p $PREFIX/$1 37 | echo "Building bundle for Python at $anaconda" 38 | 39 | pushd $PREFIX/$1 40 | 41 | ( bundle-anaconda _python.tar.gz $anaconda 42 | rotate python.tar.gz _python.tar.gz 43 | ) & 44 | 45 | ( MPICC=cc bundle-pip _mpi4py.tar.gz mpi4py 46 | rotate mpi4py.tar.gz _mpi4py.tar.gz 47 | ) & 48 | 49 | ( bundle-pip _fitsio.tar.gz https://github.com/esheldon/fitsio/archive/v0.9.8rc2.tar.gz 50 | rotate fitsio.tar.gz _fitsio.tar.gz 51 | ) & 52 | 53 | wait 54 | popd 55 | } 56 | 57 | function system { 58 | pushd $PREFIX 59 | 60 | rm -rf lib 61 | mkdir -p lib 62 | 63 | local filelist 64 | local EXEC 65 | if which python-mpi; then 66 | EXEC=python-mpi 67 | else 68 | EXEC='python -m mpi4py' 69 | fi 70 | filelist=`srun -n 1 strace $EXEC -c 'from mpi4py import MPI' 2>&1 \ 71 | | grep "= 3$" | grep "\\.so" | sed -s 's;open(";;' | sed -s 's;".*;;' \ 72 | | sort | uniq | grep "\\.so" |grep -v "ld.so.conf"` 73 | 74 | srun -n 1 cp -aL $filelist lib/ 75 | srun -n 1 cp -aL /lib64/* lib/ 76 | echo $filelist > filelist 77 | tar -czf _system-libraries.tar.gz lib/ 78 | 79 | rotate system-libraries.tar.gz _system-libraries.tar.gz 80 | popd 81 | } 82 | 83 | echo "-------------------------------------------------" 84 | echo "Working on host $NERSC_HOST, output to $PREFIX" 85 | 86 | echo "First build the system-libraries bundle" 87 | 88 | ( system ) 89 | 90 | echo "Second build the python environments; output is messy" 91 | 92 | #( build 2.7-anaconda ) & 93 | #( build 3.4-anaconda ) & 94 | #( build 3.5-anaconda ) & 95 | 96 | #wait 97 | 98 | echo "-------------------------------------------------" 99 | 100 | -------------------------------------------------------------------------------- /nersc/deploy.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | PREFIX=${NERSC_HOST} 4 | 5 | echo "Installing scripts ..." 6 | install -d /usr/common/contrib/bccp/python-mpi-bcast 7 | 8 | ( 9 | cd .. 10 | install bcast activate.sh tar-pip.sh tar-dir.sh tar-anaconda.sh /usr/common/contrib/bccp/python-mpi-bcast/ 11 | ) 12 | 13 | install -d /usr/common/contrib/bccp/python-mpi-bcast/nersc 14 | install activate.sh /usr/common/contrib/bccp/python-mpi-bcast/nersc/ 15 | echo "Installing bundles ..." 16 | rsync --exclude='*.gz-*' -ar $PREFIX /usr/common/contrib/bccp/python-mpi-bcast/nersc/ 17 | 18 | function tree { 19 | SEDMAGIC='s;[^/]*/;|____;g;s;____|; |;g' 20 | 21 | if [ "$#" -gt 0 ] ; then 22 | dirlist="$@" 23 | else 24 | dirlist="." 25 | fi 26 | 27 | for x in $dirlist; do 28 | find "$x" -printf "%p@%t\n" | sed -e "$SEDMAGIC"|awk -F @ '{printf("%-40s %s\n", $1, $2)}' 29 | done 30 | 31 | } 32 | 33 | echo "Done. Tree of files... " 34 | ( 35 | cd /usr/common/contrib/bccp/python-mpi-bcast/; 36 | chmod -R o+rX * 37 | tree . 38 | ) 39 | -------------------------------------------------------------------------------- /nersc/run-test.slurm: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | #SBATCH -p debug 4 | #SBATCH -t 10:00 5 | #SBATCH -N 2 6 | #SBATCH -J test-python-mpi-bcast 7 | #SBATCH -o test-python-mpi-bcast.log 8 | 9 | set -x 10 | module load python 11 | # 12 | # important to wrap in a subshell, since after activate.sh module command is broken. 13 | # 14 | ( 15 | module swap python python/2.7-anaconda 16 | module load mpi4py 17 | source ./activate.sh -t 18 | fn=`mktemp XXXXXXXX.py` 19 | cat > $fn < $fn <&2 29 | exit 1 30 | fi 31 | tar -cf - $* | gzip -9 - > $OUTPUT 32 | ) 33 | 34 | echo $OUTPUT 35 | -------------------------------------------------------------------------------- /tar-pip.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | if [ "x$1" == "x-h" ] || [ "x$1" == "x" ] ; then 3 | echo "bundle-pip bundle-name.tar.gz [-r requirements.txt] package1 package2 ..." 4 | exit 1 5 | fi 6 | 7 | OUTPUT=`readlink -f $1` 8 | shift 9 | packages="$*" 10 | 11 | DIR=`mktemp -d` 12 | 13 | trap "rm -rf $DIR" EXIT 14 | 15 | pip install --ignore-installed --no-deps --prefix=$DIR $packages || exit 1 16 | 17 | ( 18 | cd $DIR 19 | list= 20 | for dir in bin lib include share; do 21 | if [ -d $dir ]; then 22 | list="$list $dir" 23 | fi 24 | done 25 | 26 | tar -czf $OUTPUT \ 27 | --exclude='*.jpg' \ 28 | --exclude='*.jpeg' \ 29 | --exclude='*.png' \ 30 | --exclude='*.pyc' \ 31 | --exclude='*.pyo' \ 32 | $list 33 | ) || exit 1 34 | echo $OUTPUT created for $packages 35 | --------------------------------------------------------------------------------