├── wrap ├── .gitignore ├── LICENSE ├── WrapConfig.cmake └── README.md ├── src ├── c │ ├── CMakeLists.txt │ ├── test │ │ ├── print-args.f │ │ ├── CMakeLists.txt │ │ ├── exit-test.c │ │ ├── print-args.c │ │ ├── crash-test.c │ │ ├── cram-io-test.sh │ │ ├── cram-cat.c │ │ ├── cram-read-file-test.c │ │ └── cram-test.c │ └── libcram │ │ ├── CMakeLists.txt │ │ ├── cram_fargs.c │ │ ├── cram_file.h │ │ ├── cram.w │ │ └── cram_file.c ├── CMakeLists.txt ├── modules │ ├── CMakeLists.txt │ └── cram.dk.in └── python │ ├── CMakeLists.txt │ ├── llnl │ ├── __init__.py │ └── util │ │ ├── __init__.py │ │ ├── filesystem.py │ │ ├── tty │ │ ├── __init__.py │ │ ├── colify.py │ │ └── color.py │ │ └── lang.py │ └── cram │ ├── __init__.py │ ├── cmd │ ├── help.py │ ├── test.py │ ├── python.py │ ├── __init__.py │ ├── pack.py │ ├── test-gen.py │ ├── info.py │ └── test-verify.py │ ├── test │ ├── scripts │ │ ├── million-jobs.py │ │ └── test-pack-script.py │ ├── serialization.py │ ├── __init__.py │ └── cramfile.py │ ├── serialization.py │ └── cramfile.py ├── .gitignore ├── images └── cram-logo.png ├── bin ├── CMakeLists.txt ├── make-tarball ├── cram-python └── cram ├── scripts └── do-cmake-bgq ├── cmake ├── StaticAndShared.cmake ├── Platform │ ├── BlueGeneQ-static-XL-C.cmake │ ├── BlueGeneQ-dynamic-GNU-C.cmake │ ├── BlueGeneQ-dynamic-XL-C.cmake │ ├── BlueGeneQ-static-GNU-C.cmake │ ├── BlueGeneQ-static-XL-CXX.cmake │ ├── BlueGeneQ-dynamic-GNU-CXX.cmake │ ├── BlueGeneQ-dynamic-XL-CXX.cmake │ ├── BlueGeneQ-static-GNU-CXX.cmake │ ├── BlueGeneQ-dynamic-XL-Fortran.cmake │ ├── BlueGeneQ-static-GNU-Fortran.cmake │ ├── BlueGeneQ-static-XL-Fortran.cmake │ ├── BlueGeneQ-dynamic-GNU-Fortran.cmake │ ├── BlueGeneQ-static.cmake │ ├── BlueGeneQ-dynamic.cmake │ └── BlueGeneQ-base.cmake └── Toolchain │ ├── BlueGeneQ-gnu.cmake │ └── BlueGeneQ-xl.cmake ├── CMakeLists.txt └── README.md /wrap/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .DS_Store 3 | *.pyc 4 | autom4te.cache 5 | -------------------------------------------------------------------------------- /src/c/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(libcram) 2 | add_subdirectory(test) 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | .DS_Store 4 | .idea 5 | *.o 6 | *.a 7 | *.so 8 | *.dylib 9 | -------------------------------------------------------------------------------- /images/cram-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/cram/HEAD/images/cram-logo.png -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(c) 2 | add_subdirectory(python) 3 | add_subdirectory(modules) 4 | -------------------------------------------------------------------------------- /bin/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | install(PROGRAMS cram DESTINATION bin) 2 | install(PROGRAMS cram-python DESTINATION bin) 3 | -------------------------------------------------------------------------------- /src/modules/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | configure_file(cram.dk.in cram.dk @ONLY) 3 | install( 4 | FILES ${PROJECT_BINARY_DIR}/src/modules/cram.dk 5 | DESTINATION share/cram/dotkit) 6 | -------------------------------------------------------------------------------- /scripts/do-cmake-bgq: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cmake \ 4 | -DCMAKE_INSTALL_PREFIX=/collab/usr/global/tools/cram/$SYS_TYPE/cram \ 5 | -DCMAKE_BUILD_TYPE=RelWithDebInfo \ 6 | -DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain/BlueGeneQ-gnu.cmake \ 7 | .. 8 | -------------------------------------------------------------------------------- /cmake/StaticAndShared.cmake: -------------------------------------------------------------------------------- 1 | 2 | function(add_static_and_shared_library target) 3 | get_property(supports_shared_libs GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS) 4 | if (supports_shared_libs) 5 | add_library(${target} SHARED ${ARGN}) 6 | endif() 7 | add_library(${target}_static STATIC ${ARGN}) 8 | set_target_properties(${target}_static PROPERTIES OUTPUT_NAME ${target}) 9 | endfunction() 10 | -------------------------------------------------------------------------------- /src/c/test/print-args.f: -------------------------------------------------------------------------------- 1 | 2 | program farg_test 3 | implicit none 4 | integer argc, i, ierr 5 | character*64 :: arg 6 | integer*4 :: iargc 7 | 8 | call mpi_init(ierr) 9 | 10 | argc = iargc() 11 | do i=1,argc 12 | call getarg(i, arg) 13 | write (*, "(a)") arg 14 | end do 15 | 16 | call mpi_finalize(ierr) 17 | end program farg_test 18 | -------------------------------------------------------------------------------- /bin/make-tarball: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Makes a timestamped, commit-stamped archive of the current HEAD. 4 | # Author: Todd Gamblin 5 | # 6 | name=cram 7 | head=`git log -1 --format="%h" HEAD` 8 | dstamp=`git log -1 --format="%ct" HEAD | perl -MPOSIX -le 'print strftime "%F", localtime '` 9 | 10 | filename="${name}-${dstamp}-${head}" 11 | git archive --format=tar.gz --prefix="${filename}/" HEAD -o ${filename}.tar.gz 12 | -------------------------------------------------------------------------------- /src/modules/cram.dk.in: -------------------------------------------------------------------------------- 1 | #c tools/infrastructure 2 | #d Cram 1.0a1 3 | 4 | #h Cram launches many small MPI jobs as one large one. 5 | #h 6 | #h https://lc.llnl.gov/stash/projects/SCALE/repos/cram 7 | 8 | # Unuse any other python dotkits 9 | unuse -q `dk_rep python*` 10 | 11 | # Use the Python 2.7.3 package 12 | use -q python-2.7.3 13 | 14 | # Cram setup 15 | dk_setenv CRAM_HOME @CMAKE_INSTALL_PREFIX@ 16 | 17 | dk_alter PATH $CRAM_HOME/bin 18 | dk_alter LD_LIBRARY_PATH $CRAM_HOME/lib 19 | -------------------------------------------------------------------------------- /src/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PY_VERSION "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}") 2 | set(PY_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib/python${PY_VERSION}/site-packages") 3 | set(PY_COMPILE_CMD "import compileall; compileall.compile_dir('${PY_INSTALL_DIR}')") 4 | 5 | # Install all the python files into the prefix's site-packages directory 6 | install( 7 | DIRECTORY cram llnl 8 | DESTINATION "${PY_INSTALL_DIR}" 9 | FILES_MATCHING PATTERN "*.py") 10 | 11 | # Make Python precompile all the bytecode for the cram packages. 12 | install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -c \"${PY_COMPILE_CMD}\")") 13 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static-XL-C.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_static(XL C) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic-GNU-C.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_dynamic(GNU C) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic-XL-C.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_dynamic(XL C) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static-GNU-C.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_static(GNU C) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static-XL-CXX.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_static(XL CXX) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic-GNU-CXX.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_dynamic(GNU CXX) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic-XL-CXX.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_dynamic(XL CXX) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static-GNU-CXX.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_static(GNU CXX) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic-XL-Fortran.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_dynamic(XL Fortran) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static-GNU-Fortran.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_static(GNU Fortran) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static-XL-Fortran.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_static(XL Fortran) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic-GNU-Fortran.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | __BlueGeneQ_setup_dynamic(GNU Fortran) 17 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-static.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | include(Platform/BlueGeneQ-base) 17 | set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") 18 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-dynamic.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | include(Platform/BlueGeneQ-base) 17 | set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a") 18 | -------------------------------------------------------------------------------- /cmake/Toolchain/BlueGeneQ-gnu.cmake: -------------------------------------------------------------------------------- 1 | # Need this first to ensure that we include custom platform files. 2 | set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") 3 | 4 | # Avoid warnings about duplicated platform files if CMake already has them. 5 | cmake_policy(SET CMP0011 NEW) 6 | cmake_policy(SET CMP0017 NEW) 7 | 8 | # the name of the target operating system 9 | set(CMAKE_SYSTEM_NAME BlueGeneQ-dynamic) 10 | 11 | set(TOOLCHAIN_LOCATION /bgsys/drivers/ppcfloor/gnu-linux/bin) 12 | set(TOOLCHAIN_PREFIX powerpc64-bgq-linux-) 13 | 14 | set(CMAKE_C_COMPILER ${TOOLCHAIN_LOCATION}/${TOOLCHAIN_PREFIX}gcc) 15 | set(CMAKE_CXX_COMPILER ${TOOLCHAIN_LOCATION}/${TOOLCHAIN_PREFIX}g++) 16 | set(CMAKE_Fortran_COMPILER ${TOOLCHAIN_LOCATION}/${TOOLCHAIN_PREFIX}gfortran) 17 | 18 | # Make sure MPI_COMPILER wrapper matches the gnu compilers. 19 | # Prefer local machine wrappers to driver wrappers here too. 20 | find_program(MPI_COMPILER NAMES mpicxx mpic++ mpiCC mpicc 21 | PATHS 22 | /usr/local/bin 23 | /usr/bin 24 | /bgsys/drivers/ppcfloor/comm/gcc/bin) 25 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(Cram C Fortran) 2 | cmake_minimum_required(VERSION 2.8) 3 | 4 | # Use C99 5 | if (CMAKE_C_COMPILER_ID STREQUAL "XL") 6 | # XL compilers are different 7 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qlanglvl=extc99") 8 | else() 9 | # clang, gcc, and icc all use this arg. 10 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") 11 | endif() 12 | 13 | # Add local module path with some add-ons 14 | set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") 15 | include(StaticAndShared) 16 | 17 | # Required dependencies 18 | find_package(MPI REQUIRED) 19 | find_package(PythonInterp REQUIRED) 20 | if (PYTHON_VERSION_STRING VERSION_LESS "2.7") 21 | message(FATAL_ERROR "Cram requires Python 2.7 or later.") 22 | endif() 23 | 24 | # Use wrap.py wrapper generator 25 | set(WRAP ${PROJECT_SOURCE_DIR}/wrap/wrap.py) 26 | include(${PROJECT_SOURCE_DIR}/wrap/WrapConfig.cmake) 27 | 28 | # Enable CTest and add Cram's python unit tests to the suite 29 | enable_testing() 30 | add_test(cram-python-tests ${PROJECT_SOURCE_DIR}/bin/cram test) 31 | 32 | # Everything that needs to be built is in src. 33 | add_subdirectory(bin) 34 | add_subdirectory(src) 35 | -------------------------------------------------------------------------------- /cmake/Toolchain/BlueGeneQ-xl.cmake: -------------------------------------------------------------------------------- 1 | # Need this first to ensure that we include custom platform files. 2 | set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") 3 | 4 | # Avoid warnings about duplicated platform files if CMake already has them. 5 | cmake_policy(SET CMP0011 NEW) 6 | cmake_policy(SET CMP0017 NEW) 7 | 8 | # the name of the target operating system 9 | set(CMAKE_SYSTEM_NAME BlueGeneQ-dynamic) 10 | 11 | # Set search paths to prefer local, admin-installed wrappers for the BG backend compilers 12 | set(BGQ_XL_COMPILER_SEARCH_PATHS 13 | /usr/local/bin 14 | /usr/bin 15 | /opt/ibmcmp/vac/bg/12.1/bin 16 | /opt/ibmcmp/vacpp/bg/12.1/bin 17 | /opt/ibmcmp/xlf/bg/14.1/bin) 18 | 19 | # XL C Compilers 20 | find_program(CMAKE_C_COMPILER bgxlc_r ${BGQ_XL_COMPILER_SEARCH_PATHS}) 21 | find_program(CMAKE_CXX_COMPILER bgxlc_r ${BGQ_XL_COMPILER_SEARCH_PATHS}) 22 | find_program(CMAKE_Fortran_COMPILER bgxlf90_r ${BGQ_XL_COMPILER_SEARCH_PATHS}) 23 | 24 | # Make sure MPI_COMPILER wrapper matches the gnu compilers. 25 | # Prefer local machine wrappers to driver wrappers here too. 26 | find_program(MPI_COMPILER NAMES mpixlc mpxlc 27 | PATHS 28 | /usr/local/bin 29 | /usr/bin 30 | /bgsys/drivers/ppcfloor/comm/gcc/bin) 31 | -------------------------------------------------------------------------------- /src/python/llnl/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | -------------------------------------------------------------------------------- /src/python/llnl/util/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | -------------------------------------------------------------------------------- /src/c/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories( 2 | ${PROJECT_SOURCE_DIR}/src/c/libcram 3 | ${MPI_C_INCLUDE_PATH}) 4 | 5 | function(add_cram_test exe_name source_file) 6 | add_executable(${exe_name} ${source_file}) 7 | target_link_libraries(${exe_name} cram ${MPI_C_LIBRARIES}) 8 | endfunction() 9 | 10 | function(add_fcram_test exe_name source_file) 11 | add_executable(${exe_name} ${source_file}) 12 | target_link_libraries(${exe_name} fcram ${MPI_Fortran_LIBRARIES}) 13 | 14 | if (CMAKE_Fortran_COMPILER_ID STREQUAL GNU) 15 | # Allow multiple defs to override fortran intrinsics 16 | set_target_properties(${exe_name} PROPERTIES LINK_FLAGS -Wl,-zmuldefs) 17 | endif() 18 | endfunction() 19 | 20 | # 21 | # Cram test executables 22 | # 23 | add_cram_test(cram-cat cram-cat.c) 24 | add_cram_test(cram-read-file-test cram-read-file-test.c) 25 | add_cram_test(cram-test cram-test.c) 26 | install(TARGETS cram-test cram-read-file-test DESTINATION libexec/cram) 27 | 28 | # This test runs a bash script to compare cram-cat output with cram info output. 29 | add_test(cram-io-test 30 | ${PROJECT_SOURCE_DIR}/src/c/test/cram-io-test.sh ${PROJECT_SOURCE_DIR}/bin/cram cram-cat) 31 | 32 | 33 | add_fcram_test(print-args-fortran print-args.f) 34 | add_cram_test(print-args-c print-args.c) 35 | 36 | add_cram_test(crash-test crash-test.c) 37 | add_cram_test(exit-test crash-test.c) 38 | -------------------------------------------------------------------------------- /src/c/libcram/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # This defines the C cram library 4 | # 5 | add_wrapped_file(cram.c cram.w) 6 | set(CRAM_SOURCES 7 | cram.c 8 | cram_file.c) 9 | add_static_and_shared_library(cram ${CRAM_SOURCES}) 10 | 11 | # 12 | # This build the Fortran cram library, with fortran arg handling and 13 | # fortran MPI wrappers. 14 | # 15 | add_wrapped_file(cram_fortran.c cram.w -f -i pmpi_init) 16 | set(CRAM_FORTRAN_SOURCES 17 | cram_fortran.c 18 | cram_fargs.c 19 | cram_file.c) 20 | add_library(fcram STATIC ${CRAM_FORTRAN_SOURCES}) 21 | 22 | # 23 | # This command post-processes the fortran library so that the various 24 | # iargc/getarg functions will override compiler intrinsics 25 | # 26 | # Add support for additional compilers here by adding their getarg and 27 | # iargc bindings. 28 | # 29 | add_custom_command(TARGET fcram POST_BUILD 30 | COMMAND objcopy 31 | --redefine-sym getarg_gnu=_gfortran_getarg_i4@@GFORTRAN_1.0 32 | --redefine-sym iargc_gnu=_gfortran_iargc@@GFORTRAN_1.0 33 | --redefine-sym getarg_xl=getarg@@XLF_1.0 34 | --redefine-sym iargc_xl=iargc@@XLF_1.0 35 | $ tmp.a 36 | COMMAND mv tmp.a $) 37 | 38 | include_directories(${MPI_C_INCLUDE_PATH} 39 | ${PROJECT_SOURCE_DIR}/src/c/libcram) 40 | 41 | install(TARGETS cram cram_static fcram DESTINATION lib) 42 | 43 | -------------------------------------------------------------------------------- /src/python/cram/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | cram_version = '0.9' 26 | 27 | from cram.cramfile import CramFile, Job 28 | -------------------------------------------------------------------------------- /src/c/test/exit-test.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | // 26 | // This test program causes process 0 to fail with exit(1) 27 | // 28 | #include 29 | #include 30 | 31 | int main(int argc, char **argv) { 32 | MPI_Init(&argc, &argv); 33 | 34 | int rank; 35 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 36 | 37 | if (rank == 0) { 38 | exit(1); 39 | } 40 | 41 | MPI_Finalize(); 42 | } 43 | -------------------------------------------------------------------------------- /src/python/cram/cmd/help.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import sys 26 | 27 | description = "Get help on cram and its commands" 28 | 29 | def setup_parser(subparser): 30 | subparser.add_argument('help_command', nargs='?', default=None, 31 | help='command to get help on') 32 | 33 | def help(parser, args): 34 | if args.help_command: 35 | parser.parse_args([args.help_command, '-h']) 36 | else: 37 | parser.print_help() 38 | -------------------------------------------------------------------------------- /bin/cram-python: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # cram-python 4 | # 5 | # If you want to write your own executable Python script that uses 6 | # Cram, modules, on Mac OS or maybe some others, you can use this 7 | # script to do it like this: 8 | # 9 | # #!/usr/bin/env cram-python 10 | # 11 | # We provide a separate script for this because most Linuxes don't 12 | # support more than one argument after the shebang command. Mac OS 13 | # will actually let you do this: 14 | # 15 | # #!/usr/bin/env cram python 16 | # 17 | # But it is non-standard so we provide this script as a workaround. 18 | # 19 | # NOTE: This script needs to be installed alongside the main cram 20 | # script to work properly. 21 | # 22 | 23 | # 24 | # Get readlink -f behavior when readlink doesn't support it 25 | # 26 | function readlink_f { 27 | _target_file=$1 28 | 29 | cd `dirname $_target_file` 30 | _target_file=`basename $_target_file` 31 | 32 | # Iterate down a (possible) chain of symlinks 33 | while [ -L "$_target_file" ]; do 34 | _target_file=`readlink $_target_file` 35 | cd `dirname $_target_file` 36 | _target_file=`basename $_target_file` 37 | done 38 | 39 | # Compute the canonicalized name by finding the physical path 40 | # for the directory we're in and appending the target file. 41 | _phys_dir=`pwd -P` 42 | _result=$_phys_dir/$_target_file 43 | echo $_result 44 | } 45 | 46 | # Get directory of this script. 47 | mydir=$(dirname $(readlink_f "$0")) 48 | 49 | # Make sure the cram executable is in the same directory as cram-python 50 | cram="$mydir/cram" 51 | if [ ! -x "$cram" ]; then 52 | echo "ERROR: cram-python must be installed alongside the cram script!" 53 | fi 54 | 55 | # Execute Cram with the python option 56 | $cram python "$@" 57 | -------------------------------------------------------------------------------- /src/python/cram/test/scripts/million-jobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram-python 2 | ############################################################################## 3 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 4 | # Produced at the Lawrence Livermore National Laboratory. 5 | # 6 | # This file is part of Cram. 7 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 8 | # LLNL-CODE-661100 9 | # 10 | # For details, see https://github.com/scalability-llnl/cram. 11 | # Please also see the LICENSE file for our notice and the LGPL. 12 | # 13 | # This program is free software; you can redistribute it and/or modify 14 | # it under the terms of the GNU General Public License (as published by 15 | # the Free Software Foundation) version 2.1 dated February 1999. 16 | # 17 | # This program is distributed in the hope that it will be useful, but 18 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 20 | # conditions of the GNU General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public License 23 | # along with this program; if not, write to the Free Software Foundation, 24 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 25 | ############################################################################## 26 | 27 | import os 28 | import getpass 29 | from cram import * 30 | 31 | env = os.environ 32 | user = getpass.getuser() 33 | 34 | cf = CramFile('cram.job', 'w') 35 | for i in xrange(1048576): 36 | env["SCRATCH_DIR"] = "/p/lscratcha/%s/scratch-%08d" % (user, i) 37 | args = ["input.%08d" % i] 38 | cf.pack(1, '/home/%s/ensemble/run-%08d' % (user, i), args, env) 39 | cf.close() 40 | -------------------------------------------------------------------------------- /src/c/test/print-args.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | // 26 | // This test program prints out arguments. 27 | // 28 | #include 29 | #include 30 | #include 31 | 32 | int main(int argc, char **argv) { 33 | MPI_Init(&argc, &argv); 34 | 35 | int rank; 36 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 37 | 38 | if (rank == 0) { 39 | for (int i=0; i < argc; i++) { 40 | printf("%s\n", argv[i]); 41 | } 42 | } 43 | 44 | MPI_Finalize(); 45 | } 46 | -------------------------------------------------------------------------------- /src/c/test/crash-test.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | // 26 | // This test program causes process 0 to fail with a SEGV. 27 | // 28 | #include 29 | #include 30 | #include 31 | 32 | int main(int argc, char **argv) { 33 | MPI_Init(&argc, &argv); 34 | 35 | // int rank; 36 | // MPI_Comm_rank(MPI_COMM_WORLD, &rank); 37 | 38 | // if (rank == 0) { 39 | int *bad_pointer = NULL; 40 | int v = *bad_pointer; 41 | printf("Value was %d.\n", v); 42 | // } 43 | 44 | MPI_Finalize(); 45 | } 46 | -------------------------------------------------------------------------------- /src/c/test/cram-io-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # This test builds a cram file and compares the output of cram info 4 | # with the output of cram-cat. The test cram file is structured to 5 | # cover all I/O decompression cases. 6 | # 7 | # This is essentially a test of the full path from the Python cramfile 8 | # output, and the C input and decompression functions. it is assumed 9 | # that the Python cram file reader code works: that is tested by 10 | # cram's Python test suite. 11 | # 12 | 13 | cram_cat="$1" 14 | cram="$2" 15 | 16 | if [ -z "$cram_cat" -o -z "$cram" ]; then 17 | echo "Usage: cram-io-test.sh " 18 | exit 1 19 | fi 20 | 21 | bgq_run='' 22 | if [ "$SYS_TYPE" = "bgqos_0" ]; then 23 | bgq_run='srun -n 1 -ppdebug' 24 | fi 25 | 26 | # Clean up old stale test data if necessary 27 | rm -f cram-info.txt cram-cat.txt test-cram.job 28 | 29 | export TEST_VAR1='foo' 30 | export TEST_VAR2='bar' 31 | export TEST_VAR3='baz' 32 | $cram pack -f test-cram.job -n 35 foo bar baz 33 | 34 | export TEST_VAR1='bar' 35 | export TEST_VAR2='baz' 36 | export TEST_VAR3='quux' 37 | $cram pack -f test-cram.job -n 24 foo bar 38 | 39 | unset TEST_VAR1 40 | export TEST_VAR1 41 | export TEST_VAR3='bar' 42 | $cram pack -f test-cram.job -n 12 foo bar baz quux 43 | 44 | unset TEST_VAR2 45 | export TEST_VAR2 46 | $cram pack -f test-cram.job -n 35 foo 47 | 48 | $cram info -a test-cram.job > cram-info.txt 49 | $bgq_run cram-cat test-cram.job > cram-cat.txt 50 | 51 | echo ===== CRAM INFO IS HERE ================================== 52 | cat cram-info.txt 53 | echo 54 | echo ===== CRAM CAT IS HERE =================================== 55 | cat cram-cat.txt 56 | 57 | diff=$(diff cram-info.txt cram-cat.txt) 58 | if [ ! -z "$diff" ]; then 59 | echo "FAILED" 60 | echo "cram-info.txt and cram-cat.txt differ. diff output:" 61 | echo "$diff" 62 | exit 1 63 | else 64 | echo "SUCCESS" 65 | rm -f cram-info.txt cram-cat.txt test-cram.job 66 | exit 0 67 | fi 68 | -------------------------------------------------------------------------------- /src/c/test/cram-cat.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "cram_file.h" 30 | 31 | 32 | int main(int argc, char **argv) { 33 | if (argc < 2) { 34 | fprintf(stderr, "Usage: cram-cat \n"); 35 | fprintf(stderr, " Prints out the entire contents of a cram file " 36 | "as cram info would.\n"); 37 | exit(1); 38 | } 39 | const char *filename = argv[1]; 40 | 41 | cram_file_t file; 42 | if (!cram_file_open(filename, &file)) { 43 | printf("failed to map with errno %d: %s\n", errno, strerror(errno)); 44 | exit(1); 45 | } 46 | 47 | printf("Name:%25s\n", filename); 48 | cram_file_cat(&file); 49 | } 50 | -------------------------------------------------------------------------------- /src/python/cram/cmd/test.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | from pprint import pprint 26 | 27 | import cram.test 28 | from llnl.util.lang import list_modules 29 | from llnl.util.tty.colify import colify 30 | 31 | description ="Run unit tests" 32 | 33 | def setup_parser(subparser): 34 | subparser.add_argument( 35 | 'names', nargs='*', help="Names of tests to run.") 36 | subparser.add_argument( 37 | '-l', '--list', action='store_true', dest='list', help="Show available tests") 38 | subparser.add_argument( 39 | '-v', '--verbose', action='store_true', dest='verbose', 40 | help="verbose output") 41 | 42 | 43 | def test(parser, args): 44 | if args.list: 45 | print "Available tests:" 46 | colify(cram.test.list_tests(), indent=2) 47 | 48 | else: 49 | cram.test.run(args.names, args.verbose) 50 | -------------------------------------------------------------------------------- /bin/cram: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | def add_prefix_to_path(pkg_name, *pkg_paths): 4 | """Assuming the containing script is in $prefix/bin. Given no args, 5 | adds $prefix to PYTHONPATH. Given a prefix-relative path, adds 6 | $prefix/relative/path to PYTHONPATH.""" 7 | my_file = os.path.realpath(os.path.expanduser(__file__)) 8 | prefix = os.path.dirname(os.path.dirname(my_file)) 9 | if pkg_paths: 10 | for path in pkg_paths: 11 | pkg_path = os.path.join(prefix, path) 12 | pkg_dir = os.path.join(pkg_path, pkg_name) 13 | if os.path.isdir(pkg_dir): 14 | sys.path.insert(0, pkg_path) 15 | else: 16 | sys.path.insert(0, prefix) 17 | 18 | 19 | def check_python_version(version_tuple): 20 | """Die if the python version isn't newer than version_tuple""" 21 | sys_ver = sys.version_info[:len(version_tuple)] 22 | if not sys_ver >= version_tuple: 23 | print "Cramrequires Python %s, but this is Python %s." % ( 24 | '.'.join(str(v) for v in version_tuple), 25 | '.'.join(str(v) for v in sys_ver)) 26 | sys.exit(1) 27 | 28 | 29 | import sys 30 | import os 31 | check_python_version((2,7)) 32 | add_prefix_to_path( 33 | 'cram', 'src/python', 34 | 'lib/python%d.%d/site-packages' % sys.version_info[:2]) 35 | 36 | import cram 37 | import argparse 38 | 39 | # Command parsing 40 | parser = argparse.ArgumentParser( 41 | description='cram: Crams small MPI jobs into a single large one.') 42 | 43 | # each command module implements a parser() function, to which we pass its 44 | # subparser for setup. 45 | subparsers = parser.add_subparsers(metavar='SUBCOMMAND', dest="subcommand") 46 | 47 | import cram.cmd 48 | for cmd in cram.cmd.commands: 49 | module = cram.cmd.get_module(cmd) 50 | subparser = subparsers.add_parser(cmd, help=module.description) 51 | module.setup_parser(subparser) 52 | args = parser.parse_args() 53 | 54 | # Try to load the particular command asked for and run it 55 | subcommand = cram.cmd.get_command(args.subcommand) 56 | try: 57 | subcommand(parser, args) 58 | except KeyboardInterrupt: 59 | print "Got a keyboard interrupt from the user." 60 | sys.exit(1) 61 | -------------------------------------------------------------------------------- /src/python/cram/cmd/python.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import code 26 | import os 27 | import platform 28 | from contextlib import closing 29 | 30 | import cram 31 | 32 | def setup_parser(subparser): 33 | subparser.add_argument('file', nargs='?', help="file to run") 34 | 35 | description = "Launch an interpreter as cram would and run commands" 36 | 37 | def python(parser, args): 38 | console = code.InteractiveConsole() 39 | 40 | if "PYTHONSTARTUP" in os.environ: 41 | startup_file = os.environ["PYTHONSTARTUP"] 42 | if os.path.isfile(startup_file): 43 | with closing(open(startup_file)) as startup: 44 | console.runsource(startup.read(), startup_file, 'exec') 45 | 46 | if args.file: 47 | with closing(open(args.file)) as file: 48 | console.runsource(file.read(), args.file, 'exec') 49 | else: 50 | console.interact("Cram version %s\nPython %s, %s %s""" 51 | % (cram.cram_version, platform.python_version(), 52 | platform.system(), platform.machine())) 53 | -------------------------------------------------------------------------------- /src/python/cram/cmd/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import llnl.util.tty as tty 26 | from llnl.util.lang import * 27 | from llnl.util.filesystem import ancestor 28 | 29 | commands = sorted(list_modules(ancestor(__file__))) 30 | SETUP_PARSER = "setup_parser" 31 | DESCRIPTION = "description" 32 | 33 | def get_cmd_function_name(name): 34 | return name.replace("-", "_") 35 | 36 | 37 | def get_module(name): 38 | """Imports the module for a particular command name and returns it.""" 39 | module_name = "%s.%s" % (__name__, name) 40 | module = __import__( 41 | module_name, fromlist=[name, SETUP_PARSER, DESCRIPTION], level=0) 42 | 43 | attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op 44 | attr_setdefault(module, DESCRIPTION, "") 45 | 46 | fn_name = get_cmd_function_name(name) 47 | if not hasattr(module, fn_name): 48 | tty.die("Command module %s (%s) must define function '%s'." 49 | % (module.__name__, module.__file__, fn_name)) 50 | 51 | return module 52 | 53 | 54 | def get_command(name): 55 | """Imports the command's function from a module and returns it.""" 56 | return getattr(get_module(name), get_cmd_function_name(name)) 57 | 58 | -------------------------------------------------------------------------------- /src/python/cram/serialization.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | """\ 26 | This file contains utility functions for serializing data to be 27 | ready by the cram backend. 28 | """ 29 | 30 | import struct 31 | import array 32 | 33 | # Int formats from struct module. 34 | _int_sizes = { 1 : '>B', 35 | 2 : '>H', 36 | 4 : '>I', 37 | 8 : '>Q' } 38 | 39 | # Use big endian, 32-bit unsigned int 40 | _default_int_format = '>I' 41 | 42 | 43 | def write_int(stream, integer, fmt=_default_int_format): 44 | if isinstance(fmt, int): 45 | fmt = _int_sizes[fmt] 46 | packed = struct.pack(fmt, integer) 47 | stream.write(packed) 48 | return len(packed) 49 | 50 | 51 | def read_int(stream, fmt=_default_int_format): 52 | if isinstance(fmt, int): 53 | fmt = _int_sizes[fmt] 54 | size = struct.calcsize(fmt) 55 | packed = stream.read(size) 56 | if len(packed) < size: 57 | raise IOError("Premature end of file") 58 | return struct.unpack(fmt, packed)[0] 59 | 60 | 61 | def write_string(stream, string): 62 | length = len(string) 63 | int_len = write_int(stream, length) 64 | stream.write(string) 65 | return len(string) + int_len 66 | 67 | 68 | def read_string(stream): 69 | length = read_int(stream) 70 | string = stream.read(length) 71 | if len(string) < length: 72 | raise IOError("Premature end of file") 73 | return string 74 | -------------------------------------------------------------------------------- /src/python/llnl/util/filesystem.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import os 26 | import re 27 | import shutil 28 | import errno 29 | import getpass 30 | from contextlib import contextmanager, closing 31 | 32 | @contextmanager 33 | def working_dir(dirname): 34 | orig_dir = os.getcwd() 35 | os.chdir(dirname) 36 | yield 37 | os.chdir(orig_dir) 38 | 39 | 40 | def touch(path): 41 | with closing(open(path, 'a')) as file: 42 | os.utime(path, None) 43 | 44 | 45 | def mkdirp(*paths): 46 | """Python equivalent of mkdir -p: make directory and any parent directories, 47 | and don't raise errors if any of these already exist.""" 48 | for path in paths: 49 | try: 50 | os.makedirs(path) 51 | except OSError as exc: 52 | if (not exc.errno == errno.EEXIST or 53 | not os.path.isdir(path)): 54 | raise 55 | 56 | 57 | def join_path(prefix, *args): 58 | path = str(prefix) 59 | for elt in args: 60 | path = os.path.join(path, str(elt)) 61 | return path 62 | 63 | 64 | def ancestor(dir, n=1): 65 | """Get the nth ancestor of a directory.""" 66 | parent = os.path.abspath(dir) 67 | for i in range(n): 68 | parent = os.path.dirname(parent) 69 | return parent 70 | 71 | 72 | def can_access(file_name): 73 | """True if we have read/write access to the file.""" 74 | return os.access(file_name, os.R_OK|os.W_OK) 75 | -------------------------------------------------------------------------------- /src/python/cram/cmd/pack.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import argparse 26 | from contextlib import closing 27 | 28 | import llnl.util.tty as tty 29 | from cram.cramfile import * 30 | 31 | description = "Pack a command invocation into a cramfile" 32 | 33 | def setup_parser(subparser): 34 | subparser.add_argument('-n', "--nprocs", type=int, dest='nprocs', required=True, 35 | help="Number of processes to run with") 36 | subparser.add_argument('-f', "--file", dest='file', default='cram.job', required=True, 37 | help="File to store command invocation in. Default is 'cram.job'") 38 | subparser.add_argument('-e', "--exe", dest='exe', default=USE_APP_EXE, 39 | help="Optionally specify the executable name for the cram job.") 40 | subparser.add_argument('arguments', nargs=argparse.REMAINDER, 41 | help="Arguments to pass to executable.") 42 | 43 | 44 | def pack(parser, args): 45 | if not args.arguments: 46 | tty.die("You must supply command line arguments to cram pack.") 47 | 48 | if not args.nprocs: 49 | tty.die("You must supply a number of processes to run with.") 50 | 51 | if os.path.isdir(args.file): 52 | tty.die("%s is a directory." % args.file) 53 | 54 | with closing(CramFile(args.file, 'a')) as cf: 55 | cf.pack(args.nprocs, os.getcwd(), args.arguments, os.environ, exe=args.exe) 56 | -------------------------------------------------------------------------------- /src/python/cram/test/scripts/test-pack-script.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env cram-python 2 | ############################################################################## 3 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 4 | # Produced at the Lawrence Livermore National Laboratory. 5 | # 6 | # This file is part of Cram. 7 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 8 | # LLNL-CODE-661100 9 | # 10 | # For details, see https://github.com/scalability-llnl/cram. 11 | # Please also see the LICENSE file for our notice and the LGPL. 12 | # 13 | # This program is free software; you can redistribute it and/or modify 14 | # it under the terms of the GNU General Public License (as published by 15 | # the Free Software Foundation) version 2.1 dated February 1999. 16 | # 17 | # This program is distributed in the hope that it will be useful, but 18 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 20 | # conditions of the GNU General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public License 23 | # along with this program; if not, write to the Free Software Foundation, 24 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 25 | ############################################################################## 26 | 27 | import os 28 | import sys 29 | from tempfile import mktemp 30 | from contextlib import closing 31 | 32 | from cram import * 33 | from cram.test.cramfile import tempfile 34 | 35 | # 36 | # Set up some test jobs. 37 | # 38 | env = os.environ 39 | jobs = [ 40 | Job(2, '/home/gamblin2/ensemble/run.00', [ 'arg1', 'arg2', 'arg3' ], env.copy()), 41 | Job(4, '/home/gamblin2/ensemble/run.01', ['a', 'b', 'c'], env.copy()), 42 | Job(64, '/home/gamblin2/ensemble/run.02', ['d', 'e', 'f'], env.copy()), 43 | Job(128, '/home/gamblin2/ensemble/run.03', ['g', 'h', 'i'], env.copy()), 44 | Job(2, '/home/gamblin2/ensemble/run.04', ['j', 'k', 'l'], env.copy()), 45 | Job(1, '/home/gamblin2/ensemble/run.05', ['m', 'n', 'o'], env.copy()), 46 | Job(3, '/home/gamblin2/ensemble/run.06', ['p', 'q', 'r'], env.copy())] 47 | 48 | jobs[0].env['SPECIAL_VALUE'] = "foo" 49 | jobs[3].env['SPECIAL_VALUE2'] = "bar" 50 | jobs[5].env['SPECIAL_VALUE3'] = "baz" 51 | 52 | 53 | with tempfile() as file_name: 54 | # Make a test cram file. 55 | with closing(CramFile(file_name, 'w')) as cf: 56 | for job in jobs: 57 | cf.pack(job) 58 | 59 | # Verify the test cram file. 60 | with closing(CramFile(file_name, 'r')) as cf: 61 | for i, job in enumerate(cf): 62 | if jobs[i] != job: 63 | print "ERROR: validation failed." 64 | print "Expected: %s" % jobs[i] 65 | print "Got: %s" % cf 66 | sys.exit(1) 67 | -------------------------------------------------------------------------------- /src/python/cram/test/serialization.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import sys 26 | import unittest 27 | import string 28 | import random 29 | from StringIO import StringIO 30 | 31 | from cram.serialization import * 32 | 33 | class SerializationTest(unittest.TestCase): 34 | 35 | def test_integers(self): 36 | """Test writing and reading integers of different sizes from a 37 | stream.""" 38 | stream = StringIO() 39 | sizes = [1, 2, 4, 8] 40 | 41 | for size in sizes: 42 | i = 2 43 | while i < (2**(size*8) - 1): 44 | write_int(stream, i, 8) 45 | write_int(stream, i + 1, 8) 46 | write_int(stream, i - 1, 8) 47 | i **= 2 48 | 49 | stream = StringIO(stream.getvalue()) 50 | for size in sizes: 51 | i = 2 52 | while i < (2**(size*8) - 1): 53 | self.assertEqual(read_int(stream, 8), i) 54 | self.assertEqual(read_int(stream, 8), i + 1) 55 | self.assertEqual(read_int(stream, 8), i - 1) 56 | i **= 2 57 | 58 | 59 | def test_strings(self): 60 | """Test writing and reading strings from a stream.""" 61 | # Make 100 random strings of widely varying length 62 | strings = [] 63 | lengths = range(1, 4096) 64 | for n in range(100): 65 | strings.append(''.join(random.choice(string.ascii_uppercase) 66 | for i in range(random.choice(lengths)))) 67 | 68 | # write all the strings then see if they're the same when we read them. 69 | stream = StringIO() 70 | 71 | for s in strings: 72 | write_string(stream, s) 73 | 74 | stream = StringIO(stream.getvalue()) 75 | 76 | for s in strings: 77 | self.assertEqual(read_string(stream), s) 78 | -------------------------------------------------------------------------------- /wrap/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Lawrence Livermore National Security, LLC. 2 | Produced at the Lawrence Livermore National Laboratory 3 | Written by Todd Gamblin, tgamblin@llnl.gov. 4 | LLNL-CODE-417602 5 | All rights reserved. 6 | 7 | This file is part of Libra. For details, see http://github.com/tgamblin/libra. 8 | Please also read the LICENSE file for further information. 9 | 10 | Redistribution and use in source and binary forms, with or without modification, are 11 | permitted provided that the following conditions are met: 12 | 13 | * Redistributions of source code must retain the above copyright notice, this list of 14 | conditions and the disclaimer below. 15 | * Redistributions in binary form must reproduce the above copyright notice, this list of 16 | conditions and the disclaimer (as noted below) in the documentation and/or other materials 17 | provided with the distribution. 18 | * Neither the name of the LLNS/LLNL nor the names of its contributors may be used to endorse 19 | or promote products derived from this software without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS 22 | OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 23 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 24 | LAWRENCE LIVERMORE NATIONAL SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE 25 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 28 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 29 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | 32 | Additional BSD Notice 33 | 34 | 1. This notice is required to be provided under our contract with the U.S. Department of 35 | Energy (DOE). This work was produced at Lawrence Livermore National Laboratory under 36 | Contract No. DE-AC52-07NA27344 with the DOE. 37 | 38 | 2. Neither the United States Government nor Lawrence Livermore National Security, LLC nor 39 | any of their employees, makes any warranty, express or implied, or assumes any liability 40 | or responsibility for the accuracy, completeness, or usefulness of any information, 41 | apparatus, product, or process disclosed, or represents that its use would not infringe 42 | privately-owned rights. 43 | 44 | 3. Also, reference herein to any specific commercial products, process, or services by trade 45 | name, trademark, manufacturer or otherwise does not necessarily constitute or imply its 46 | endorsement, recommendation, or favoring by the United States Government or Lawrence 47 | Livermore National Security, LLC. The views and opinions of authors expressed herein do 48 | not necessarily state or reflect those of the United States Government or Lawrence 49 | Livermore National Security, LLC, and shall not be used for advertising or product 50 | endorsement purposes. 51 | -------------------------------------------------------------------------------- /src/c/libcram/cram_fargs.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | 26 | // These are defined in the cram library -- they're set when MPI_INIT is called. 27 | extern int cram_argc; 28 | extern const char **cram_argv; 29 | 30 | 31 | /// 32 | /// Cram version of iargc intrinsic supported by many (nearly all?) 33 | /// Fortran 77/90 compilers. This just returns the argc set by Cram 34 | /// in MPI_INIT. 35 | /// 36 | static inline int cram_iargc() { 37 | return cram_argc; 38 | } 39 | 40 | 41 | /// 42 | /// Cram version of getarg intrinsic supported by many (nearly all?) 43 | /// Fortran 77/90 compilers. This version copies cram arguments into 44 | /// variables it's called on. 45 | /// 46 | static inline void cram_getarg(int *i, char *var, int var_len) { 47 | int c = 0; 48 | 49 | if (*i > 0 && *i < cram_argc) { 50 | // copy value of argument into destination. 51 | for (; c < var_len && cram_argv[*i][c]; c++) { 52 | var[c] = cram_argv[*i][c]; 53 | } 54 | } 55 | 56 | // fill the rest with null chars because Fortran. 57 | for (; c < var_len; c++) { 58 | var[c] = '\0'; 59 | } 60 | } 61 | 62 | 63 | /// 64 | /// This macro defines versions of iargc and getarg for each compiler. 65 | /// Each definition just calls the functions above. 66 | /// 67 | /// The names here are temporary; we name them after compilers, but the 68 | /// intrinsics have symbol names like getarg@@XLF_1.0, which a C compiler 69 | /// won't emit. So we write the names out with various suffixes, and we use 70 | /// objcopy afterwards to give the symbols the names we need to override 71 | /// compiler intrinsics. 72 | /// 73 | #define define_for_suffix(s) \ 74 | int iargc_##s() { return cram_iargc(); } \ 75 | void getarg_##s(int *i, char *var, int var_len) { \ 76 | cram_getarg(i, var, var_len); \ 77 | } 78 | 79 | define_for_suffix(gnu) 80 | define_for_suffix(xl) 81 | -------------------------------------------------------------------------------- /src/python/cram/test/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import sys 26 | import unittest 27 | 28 | import llnl.util.tty as tty 29 | 30 | # Module where test packages are located 31 | _test_module = 'cram.test' 32 | 33 | # Names of tests to be included in the test suite 34 | _test_names = ['serialization', 35 | 'cramfile'] 36 | 37 | 38 | def list_tests(): 39 | """Return names of all tests that can be run.""" 40 | return _test_names 41 | 42 | 43 | def run(names, verbose=False): 44 | """Run tests with the supplied names. Names should be a list. If 45 | it's empty, run ALL of the tests.""" 46 | verbosity = 1 if not verbose else 2 47 | 48 | # If they didn't provide names of tests to run, then use the default 49 | # list above. 50 | if not names: 51 | names = _test_names 52 | else: 53 | for test in names: 54 | if test not in _test_names: 55 | tty.error("%s is not a valid test name." % test, 56 | "Valid names are:") 57 | colify(_test_names, indent=4) 58 | sys.exit(1) 59 | 60 | runner = unittest.TextTestRunner(verbosity=verbosity) 61 | 62 | testsRun = errors = failures = skipped = 0 63 | for test in names: 64 | module = _test_module + '.' + test 65 | print module, test 66 | suite = unittest.defaultTestLoader.loadTestsFromName(module) 67 | 68 | tty.msg("Running test: %s" % test) 69 | result = runner.run(suite) 70 | testsRun += result.testsRun 71 | errors += len(result.errors) 72 | failures += len(result.failures) 73 | skipped += len(result.skipped) 74 | 75 | succeeded = not errors and not failures 76 | tty.msg("Tests Complete.", 77 | "%5d tests run" % testsRun, 78 | "%5d skipped" % skipped, 79 | "%5d failures" % failures, 80 | "%5d errors" % errors) 81 | 82 | if not errors and not failures: 83 | tty.info("OK", format='g') 84 | else: 85 | tty.info("FAIL", format='r') 86 | sys.exit(1) 87 | -------------------------------------------------------------------------------- /src/python/cram/cmd/test-gen.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import os 26 | import llnl.util.tty as tty 27 | from llnl.util.filesystem import mkdirp 28 | 29 | from cram import * 30 | 31 | description = "Generate directory structure and a cram file for a test problem." 32 | 33 | def setup_parser(subparser): 34 | subparser.add_argument("nprocs", type=int, help="Number of processes to run with.") 35 | subparser.add_argument("job_size", type=int, help="Size of each test job.") 36 | subparser.add_argument("--print-mem-usage", action='store_true', dest='print_mem', 37 | default=False, help="Print memory usage when done.") 38 | subparser.add_argument("--jobs-per-dir", type=int, dest='jobs_per_dir', default=1024, 39 | help="Number of jobs per directory") 40 | 41 | 42 | def make_test(num_procs, job_size, jobs_per_dir): 43 | test_dir = "%s/cram-test-outputs/%s/%s" % ( 44 | os.getcwd(), num_procs, job_size) 45 | mkdirp(test_dir) 46 | 47 | cfname = os.path.join(test_dir, "cram.job") 48 | 49 | cf = CramFile(cfname, 'w') 50 | for i, rank in enumerate(xrange(0, num_procs, job_size)): 51 | os.environ["CRAM_JOB_ID"] = str(i) 52 | args = ['foo', 'bar', 'baz', str(i)] 53 | 54 | if i % jobs_per_dir == 0: 55 | wdir = "%s/wdir.%d" % (test_dir, i / jobs_per_dir) 56 | mkdirp(wdir) 57 | 58 | cf.pack(job_size, wdir, args, os.environ) 59 | 60 | del os.environ["CRAM_JOB_ID"] 61 | cf.close() 62 | 63 | return test_dir, cfname 64 | 65 | 66 | def test_gen(parser, args): 67 | test_dir, cram_file = make_test(args.nprocs, args.job_size, args.jobs_per_dir) 68 | tty.msg("Created a test directory:", test_dir) 69 | tty.msg("And a cram file:", cram_file) 70 | tty.msg("To check that everything works:", 71 | "1. Run $CRAM_PREFIX/libexec/cram/cram-test with CRAM_FILE=%s." % cram_file, 72 | "2. Run cram test-verify on the directory.") 73 | if args.print_mem: 74 | import resource 75 | kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss 76 | tty.msg("Memory usage: %d KiB" % kb) 77 | 78 | 79 | -------------------------------------------------------------------------------- /src/c/test/cram-read-file-test.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "cram_file.h" 30 | 31 | 32 | void read_entire_cram_file(cram_file_t *file) { 33 | if (!cram_file_has_more_jobs(file)) { 34 | return; 35 | } 36 | 37 | // space for raw, compressed job record. 38 | char *job_record = malloc(file->max_job_size); 39 | 40 | // First job is special because we don't have to decompress 41 | cram_job_t first_job; 42 | cram_file_next_job(file, job_record); 43 | cram_job_decompress(job_record, NULL, &first_job); 44 | 45 | // Rest of jobs are based on first job. Do not decompress any 46 | // of them. This is just a read benchmark. 47 | while (cram_file_has_more_jobs(file)) { 48 | cram_file_next_job(file, job_record); 49 | } 50 | 51 | // free everything up. 52 | free(job_record); 53 | cram_job_free(&first_job); 54 | } 55 | 56 | 57 | /// 58 | /// Simple test to read in a cram file. Prints out entire time it took to 59 | /// read. Use this with CRAM_BUFFER_SIZE environment variable to test the 60 | /// performance of various buffer sizes. 61 | /// 62 | int main(int argc, char **argv) { 63 | PMPI_Init(&argc, &argv); 64 | 65 | int rank; 66 | PMPI_Comm_rank(MPI_COMM_WORLD, &rank); 67 | 68 | if (argc < 2) { 69 | if (rank == 0) { 70 | fprintf(stderr, "Usage: cram-read-file-test \n"); 71 | fprintf(stderr, " Reads an entire cram file using the C API and prints " 72 | "out the time it took.\n"); 73 | } 74 | PMPI_Finalize(); 75 | exit(1); 76 | } 77 | 78 | const char *filename = argv[1]; 79 | 80 | cram_file_t file; 81 | if (!cram_file_open(filename, &file)) { 82 | printf("failed to map with errno %d: %s\n", errno, strerror(errno)); 83 | PMPI_Finalize(); 84 | exit(1); 85 | } 86 | 87 | printf("Reading file: %25s\n", filename); 88 | 89 | double start_time = PMPI_Wtime(); 90 | read_entire_cram_file(&file); 91 | double end_time = PMPI_Wtime(); 92 | 93 | double elapsed = end_time - start_time; 94 | printf("Read entire file in %.6f seconds\n", elapsed); 95 | 96 | PMPI_Finalize(); 97 | exit(0); 98 | } 99 | -------------------------------------------------------------------------------- /wrap/WrapConfig.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # @file WrapConfig.cmake 3 | # Contains macros for using the wrap script in a build environment. 4 | # 5 | # @author Todd Gamblin 6 | # @date 19 May 2011 7 | if(NOT DEFINED WRAP) 8 | message(FATAL_ERROR 9 | "WRAP variable must be set to location of wrap.py before including WrapConfig.cmake!") 10 | endif() 11 | 12 | if (NOT Wrap_CONFIG_LOADED) 13 | set(Wrap_CONFIG_LOADED TRUE) 14 | 15 | # This variable allows users to use the wrap.py script directly, if desired. 16 | set(Wrap_EXECUTABLE ${WRAP}) 17 | 18 | # add_wrapped_file(file_name wrapper_name [flags]) 19 | # 20 | # This macro adds a command to generate from to the 21 | # build. Properties on are also set so that CMake knows that it 22 | # is generated. 23 | # 24 | # Optionally, flags may be supplied to pass to the wrapper generator. 25 | # 26 | function(add_wrapped_file file_name wrapper_name) 27 | set(file_path ${CMAKE_CURRENT_BINARY_DIR}/${file_name}) 28 | set(wrapper_path ${CMAKE_CURRENT_SOURCE_DIR}/${wrapper_name}) 29 | 30 | # Play nice with FindPythonInterp -- use the interpreter if it was found, 31 | # otherwise use the script directly. 32 | if (PYTHON_EXECUTABLE) 33 | set(command ${PYTHON_EXECUTABLE}) 34 | set(script_arg ${Wrap_EXECUTABLE}) 35 | else() 36 | set(command ${Wrap_EXECUTABLE}) 37 | set(script_arg "") 38 | endif() 39 | 40 | # Backward compatibility for old FindMPIs that did not have MPI_C_INCLUDE_PATH 41 | if (NOT MPI_C_INCLUDE_PATH) 42 | set(MPI_C_INCLUDE_PATH ${MPI_INCLUDE_PATH}) 43 | endif() 44 | if (NOT MPI_C_COMPILER) 45 | set(MPI_C_COMPILER ${MPI_COMPILER}) 46 | endif() 47 | 48 | # Play nice with FindMPI. This will deduce the appropriate MPI compiler to use 49 | # for generating wrappers 50 | if (MPI_C_INCLUDE_PATH) 51 | set(wrap_includes "") 52 | foreach(include ${MPI_C_INCLUDE_PATH}) 53 | set(wrap_includes ${wrap_includes} -I ${include}) 54 | endforeach() 55 | endif() 56 | set(wrap_compiler -c ${CMAKE_C_COMPILER}) 57 | if (MPI_C_COMPILER) 58 | set(wrap_compiler -c ${MPI_C_COMPILER}) 59 | endif() 60 | 61 | if (ARGN) 62 | # Prefer directly passed in flags. 63 | list(GET ARGN 0 wrap_flags) 64 | else() 65 | # Otherwise, look in the source file properties 66 | get_source_file_property(wrap_flags ${wrapper_name} WRAP_FLAGS) 67 | if (wrap_flags STREQUAL NOTFOUND) 68 | # If no spefific flags, grab them from the WRAP_FLAGS environment variable. 69 | set(wrap_flags "") 70 | if (NOT WRAP_FLAGS STREQUAL "") 71 | set(wrap_flags "${WRAP_FLAGS}") 72 | endif() 73 | endif() 74 | endif() 75 | 76 | # Mark target file as generated so the build system knows what to do w/it 77 | set_source_files_properties(${file_path} PROPERTIES GENERATED TRUE) 78 | 79 | # Add a command to automatically wrap files. 80 | add_custom_command( 81 | OUTPUT ${file_path} 82 | COMMAND ${command} 83 | ARGS ${script_arg} ${wrap_compiler} ${wrap_includes} ${wrap_flags} ${wrapper_path} -o ${file_path} 84 | WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" 85 | DEPENDS ${wrapper_path} 86 | COMMENT "Generating ${file_name} from ${wrapper_name}" 87 | VERBATIM) 88 | 89 | # Add generated files to list of things to be cleaned for the directory. 90 | get_directory_property(cleanfiles ADDITIONAL_MAKE_CLEAN_FILES) 91 | list(APPEND cleanfiles ${file_name}) 92 | set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${cleanfiles}") 93 | endfunction() 94 | 95 | endif() 96 | -------------------------------------------------------------------------------- /src/python/llnl/util/tty/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import sys 26 | from llnl.util.tty.color import * 27 | 28 | debug = False 29 | verbose = False 30 | indent = " " 31 | 32 | def msg(message, *args): 33 | cprint("@*b{==>} %s" % cescape(message)) 34 | for arg in args: 35 | print indent + str(arg) 36 | 37 | 38 | def info(message, *args, **kwargs): 39 | format = kwargs.get('format', '*b') 40 | cprint("@%s{==>} %s" % (format, cescape(message))) 41 | for arg in args: 42 | print indent + str(arg) 43 | 44 | 45 | def verbose(message, *args): 46 | if verbose: 47 | info(message, *args, format='c') 48 | 49 | 50 | def debug(*args): 51 | if debug: 52 | info("Debug: " + message, *args, format='*g') 53 | 54 | 55 | def error(message, *args): 56 | info("Error: " + message, *args, format='*r') 57 | 58 | 59 | def warn(message, *args): 60 | info("Warning: " + message, *args, format='*Y') 61 | 62 | 63 | def die(message, *args): 64 | error(message, *args) 65 | sys.exit(1) 66 | 67 | 68 | def pkg(message): 69 | """Outputs a message with a package icon.""" 70 | import platform 71 | from version import Version 72 | 73 | mac_ver = platform.mac_ver()[0] 74 | if mac_ver and Version(mac_ver) >= Version('10.7'): 75 | print u"\U0001F4E6" + indent, 76 | else: 77 | cwrite('@*g{[+]} ') 78 | print message 79 | 80 | 81 | def get_number(prompt, **kwargs): 82 | default = kwargs.get('default', None) 83 | abort = kwargs.get('abort', None) 84 | 85 | if default is not None and abort is not None: 86 | prompt += ' (default is %s, %s to abort) ' % (default, abort) 87 | elif default is not None: 88 | prompt += ' (default is %s) ' % default 89 | elif abort is not None: 90 | prompt += ' (%s to abort) ' % abort 91 | 92 | number = None 93 | while number is None: 94 | ans = raw_input(prompt) 95 | if ans == str(abort): 96 | return None 97 | 98 | if ans: 99 | try: 100 | number = int(ans) 101 | if number < 1: 102 | msg("Please enter a valid number.") 103 | number = None 104 | except ValueError: 105 | msg("Please enter a valid number.") 106 | elif default is not None: 107 | number = default 108 | return number 109 | -------------------------------------------------------------------------------- /src/python/cram/cmd/info.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import argparse 26 | import math 27 | 28 | import llnl.util.tty as tty 29 | from cram.cramfile import * 30 | 31 | description = "Display information about a cramfile." 32 | 33 | def setup_parser(subparser): 34 | subparser.add_argument('-a', "--all", action='store_true', dest='all', 35 | help="Print information on ALL jobs in the cram file.") 36 | subparser.add_argument('-j', "--job", type=int, dest='job', 37 | help="Specific job id to display in more detail.") 38 | subparser.add_argument('-n', type=int, dest='num_lines', default=10, 39 | help="Number of job lines to print") 40 | subparser.add_argument('cramfile', help="Cram file to display.") 41 | 42 | 43 | def write_header(args, cf): 44 | print "Name:%25s" % args.cramfile 45 | print "Number of Jobs: %12d" % cf.num_jobs 46 | print "Total Procs: %12d" % cf.num_procs 47 | print "Cram version: %12d" % cf.version 48 | print "Max job record: %12d" % cf.max_job_size 49 | 50 | 51 | def write_job_summary(args, cf): 52 | print "Job command lines:" 53 | 54 | for i, job in enumerate(cf): 55 | if i >= args.num_lines: 56 | print " ..." 57 | print " [%d more]" % (cf.num_jobs - i) 58 | break 59 | print "%5d %5d procs %s" % (i, job.num_procs, ' '.join(job.args)) 60 | 61 | 62 | def write_job_info(job): 63 | print " Num procs: %d" % job.num_procs 64 | print " Working dir: %s" % job.working_dir 65 | print " Arguments:" 66 | print " " + ' '.join(job.args) 67 | print " Environment:" 68 | for key in sorted(job.env): 69 | print " '%s' : '%s'" % (key, job.env[key]) 70 | 71 | 72 | def info(parser, args): 73 | if not args.cramfile: 74 | tty.error("You must specify a file to display with cram info.") 75 | 76 | with closing(CramFile(args.cramfile, 'r')) as cf: 77 | if args.all: 78 | write_header(args, cf) 79 | print 80 | print "Job information:" 81 | for i, job in enumerate(cf): 82 | print "Job %d:" % i 83 | write_job_info(job) 84 | 85 | elif args.job is not None: 86 | if args.job < 0 or args.job >= len(cf): 87 | tty.die("No job %d in this cram file." % args.job) 88 | print "Job %d:" % args.job 89 | for i, job in enumerate(cf): 90 | if i == args.job: 91 | write_job_info(job) 92 | break 93 | 94 | else: 95 | write_header(args, cf) 96 | print 97 | write_job_summary(args, cf) 98 | -------------------------------------------------------------------------------- /src/python/cram/cmd/test-verify.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import os 26 | import re 27 | import glob 28 | from contextlib import closing 29 | 30 | import llnl.util.tty as tty 31 | from llnl.util.filesystem import mkdirp 32 | 33 | from cram import * 34 | 35 | description = "Check a test created by cram test-gen." 36 | 37 | def setup_parser(subparser): 38 | subparser.add_argument( 39 | "directory", type=str, help="Test directory created by cram test-gen.") 40 | 41 | 42 | def check_file(file_name, *tests): 43 | """Check a file for some important values and die if they're not 44 | as expected.""" 45 | results = [None] * len(tests) 46 | with closing(open(file_name)) as f: 47 | for line in f: 48 | count = 0 49 | for i, (regex, t, e, m) in enumerate(tests): 50 | match = re.search(regex, line) 51 | if match: 52 | results[i] = t(match.group(1)) 53 | count += 1 54 | 55 | if count == len(tests): 56 | break 57 | 58 | for i, (regex, t, expected, msg) in enumerate(tests): 59 | if results[i] is None or results[i] != expected: 60 | tty.die(msg % (file_name, expected)) 61 | 62 | 63 | def verify_test(test_dir): 64 | cfname = os.path.join(test_dir, "cram.job") 65 | 66 | test_dir = test_dir.rstrip('/') 67 | match = re.search(r'/(\d+)/(\d+)', test_dir) 68 | if not match: 69 | tty.die("This directory was not created by cram test-gen") 70 | 71 | num_procs = int(match.group(1)) 72 | job_size = int(match.group(2)) 73 | 74 | tty.msg("Checking test with %d procs and %d procs per job." % ( 75 | num_procs, job_size)) 76 | 77 | cf = CramFile(cfname, 'r') 78 | if not cf.num_procs == num_procs: 79 | tty.die("Cram file had %d procs but expected %d" % (cf.num_procs, num_procs)) 80 | cf.close() 81 | 82 | os.chdir(test_dir) 83 | files = glob.glob('*/cram*out') 84 | 85 | short_names = set(os.path.basename(f) for f in files) 86 | for i, rank in enumerate(xrange(0, num_procs, job_size)): 87 | name = "cram.%d.out" % i 88 | if not name in short_names: 89 | tty.die("Couldn't find this output file: %s.") 90 | 91 | for file_name in files: 92 | match = re.search(r'cram.(\d+).out', file_name) 93 | assert(match) 94 | job_id = int(match.group(1)) 95 | 96 | results = check_file( 97 | file_name, 98 | # regex with group type expected error msg if not equal to expected 99 | (r'CRAM_JOB_ID=(\d+)', int, job_id, "%s does not contain CRAM_JOB_ID=%d"), 100 | (r'foo bar baz (\d+)', int, job_id, "%s does not contain args 'foo bar baz %d'"), 101 | (r'Job size: +(\d+)', int, job_size, "%s has incorrect job size: %d"), 102 | (r'Real job size: +(\d+)', int, num_procs, "%s has incorrect job size: %d")) 103 | 104 | def test_verify(parser, args): 105 | verify_test(args.directory) 106 | tty.msg("Success! All jobs look ok.") 107 | -------------------------------------------------------------------------------- /src/c/test/cram-test.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | #include 26 | #include 27 | #include 28 | 29 | extern const char **environ; 30 | 31 | int main(int argc, char **argv) { 32 | MPI_Init(&argc, &argv); 33 | 34 | // Get ranks and real ranks for this job. 35 | int rank, size; 36 | MPI_Comm_size(MPI_COMM_WORLD, &size); 37 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 38 | 39 | int real_rank, real_size; 40 | PMPI_Comm_size(MPI_COMM_WORLD, &real_size); 41 | PMPI_Comm_rank(MPI_COMM_WORLD, &real_rank); 42 | 43 | // Try a simple allreduce 44 | int sum; 45 | MPI_Allreduce(&rank, &sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); 46 | 47 | // check the result against a sequential sum 48 | int check_sum = 0; 49 | for (int i=0; i < size; i++) { 50 | check_sum += i; 51 | } 52 | 53 | // Gather real ranks so we can write out the mapping. 54 | int real_ranks[size]; 55 | MPI_Gather(&real_rank, 1, MPI_INT, 56 | real_ranks, 1, MPI_INT, 0, MPI_COMM_WORLD); 57 | 58 | if (rank == 0) { 59 | // Print out arguments 60 | // Print out status of various tests 61 | fprintf(stdout, "=========================================================\n"); 62 | fprintf(stdout, "Test results\n"); 63 | fprintf(stdout, "=========================================================\n"); 64 | if (sum == check_sum) { 65 | fprintf(stdout, "Allreduce checksum passed.\n"); 66 | } else { 67 | fprintf(stdout, "Allreduce checksum failed!.\n"); 68 | fprintf(stdout, " Expected: %d\n", check_sum); 69 | fprintf(stdout, " Actual: %d\n", sum); 70 | } 71 | fprintf(stdout, "\n\n"); 72 | 73 | // Print out job size information, as well as real size info. 74 | fprintf(stdout, "=========================================================\n"); 75 | fprintf(stdout, "Job info\n"); 76 | fprintf(stdout, "=========================================================\n"); 77 | fprintf(stdout, " Job size: %d\n", size); 78 | fprintf(stdout, " Real job size: %d\n", real_size); 79 | fprintf(stdout, "\n"); 80 | fprintf(stdout, " Arguments\n"); 81 | fprintf(stdout, "---------------------------------------------------------\n"); 82 | fprintf(stdout, " "); 83 | for (int i=0; i < argc; i++) { 84 | if (i != 0) fprintf(stdout, " "); 85 | fprintf(stdout, "%s", argv[i]); 86 | } 87 | fprintf(stdout, "\n\n"); 88 | 89 | // Rank mapping 90 | fprintf(stdout, " Rank mapping\n"); 91 | fprintf(stdout, "---------------------------------------------------------\n"); 92 | for (int i=0; i < size; i++) { 93 | fprintf(stdout, " %5d <- %5d\n", i, real_ranks[i]); 94 | } 95 | fprintf(stdout, "\n\n"); 96 | 97 | // Print out environment 98 | fprintf(stdout, " Environment variables\n"); 99 | fprintf(stdout, "---------------------------------------------------------\n"); 100 | for (const char **var=environ; *var; var++) { 101 | fprintf(stdout, " %s\n", *var); 102 | } 103 | 104 | // Print something to stderr just to make sure that works too. 105 | fprintf(stderr, "Can print to stderr.\n"); 106 | } 107 | 108 | MPI_Finalize(); 109 | } 110 | -------------------------------------------------------------------------------- /src/python/cram/test/cramfile.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import os 26 | from tempfile import mktemp 27 | import unittest 28 | import random 29 | from contextlib import contextmanager, closing 30 | 31 | import cram.cramfile as cramfile 32 | from cram.cramfile import CramFile, Job 33 | 34 | many_jobs = 4096 35 | 36 | @contextmanager 37 | def tempfile(): 38 | tmp = mktemp('.tmp', 'cramfile-test-') 39 | yield tmp 40 | os.unlink(tmp) 41 | 42 | 43 | def random_jobs(num_jobs): 44 | """Generate num_jobs random jobs. The jobs' environments and command 45 | line arguments will differ slightly""" 46 | jobs = [] 47 | for i in range(num_jobs): 48 | # randomly construct a working environment 49 | num_procs = random.choice((1,2,4,8,16)) 50 | working_dir = '/path/to/run-%d-%d' % (num_procs, i) 51 | args = ['-n', str(num_procs), 'foo', 'bar', 'baz', str(i)] 52 | 53 | env = os.environ.copy() 54 | env['WORKING_DIR'] = working_dir 55 | env['INDEX'] = str(i) 56 | if i == 0: 57 | if 'PATH' not in env: 58 | env['PATH'] = "this/is/a:/fake/path" 59 | elif random.choice((0,1,2)): 60 | del env['PATH'] 61 | 62 | jobs.append(Job(num_procs, working_dir, args, env)) 63 | 64 | return jobs 65 | 66 | 67 | class CramFileTest(unittest.TestCase): 68 | 69 | def test_open_for_write(self): 70 | """Test writing out a header and reading it back in.""" 71 | with tempfile() as tmp: 72 | with closing(CramFile(tmp, 'w')) as cf: 73 | self.assertEqual(cf.num_jobs, 0) 74 | self.assertEqual(cf.version, cramfile._version) 75 | 76 | with closing(CramFile(tmp, 'r')) as cf: 77 | self.assertTrue(cf.num_jobs == 0) 78 | 79 | 80 | def check_write_single_record(self, mode): 81 | """Test that packing a single, simple job record works in both write 82 | mode and append mode.""" 83 | num_procs = 64 84 | working_dir = '/foo/bar/baz' 85 | args = ['foo', 'bar', 'baz'] 86 | env = { 'foo' : 'bar', 87 | 'bar' : 'baz', 88 | 'baz' : 'quux' } 89 | 90 | with tempfile() as tmp: 91 | with closing(CramFile(tmp, mode)) as cf: 92 | cf.pack(Job(num_procs, working_dir, args, env)) 93 | self.assertEqual(cf.num_jobs, 1) 94 | self.assertEqual(cf.num_procs, 64) 95 | 96 | with closing(CramFile(tmp, 'r')) as cf: 97 | self.assertEqual(cf.num_jobs, 1) 98 | self.assertEqual(cf.num_procs, 64) 99 | 100 | job = next(iter(cf)) 101 | self.assertEqual(num_procs, job.num_procs) 102 | self.assertEqual(working_dir, job.working_dir) 103 | self.assertEqual(args, job.args) 104 | self.assertEqual(env, job.env) 105 | 106 | 107 | def test_write_single_record(self): 108 | self.check_write_single_record('w') 109 | 110 | 111 | def test_write_single_record(self): 112 | self.check_write_single_record('a') 113 | 114 | 115 | def test_write_a_lot(self): 116 | jobs = random_jobs(many_jobs) 117 | total_procs = 0 118 | 119 | with tempfile() as tmp: 120 | with closing(CramFile(tmp, 'w')) as cf: 121 | for job in jobs: 122 | cf.pack(job) 123 | total_procs += job.num_procs 124 | 125 | with closing(CramFile(tmp, 'r')) as cf: 126 | self.assertEqual(many_jobs, cf.num_jobs) 127 | self.assertEqual(total_procs, cf.num_procs) 128 | self.assertListEqual(jobs, [j for j in cf]) 129 | 130 | 131 | def test_append_a_lot(self): 132 | jobs = random_jobs(many_jobs) 133 | total_procs = 0 134 | 135 | with tempfile() as tmp: 136 | for job in jobs: 137 | with closing(CramFile(tmp, 'a')) as cf: 138 | cf.pack(job) 139 | total_procs += job.num_procs 140 | 141 | with closing(CramFile(tmp, 'r')) as cf: 142 | self.assertEqual(many_jobs, cf.num_jobs) 143 | self.assertEqual(total_procs, cf.num_procs) 144 | self.assertListEqual(jobs, [j for j in cf]) 145 | -------------------------------------------------------------------------------- /src/python/llnl/util/lang.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | import os 26 | import re 27 | import sys 28 | import functools 29 | import inspect 30 | 31 | # Ignore emacs backups when listing modules 32 | ignore_modules = [r'^\.#', '~$'] 33 | 34 | def caller_locals(): 35 | """This will return the locals of the *parent* of the caller. 36 | This allows a fucntion to insert variables into its caller's 37 | scope. Yes, this is some black magic, and yes it's useful 38 | for implementing things like depends_on and provides. 39 | """ 40 | stack = inspect.stack() 41 | try: 42 | return stack[2][0].f_locals 43 | finally: 44 | del stack 45 | 46 | 47 | def get_calling_package_name(): 48 | """Make sure that the caller is a class definition, and return 49 | the module's name. This is useful for getting the name of 50 | spack packages from inside a relation function. 51 | """ 52 | stack = inspect.stack() 53 | try: 54 | # get calling function name (the relation) 55 | relation = stack[1][3] 56 | 57 | # Make sure locals contain __module__ 58 | caller_locals = stack[2][0].f_locals 59 | finally: 60 | del stack 61 | 62 | if not '__module__' in caller_locals: 63 | raise ScopeError(relation) 64 | 65 | module_name = caller_locals['__module__'] 66 | base_name = module_name.split('.')[-1] 67 | return base_name 68 | 69 | 70 | def attr_required(obj, attr_name): 71 | """Ensure that a class has a required attribute.""" 72 | if not hasattr(obj, attr_name): 73 | tty.die("No required attribute '%s' in class '%s'" 74 | % (attr_name, obj.__class__.__name__)) 75 | 76 | 77 | def attr_setdefault(obj, name, value): 78 | """Like dict.setdefault, but for objects.""" 79 | if not hasattr(obj, name): 80 | setattr(obj, name, value) 81 | return getattr(obj, name) 82 | 83 | 84 | def has_method(cls, name): 85 | for base in inspect.getmro(cls): 86 | if base is object: 87 | continue 88 | if name in base.__dict__: 89 | return True 90 | return False 91 | 92 | 93 | def memoized(obj): 94 | """Decorator that caches the results of a function, storing them 95 | in an attribute of that function.""" 96 | cache = obj.cache = {} 97 | @functools.wraps(obj) 98 | def memoizer(*args, **kwargs): 99 | if args not in cache: 100 | cache[args] = obj(*args, **kwargs) 101 | return cache[args] 102 | return memoizer 103 | 104 | 105 | def list_modules(directory, **kwargs): 106 | """Lists all of the modules, excluding __init__.py, in a 107 | particular directory. Listed packages have no particular 108 | order.""" 109 | list_directories = kwargs.setdefault('directories', True) 110 | 111 | for name in os.listdir(directory): 112 | if name == '__init__.py': 113 | continue 114 | 115 | path = os.path.join(directory, name) 116 | if list_directories and os.path.isdir(path): 117 | init_py = os.path.join(path, '__init__.py') 118 | if os.path.isfile(init_py): 119 | yield name 120 | 121 | elif name.endswith('.py'): 122 | if not any(re.search(pattern, name) for pattern in ignore_modules): 123 | yield re.sub('.py$', '', name) 124 | 125 | 126 | def key_ordering(cls): 127 | """Decorates a class with extra methods that implement rich comparison 128 | operations and __hash__. The decorator assumes that the class 129 | implements a function called _cmp_key(). The rich comparison operations 130 | will compare objects using this key, and the __hash__ function will 131 | return the hash of this key. 132 | 133 | If a class already has __eq__, __ne__, __lt__, __le__, __gt__, or __ge__ 134 | defined, this decorator will overwrite them. If the class does not 135 | have a _cmp_key method, then this will raise a TypeError. 136 | """ 137 | def setter(name, value): 138 | value.__name__ = name 139 | setattr(cls, name, value) 140 | 141 | if not has_method(cls, '_cmp_key'): 142 | raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__) 143 | 144 | setter('__eq__', lambda s,o: o is not None and s._cmp_key() == o._cmp_key()) 145 | setter('__lt__', lambda s,o: o is not None and s._cmp_key() < o._cmp_key()) 146 | setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key()) 147 | 148 | setter('__ne__', lambda s,o: o is None or s._cmp_key() != o._cmp_key()) 149 | setter('__gt__', lambda s,o: o is None or s._cmp_key() > o._cmp_key()) 150 | setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key()) 151 | 152 | setter('__hash__', lambda self: hash(self._cmp_key())) 153 | 154 | return cls 155 | 156 | 157 | @key_ordering 158 | class HashableMap(dict): 159 | """This is a hashable, comparable dictionary. Hash is performed on 160 | a tuple of the values in the dictionary.""" 161 | def _cmp_key(self): 162 | return tuple(sorted(self.values())) 163 | 164 | 165 | def copy(self): 166 | """Type-agnostic clone method. Preserves subclass type.""" 167 | # Construct a new dict of my type 168 | T = type(self) 169 | clone = T() 170 | 171 | # Copy everything from this dict into it. 172 | for key in self: 173 | clone[key] = self[key].copy() 174 | return clone 175 | 176 | 177 | def in_function(function_name): 178 | """True if the caller was called from some function with 179 | the supplied Name, False otherwise.""" 180 | stack = inspect.stack() 181 | try: 182 | for elt in stack[2:]: 183 | if elt[3] == function_name: 184 | return True 185 | return False 186 | finally: 187 | del stack 188 | -------------------------------------------------------------------------------- /src/c/libcram/cram_file.h: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | #ifndef cram_cram_file_h 26 | #define cram_cram_file_h 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #ifdef __cplusplus 35 | #define EXTERN_C extern "C" 36 | #else 37 | #define EXTERN_C 38 | #endif // __cplusplus 39 | 40 | /// 41 | /// cram_file_t is used to read in and broadcast raw cram file data. 42 | /// 43 | struct cram_file_t { 44 | int num_jobs; //!< Total number of jobs in cram file 45 | int total_procs; //!< Total number of processes in all jobs. 46 | int version; //!< Version of cram that wrote this file. 47 | int max_job_size; //!< Size of largest job record in this file. 48 | FILE *fd; //!< C file pointer for the cram file. 49 | 50 | int cur_job_record_size; //!< Size of the current job record 51 | int cur_job_procs; //!< Number of proceses in the current job. 52 | int cur_job_id; //!< Id of the current job. 53 | }; 54 | typedef struct cram_file_t cram_file_t; 55 | 56 | 57 | /// 58 | /// Represents a single job in a cram file. This can be extracted from a 59 | /// cram_file using cram_file_find_job. 60 | /// 61 | struct cram_job_t { 62 | int num_procs; //!< Number of processes in this job. 63 | const char *working_dir; //!< Working directory to use for job 64 | 65 | int num_args; //!< Number of command line arguments 66 | const char **args; //!< Array of arguments. 67 | 68 | int num_env_vars; //!< Number of environment variables. 69 | const char **keys; //!< Array of keys of length 70 | const char **values; //!< Array of corresponding values 71 | }; 72 | typedef struct cram_job_t cram_job_t; 73 | 74 | 75 | /// 76 | /// Open a cram file into memory and read its header information 77 | /// into the supplied struct. This is a local operation. 78 | /// 79 | /// @param[in] filename Name of file to open 80 | /// @param[out] file Descriptor for the opened cram file. 81 | /// 82 | /// @return true if successful, false otherwise. 83 | /// 84 | EXTERN_C 85 | bool cram_file_open(const char *filename, cram_file_t *file); 86 | 87 | 88 | /// 89 | /// Free buffers and files associated with the cram file object. 90 | /// After calling, the file is invalid and should no longer be accessed. 91 | /// 92 | /// This is not collective and can be done at any time after cram_file_open. 93 | /// 94 | /// @param[in] file Cram file to close. 95 | /// 96 | EXTERN_C 97 | void cram_file_close(const cram_file_t *file); 98 | 99 | 100 | /// 101 | /// Whether the cram file has remaining job records to read. 102 | /// 103 | /// @param[in] file A cram file. 104 | /// 105 | bool cram_file_has_more_jobs(const cram_file_t *file); 106 | 107 | 108 | /// 109 | /// Read the next job into the job_record buffer. Metadata about 110 | /// the job can be found in the file buffer after this call. 111 | /// 112 | /// Return true if successful, false on error. 113 | /// 114 | /// @param[in] file Cram file to advance. 115 | /// @param[out] job_record Buffer to store uncompressed bytes in. 116 | /// 117 | /// Job record should be of size file->max_job_record. 118 | /// 119 | bool cram_file_next_job(cram_file_t *file, char *job_record); 120 | 121 | 122 | /// 123 | /// Broadcast a local cram file to all processes on a communicator. 124 | /// This is a collective operation. 125 | /// 126 | /// @param[in] file File to broadcast jobs from. Should be newly opened. 127 | /// @param[in] root Rank where the file is valid (i.e. root of bcast). 128 | /// @param[out] job The job this process should execute. 129 | /// @param[out] id Unique id for this process's job (use it to split world) 130 | /// @param[in] comm Communicator on which file should be bcast. 131 | /// 132 | EXTERN_C 133 | void cram_file_bcast_jobs(cram_file_t *file, int root, cram_job_t *job, int *id, 134 | MPI_Comm comm); 135 | 136 | 137 | /// 138 | /// Write out entire contents of cram file to the supplied file descriptor. 139 | /// After this operation the cram file is completely read. 140 | /// 141 | /// @param[in] file A cram file. 142 | /// 143 | EXTERN_C 144 | void cram_file_cat(cram_file_t *file); 145 | 146 | 147 | /// 148 | /// Decompress raw bytes from a job record into a cram_job_decompress. 149 | /// 150 | /// For the first job in a cram file, supply NULL for base. Subsequent jobs 151 | /// have their environment compressed by comparing it to the first job's 152 | /// environment. For these jobs, pass in a pointer to the base job so that 153 | /// this function can apply differences to the first job. 154 | /// 155 | /// @param[in] job_record Compressed job record from a cram file. 156 | /// @param[in] offset Offset in file. 157 | /// @param[in] base First job in the cram file. Pass NULL to 158 | /// read the first job out of the file. 159 | /// 160 | EXTERN_C 161 | void cram_job_decompress(const char *job_record, 162 | const cram_job_t *base, cram_job_t *job); 163 | 164 | 165 | /// 166 | /// Set up cram environment based on the supplied cram job. 167 | /// This will: 168 | /// 1. Change working directory to the job's working directory. 169 | /// 2. Munge command line arguments to be equal to those of the job. 170 | /// 3. Set environment variables per those defined in the job. 171 | /// 172 | EXTERN_C 173 | void cram_job_setup(const cram_job_t *job, int *argc, const char ***argv); 174 | 175 | 176 | /// 177 | /// Print metadata for a cram job. Includes: 178 | /// 1. Number of processes 179 | /// 2. Working dir 180 | /// 3. Arguments 181 | /// 4. Environment 182 | /// 183 | EXTERN_C 184 | void cram_job_print(const cram_job_t *job); 185 | 186 | 187 | /// 188 | /// Deep copy one cram job into another. 189 | /// 190 | /// @param[in] src Source job to copy from 191 | /// @param[out] dest Destination job to copy to 192 | /// 193 | EXTERN_C 194 | void cram_job_copy(const cram_job_t *src, cram_job_t *dest); 195 | 196 | 197 | /// 198 | /// Deallocate all memory for a job output by cram_file_find_job. 199 | /// 200 | EXTERN_C 201 | void cram_job_free(cram_job_t *job); 202 | 203 | 204 | 205 | #endif // cram_cram_file_h 206 | -------------------------------------------------------------------------------- /src/python/llnl/util/tty/colify.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | # colify 26 | # By Todd Gamblin, tgamblin@llnl.gov 27 | # 28 | # Takes a list of items as input and finds a good columnization of them, 29 | # similar to how gnu ls does. You can pipe output to this script and 30 | # get a tight display for it. This supports both uniform-width and 31 | # variable-width (tighter) columns. 32 | # 33 | # Run colify -h for more information. 34 | # 35 | import os 36 | import sys 37 | import fcntl 38 | import termios 39 | import struct 40 | 41 | def get_terminal_size(): 42 | """Get the dimensions of the console.""" 43 | def ioctl_GWINSZ(fd): 44 | try: 45 | cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) 46 | except: 47 | return 48 | return cr 49 | cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 50 | if not cr: 51 | try: 52 | fd = os.open(os.ctermid(), os.O_RDONLY) 53 | cr = ioctl_GWINSZ(fd) 54 | os.close(fd) 55 | except: 56 | pass 57 | if not cr: 58 | cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) 59 | 60 | return int(cr[1]), int(cr[0]) 61 | 62 | 63 | class ColumnConfig: 64 | def __init__(self, cols): 65 | self.cols = cols 66 | self.line_length = 0 67 | self.valid = True 68 | self.widths = [0] * cols 69 | 70 | def __repr__(self): 71 | attrs = [(a,getattr(self, a)) for a in dir(self) if not a.startswith("__")] 72 | return "" % ", ".join("%s: %r" % a for a in attrs) 73 | 74 | 75 | def config_variable_cols(elts, console_cols, padding): 76 | # Get a bound on the most columns we could possibly have. 77 | lengths = [len(elt) for elt in elts] 78 | max_cols = max(1, console_cols / (min(lengths) + padding)) 79 | max_cols = min(len(elts), max_cols) 80 | 81 | configs = [ColumnConfig(c) for c in xrange(1, max_cols+1)] 82 | for elt, length in enumerate(lengths): 83 | for i, conf in enumerate(configs): 84 | if conf.valid: 85 | col = elt / ((len(elts) + i) / (i + 1)) 86 | padded = length 87 | if col < i: 88 | padded += padding 89 | 90 | if conf.widths[col] < padded: 91 | conf.line_length += padded - conf.widths[col] 92 | conf.widths[col] = padded 93 | conf.valid = (conf.line_length < console_cols) 94 | 95 | try: 96 | config = next(conf for conf in reversed(configs) if conf.valid) 97 | except StopIteration: 98 | # If nothing was valid the screen was too narrow -- just use 1 col. 99 | config = configs[0] 100 | 101 | config.widths = [w for w in config.widths if w != 0] 102 | config.cols = len(config.widths) 103 | return config 104 | 105 | 106 | def config_uniform_cols(elts, console_cols, padding): 107 | max_len = max(len(elt) for elt in elts) + padding 108 | cols = max(1, console_cols / max_len) 109 | cols = min(len(elts), cols) 110 | config = ColumnConfig(cols) 111 | config.widths = [max_len] * cols 112 | return config 113 | 114 | 115 | def isatty(ostream): 116 | force = os.environ.get('COLIFY_TTY', 'false').lower() != 'false' 117 | return force or ostream.isatty() 118 | 119 | 120 | def colify(elts, **options): 121 | # Get keyword arguments or set defaults 122 | output = options.get("output", sys.stdout) 123 | indent = options.get("indent", 0) 124 | padding = options.get("padding", 2) 125 | 126 | # elts needs to be an array of strings so we can count the elements 127 | elts = [str(elt) for elt in elts] 128 | if not elts: 129 | return 130 | 131 | if not isatty(output): 132 | for elt in elts: 133 | output.write("%s\n" % elt) 134 | return 135 | 136 | console_cols = options.get("cols", None) 137 | if not console_cols: 138 | console_cols, console_rows = get_terminal_size() 139 | elif type(console_cols) != int: 140 | raise ValueError("Number of columns must be an int") 141 | console_cols = max(1, console_cols - indent) 142 | 143 | method = options.get("method", "variable") 144 | if method == "variable": 145 | config = config_variable_cols(elts, console_cols, padding) 146 | elif method == "uniform": 147 | config = config_uniform_cols(elts, console_cols, padding) 148 | else: 149 | raise ValueError("method must be one of: " + allowed_methods) 150 | 151 | cols = config.cols 152 | formats = ["%%-%ds" % width for width in config.widths[:-1]] 153 | formats.append("%s") # last column has no trailing space 154 | 155 | rows = (len(elts) + cols - 1) / cols 156 | rows_last_col = len(elts) % rows 157 | 158 | for row in xrange(rows): 159 | output.write(" " * indent) 160 | for col in xrange(cols): 161 | elt = col * rows + row 162 | output.write(formats[col] % elts[elt]) 163 | 164 | output.write("\n") 165 | row += 1 166 | if row == rows_last_col: 167 | cols -= 1 168 | 169 | 170 | if __name__ == "__main__": 171 | import optparse 172 | 173 | cols, rows = get_terminal_size() 174 | parser = optparse.OptionParser() 175 | parser.add_option("-u", "--uniform", action="store_true", default=False, 176 | help="Use uniformly sized columns instead of variable-size.") 177 | parser.add_option("-p", "--padding", metavar="PADDING", action="store", 178 | type=int, default=2, help="Spaces to add between columns. Default is 2.") 179 | parser.add_option("-i", "--indent", metavar="SPACES", action="store", 180 | type=int, default=0, help="Indent the output by SPACES. Default is 0.") 181 | parser.add_option("-w", "--width", metavar="COLS", action="store", 182 | type=int, default=cols, help="Indent the output by SPACES. Default is 0.") 183 | options, args = parser.parse_args() 184 | 185 | method = "variable" 186 | if options.uniform: 187 | method = "uniform" 188 | 189 | if sys.stdin.isatty(): 190 | parser.print_help() 191 | sys.exit(1) 192 | else: 193 | colify([line.strip() for line in sys.stdin], method=method, **options.__dict__) 194 | -------------------------------------------------------------------------------- /src/python/llnl/util/tty/color.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | """ 26 | This file implements an expression syntax, similar to printf, for adding 27 | ANSI colors to text. 28 | 29 | See colorize(), cwrite(), and cprint() for routines that can generate 30 | colored output. 31 | 32 | colorize will take a string and replace all color expressions with 33 | ANSI control codes. If the isatty keyword arg is set to False, then 34 | the color expressions will be converted to null strings, and the 35 | returned string will have no color. 36 | 37 | cwrite and cprint are equivalent to write() and print() calls in 38 | python, but they colorize their output. If the stream argument is 39 | not supplied, they write to sys.stdout. 40 | 41 | Here are some example color expressions: 42 | 43 | @r Turn on red coloring 44 | @R Turn on bright red coloring 45 | @*{foo} Bold foo, but don't change text color 46 | @_{bar} Underline bar, but don't change text color 47 | @*b Turn on bold, blue text 48 | @_B Turn on bright blue text with an underline 49 | @. Revert to plain formatting 50 | @*g{green} Print out 'green' in bold, green text, then reset to plain. 51 | @*ggreen@. Print out 'green' in bold, green text, then reset to plain. 52 | 53 | The syntax consists of: 54 | 55 | color-expr = '@' [style] color-code '{' text '}' | '@.' | '@@' 56 | style = '*' | '_' 57 | color-code = [krgybmcwKRGYBMCW] 58 | text = .* 59 | 60 | '@' indicates the start of a color expression. It can be followed 61 | by an optional * or _ that indicates whether the font should be bold or 62 | underlined. If * or _ is not provided, the text will be plain. Then 63 | an optional color code is supplied. This can be [krgybmcw] or [KRGYBMCW], 64 | where the letters map to black(k), red(r), green(g), yellow(y), blue(b), 65 | magenta(m), cyan(c), and white(w). Lowercase letters denote normal ANSI 66 | colors and capital letters denote bright ANSI colors. 67 | 68 | Finally, the color expression can be followed by text enclosed in {}. If 69 | braces are present, only the text in braces is colored. If the braces are 70 | NOT present, then just the control codes to enable the color will be output. 71 | The console can be reset later to plain text with '@.'. 72 | 73 | To output an @, use '@@'. To output a } inside braces, use '}}'. 74 | """ 75 | import re 76 | import sys 77 | 78 | class ColorParseError(Exception): 79 | """Raised when a color format fails to parse.""" 80 | def __init__(self, message): 81 | super(ColorParseError, self).__init__(message) 82 | 83 | # Text styles for ansi codes 84 | styles = {'*' : '1', # bold 85 | '_' : '4', # underline 86 | None : '0' } # plain 87 | 88 | # Dim and bright ansi colors 89 | colors = {'k' : 30, 'K' : 90, # black 90 | 'r' : 31, 'R' : 91, # red 91 | 'g' : 32, 'G' : 92, # green 92 | 'y' : 33, 'Y' : 93, # yellow 93 | 'b' : 34, 'B' : 94, # blue 94 | 'm' : 35, 'M' : 95, # magenta 95 | 'c' : 36, 'C' : 96, # cyan 96 | 'w' : 37, 'W' : 97 } # white 97 | 98 | # Regex to be used for color formatting 99 | color_re = r'@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)' 100 | 101 | 102 | class match_to_ansi(object): 103 | def __init__(self, color=True): 104 | self.color = color 105 | 106 | def escape(self, s): 107 | """Returns a TTY escape sequence for a color""" 108 | if self.color: 109 | return "\033[%sm" % s 110 | else: 111 | return '' 112 | 113 | def __call__(self, match): 114 | """Convert a match object generated by color_re into an ansi color code 115 | This can be used as a handler in re.sub. 116 | """ 117 | style, color, text = match.groups() 118 | m = match.group(0) 119 | 120 | if m == '@@': 121 | return '@' 122 | elif m == '@.': 123 | return self.escape(0) 124 | elif m == '@': 125 | raise ColorParseError("Incomplete color format: '%s' in %s" 126 | % (m, match.string)) 127 | 128 | string = styles[style] 129 | if color: 130 | if color not in colors: 131 | raise ColorParseError("invalid color specifier: '%s' in '%s'" 132 | % (color, match.string)) 133 | string += ';' + str(colors[color]) 134 | 135 | colored_text = '' 136 | if text: 137 | colored_text = text + self.escape(0) 138 | 139 | return self.escape(string) + colored_text 140 | 141 | 142 | def colorize(string, **kwargs): 143 | """Take a string and replace all color expressions with ANSI control 144 | codes. Return the resulting string. 145 | If color=False is supplied, output will be plain text without 146 | control codes, for output to non-console devices. 147 | """ 148 | color = kwargs.get('color', True) 149 | return re.sub(color_re, match_to_ansi(color), string) 150 | 151 | 152 | def cwrite(string, stream=sys.stdout, color=None): 153 | """Replace all color expressions in string with ANSI control 154 | codes and write the result to the stream. If color is 155 | False, this will write plain text with o color. If True, 156 | then it will always write colored output. If not supplied, 157 | then it will be set based on stream.isatty(). 158 | """ 159 | if color is None: 160 | color = stream.isatty() 161 | stream.write(colorize(string, color=color)) 162 | 163 | 164 | def cprint(string, stream=sys.stdout, color=None): 165 | """Same as cwrite, but writes a trailing newline to the stream.""" 166 | cwrite(string + "\n", stream, color) 167 | 168 | def cescape(string): 169 | """Replace all @ with @@ in the string provided.""" 170 | return str(string).replace('@', '@@') 171 | 172 | 173 | class ColorStream(object): 174 | def __init__(self, stream, color=None): 175 | self.__class__ = type(stream.__class__.__name__, 176 | (self.__class__, stream.__class__), {}) 177 | self.__dict__ = stream.__dict__ 178 | self.color = color 179 | self.stream = stream 180 | 181 | def write(self, string, **kwargs): 182 | if kwargs.get('raw', False): 183 | super(ColorStream, self).write(string) 184 | else: 185 | cwrite(string, self.stream, self.color) 186 | 187 | def writelines(self, sequence, **kwargs): 188 | raw = kwargs.get('raw', False) 189 | for string in sequence: 190 | self.write(string, self.color, raw=raw) 191 | -------------------------------------------------------------------------------- /cmake/Platform/BlueGeneQ-base.cmake: -------------------------------------------------------------------------------- 1 | 2 | #============================================================================= 3 | # Copyright 2010 Kitware, Inc. 4 | # Copyright 2010 Todd Gamblin 5 | # 6 | # Distributed under the OSI-approved BSD License (the "License"); 7 | # see accompanying file Copyright.txt for details. 8 | # 9 | # This software is distributed WITHOUT ANY WARRANTY; without even the 10 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 11 | # See the License for more information. 12 | #============================================================================= 13 | # (To distribute this file outside of CMake, substitute the full 14 | # License text for the above reference.) 15 | 16 | # 17 | # Blue Gene/Q base platform file. 18 | # 19 | # NOTE: Do not set your platform to "BlueGeneQ-base". This file is included 20 | # by the real platform files. Use one of these two platforms instead: 21 | # 22 | # BlueGeneQ-dynamic For dynamically linked executables 23 | # BlueGeneQ-static For statically linked executables 24 | # 25 | # The platform you choose doesn't affect whether or not you can build 26 | # shared or static libraries -- it ONLY changs whether exeuatbles are linked 27 | # statically or dynamically. 28 | # 29 | # This platform file tries its best to adhere to the behavior of the MPI 30 | # compiler wrappers included with the latest BG/P drivers. 31 | # 32 | 33 | # 34 | # This adds directories that find commands should specifically ignore for cross compiles. 35 | # Most of these directories are the includeand lib directories for the frontend on BG/P systems. 36 | # Not ignoring these can cause things like FindX11 to find a frontend PPC version mistakenly. 37 | # We use this on BG instead of re-rooting because backend libraries are typically strewn about 38 | # the filesystem, and we can't re-root ALL backend libraries to a single place. 39 | # 40 | set(CMAKE_SYSTEM_IGNORE_PATH 41 | /lib /lib64 /include 42 | /usr/lib /usr/lib64 /usr/include 43 | /usr/local/lib /usr/local/lib64 /usr/local/include 44 | /usr/X11/lib /usr/X11/lib64 /usr/X11/include 45 | /usr/lib/X11 /usr/lib64/X11 /usr/include/X11 46 | /usr/X11R6/lib /usr/X11R6/lib64 /usr/X11R6/include 47 | /usr/X11R7/lib /usr/X11R7/lib64 /usr/X11R7/include 48 | ) 49 | 50 | # 51 | # Indicate that this is a unix-like system 52 | # 53 | set(UNIX 1) 54 | 55 | # 56 | # Library prefixes, suffixes, extra libs. 57 | # 58 | set(CMAKE_LINK_LIBRARY_SUFFIX "") 59 | set(CMAKE_STATIC_LIBRARY_PREFIX "lib") # lib 60 | set(CMAKE_STATIC_LIBRARY_SUFFIX ".a") # .a 61 | 62 | set(CMAKE_SHARED_LIBRARY_PREFIX "lib") # lib 63 | set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") # .so 64 | set(CMAKE_EXECUTABLE_SUFFIX "") # .exe 65 | 66 | set(CMAKE_DL_LIBS "dl") 67 | 68 | # 69 | # BG/Q supports dynamic libraries regardless of whether we're building 70 | # static or dynamic *executables*. 71 | # 72 | set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE) 73 | set(CMAKE_FIND_LIBRARY_PREFIXES "lib") 74 | 75 | # 76 | # For BGQ builds, we're cross compiling, but we don't want to re-root things 77 | # (e.g. with CMAKE_FIND_ROOT_PATH) because users may have libraries anywhere on 78 | # the shared filesystems, and this may lie outside the root. Instead, we set the 79 | # system directories so that the various system BG CNK library locations are 80 | # searched first. This is not the clearest thing in the world, given IBM's driver 81 | # layout, but this should cover all the standard ones. 82 | # 83 | macro(__BlueGeneQ_common_setup compiler_id lang) 84 | # Need to use the version of the comm lib compiled with the right compiler. 85 | set(__BlueGeneQ_commlib_dir gcc) 86 | if (${compiler_id} STREQUAL XL) 87 | set(__BlueGeneQ_commlib_dir xl) 88 | endif() 89 | 90 | set(CMAKE_SYSTEM_LIBRARY_PATH 91 | /bgsys/drivers/ppcfloor/comm/default/lib # default comm layer (used by mpi compiler wrappers) 92 | /bgsys/drivers/ppcfloor/comm/${__BlueGeneQ_commlib_dir}/lib # PAMI, other lower-level comm libraries 93 | /bgsys/drivers/ppcfloor/gnu-linux/lib # CNK python installation directory 94 | /bgsys/drivers/ppcfloor/gnu-linux/powerpc64-bgq-linux/lib # CNK Linux image -- standard runtime libs, pthread, etc. 95 | ) 96 | 97 | # Add all the system include paths. 98 | set(CMAKE_SYSTEM_INCLUDE_PATH 99 | /bgsys/drivers/ppcfloor/comm/sys/include 100 | /bgsys/drivers/ppcfloor/ 101 | /bgsys/drivers/ppcfloor/spi/include 102 | /bgsys/drivers/ppcfloor/spi/include/kernel/cnk 103 | /bgsys/drivers/ppcfloor/comm/${__BlueGeneQ_commlib_dir}/include 104 | ) 105 | 106 | # Ensure that the system directories are included with the regular compilers, as users will expect this 107 | # to do the same thing as the MPI compilers, which add these flags. 108 | set(BGQ_SYSTEM_INCLUDES "") 109 | foreach(dir ${CMAKE_SYSTEM_INCLUDE_PATH}) 110 | set(BGQ_SYSTEM_INCLUDES "${BGQ_SYSTEM_INCLUDES} -I${dir}") 111 | endforeach() 112 | set(CMAKE_C_COMPILE_OBJECT " ${BGQ_SYSTEM_INCLUDES} -o -c ") 113 | set(CMAKE_CXX_COMPILE_OBJECT " ${BGQ_SYSTEM_INCLUDES} -o -c ") 114 | 115 | # 116 | # Code below does setup for shared libraries. That this is done 117 | # regardless of whether the platform is static or dynamic -- you can make 118 | # shared libraries even if you intend to make static executables, you just 119 | # can't make a dynamic executable if you use the static platform file. 120 | # 121 | if (${compiler_id} STREQUAL XL) 122 | # Flags for XL compilers if we explicitly detected XL 123 | set(CMAKE_SHARED_LIBRARY_${lang}_FLAGS "-qpic") 124 | set(CMAKE_SHARED_LIBRARY_CREATE_${lang}_FLAGS "-qmkshrobj -qnostaticlink") 125 | else() 126 | # Assume flags for GNU compilers (if the ID is GNU *or* anything else). 127 | set(CMAKE_SHARED_LIBRARY_${lang}_FLAGS "-fPIC") 128 | set(CMAKE_SHARED_LIBRARY_CREATE_${lang}_FLAGS "-shared") 129 | endif() 130 | 131 | # Both toolchains use the GNU linker on BG/P, so these options are shared. 132 | set(CMAKE_SHARED_LIBRARY_RUNTIME_${lang}_FLAG "-Wl,-rpath,") 133 | set(CMAKE_SHARED_LIBRARY_RPATH_LINK_${lang}_FLAG "-Wl,-rpath-link,") 134 | set(CMAKE_SHARED_LIBRARY_SONAME_${lang}_FLAG "-Wl,-soname,") 135 | set(CMAKE_EXE_EXPORTS_${lang}_FLAG "-Wl,--export-dynamic") 136 | set(CMAKE_SHARED_LIBRARY_LINK_${lang}_FLAGS "") # +s, flag for exe link to use shared lib 137 | set(CMAKE_SHARED_LIBRARY_RUNTIME_${lang}_FLAG_SEP ":") # : or empty 138 | 139 | endmacro() 140 | 141 | # 142 | # This macro needs to be called for dynamic library support. Unfortunately on BG, 143 | # We can't support both static and dynamic links in the same platform file. The 144 | # dynamic link platform file needs to call this explicitly to set up dynamic linking. 145 | # 146 | macro(__BlueGeneQ_setup_dynamic compiler_id lang) 147 | __BlueGeneQ_common_setup(${compiler_id} ${lang}) 148 | 149 | if (${compiler_id} STREQUAL XL) 150 | set(BGQ_${lang}_DYNAMIC_EXE_FLAGS "-qnostaticlink -qnostaticlink=libgcc") 151 | else() 152 | set(BGQ_${lang}_DYNAMIC_EXE_FLAGS "-dynamic") 153 | endif() 154 | 155 | # For dynamic executables, need to provide special BG/Q arguments. 156 | set(BGQ_${lang}_DEFAULT_EXE_FLAGS 157 | " -o ") 158 | set(CMAKE_${lang}_LINK_EXECUTABLE 159 | " -Wl,-relax ${BGQ_${lang}_DYNAMIC_EXE_FLAGS} ${BGQ_${lang}_DEFAULT_EXE_FLAGS}") 160 | endmacro() 161 | 162 | # 163 | # This macro needs to be called for static builds. Right now it just adds -Wl,-relax 164 | # to the link line. 165 | # 166 | macro(__BlueGeneQ_setup_static compiler_id lang) 167 | __BlueGeneQ_common_setup(${compiler_id} ${lang}) 168 | 169 | # For static executables, use default link settings. 170 | set(BGQ_${lang}_DEFAULT_EXE_FLAGS 171 | " -o ") 172 | set(CMAKE_${lang}_LINK_EXECUTABLE 173 | " -Wl,-relax ${BGQ_${lang}_DEFAULT_EXE_FLAGS}") 174 | endmacro() 175 | -------------------------------------------------------------------------------- /src/c/libcram/cram.w: -------------------------------------------------------------------------------- 1 | // -*- c -*- 2 | // 3 | // This file defines MPI wrappers for cram. 4 | // 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "cram_file.h" 16 | 17 | // Local world communicator for each job run concurrently. 18 | static MPI_Comm local_world; 19 | 20 | // This function modifies its parameter by swapping it with local world. 21 | // if it is MPI_COMM_WORLD. 22 | #define swap_world(world) \ 23 | do { \ 24 | if (world == MPI_COMM_WORLD) { \ 25 | world = local_world; \ 26 | } \ 27 | } while (0) 28 | 29 | // 30 | // Acceptable output modes for Cram. 31 | // 32 | typedef enum { 33 | cram_output_system, // Stick with system's original stdout/stderr settings. 34 | cram_output_none, // All processes freopen stdout and stderr to /dev/null 35 | cram_output_rank0, // Rank 0 in each job opens its own stdout/stderr; others /dev/null 36 | cram_output_all, // All ranks in all jobs open their own stdout/stderr 37 | } cram_output_mode_t; 38 | 39 | 40 | // Global for Cram output mode, set in MPI_Init. 41 | static cram_output_mode_t cram_output_mode = cram_output_rank0; 42 | 43 | // Original stderr pointer. So that we can print last-ditch error messages. 44 | static FILE *original_stderr = NULL; 45 | 46 | // Some information about this job. 47 | static int job_id = -1; 48 | static int local_rank = -1; 49 | 50 | // 51 | // Gets the output mode from the CRAM_OUTPUT environment variable. 52 | // Possible values are: 53 | // 54 | // NONE -> cram_output_none 55 | // RANK0 -> cram_output_rank0 56 | // ALL -> cram_output_all 57 | // 58 | // These map to corresponding cram_output_mode_t values. 59 | // 60 | static cram_output_mode_t get_output_mode() { 61 | const char *mode = getenv("CRAM_OUTPUT"); 62 | 63 | if (!mode || strcasecmp(mode, "rank0") == 0) { 64 | return cram_output_rank0; 65 | 66 | } else if (strcasecmp(mode, "system") == 0) { 67 | return cram_output_system; 68 | 69 | } else if (strcasecmp(mode, "none") == 0) { 70 | return cram_output_none; 71 | 72 | } else if (strcasecmp(mode, "all") == 0) { 73 | return cram_output_all; 74 | } 75 | return cram_output_rank0; 76 | } 77 | 78 | 79 | // 80 | // Redirect I/O to the supplied output and error files, saving stderr in 81 | // original_stderr (if possible on the particular platform). 82 | // 83 | static void redirect_io(const char *out, const char *err) { 84 | freopen(out, "w", stdout); 85 | 86 | // If each process has its own output stream, then write errors to the 87 | // per-process error stream, not to the original error stream. 88 | if (cram_output_mode == cram_output_all) { 89 | freopen(err, "w", stderr); 90 | original_stderr = stderr; 91 | return; 92 | } 93 | 94 | // in GLIBC, stdout and stderr are assignable. Prefer assigning to stdout 95 | // and stderr if possible, becasue BG/Q does not allow dup, dup2, fcntl, etc. 96 | // to work on the built-in stderr. There is therefore not a portable way 97 | // to do this if we care about BG/Q. 98 | #ifdef __GLIBC__ 99 | original_stderr = stderr; 100 | stderr = fopen(err, "w"); 101 | #else // not __GLIBC__ 102 | // dup the fd for stderr, underneath libc. This doesn't work on BG/Q. 103 | int fd = dup(fileno(stderr)); 104 | freopen(err, "w", stderr); 105 | original_stderr = fdopen(fd, "a"); 106 | 107 | // if the fdopen fails for some reason, just use the new error stream for this. 108 | if (!original_stderr) { 109 | original_stderr = stderr; 110 | if (cram_output_mode == cram_output_rank0) { 111 | fprintf(stderr, "WARNING: Cram couldn't preserve the original stderr stream while opening a new one.\n"); 112 | fprintf(stderr, "WARNING: You may not receive notifcations of errors if they do not happen on rank 0 in each job."); 113 | } 114 | } 115 | #endif // not __GLIBC__ 116 | } 117 | 118 | 119 | // 120 | // Handler for SEGV prints to original stderr to tell the user which process 121 | // died, then exits cleanly. 122 | // 123 | void segv_sigaction(int signal, siginfo_t *si, void *ctx) { 124 | fprintf(original_stderr, "Rank %d on cram job %d died with signal %d.\n", 125 | local_rank, job_id, signal); 126 | 127 | // Act like everything is ok. Nothing to see here... 128 | int finalized; 129 | PMPI_Finalized(&finalized); 130 | if (!finalized) { 131 | PMPI_Finalize(); 132 | } 133 | exit(0); 134 | } 135 | 136 | // 137 | // Atexit handler that disallows processes exiting with codes other than 0. 138 | // On some systems (BG/Q), this results in the entire MPI job being killed, 139 | // and we'd rather most of our cram jobs live full and productive lives. 140 | // 141 | void on_exit_handler(int err, void *arg) { 142 | if (err != 0) { 143 | fprintf(original_stderr, "Rank %d on cram job %d exited with error %d.\n", 144 | local_rank, job_id, err); 145 | 146 | // Act like everything is ok. Nothing to see here... 147 | int finalized; 148 | PMPI_Finalized(&finalized); 149 | if (!finalized) { 150 | PMPI_Finalize(); 151 | } 152 | exit(0); 153 | } 154 | } 155 | 156 | // 157 | // In a cram run, there are many simulatneous jobs, some of which may fail. 158 | // This function sets up signal handlers and other handlers that attempt to 159 | // keep the whole job from dying when a single process dies. 160 | // 161 | static void setup_crash_handlers() { 162 | // Set up signal handlers so that SEGV is called. 163 | struct sigaction sa; 164 | sa.sa_sigaction = segv_sigaction; 165 | sigemptyset(&sa.sa_mask); 166 | sa.sa_flags = 0; 167 | int err = sigaction(SIGSEGV, &sa, NULL); 168 | 169 | // Register exit handler to mask procs that return with error. 170 | on_exit(on_exit_handler, NULL); 171 | } 172 | 173 | 174 | // 175 | // MPI_Init does all the communicator setup 176 | // 177 | {{fn func MPI_Init}}{ 178 | // First call PMPI_Init() 179 | {{callfn}} 180 | 181 | // Get this process's rank. 182 | int rank; 183 | PMPI_Comm_rank(MPI_COMM_WORLD, &rank); 184 | 185 | // Tell the user they're running with cram. 186 | if (rank == 0) { 187 | fprintf(stderr, "===========================================================\n"); 188 | fprintf(stderr, " This job is running with Cram.\n"); 189 | fprintf(stderr, "\n"); 190 | } 191 | 192 | // Look for the CRAM_FILE environment variable to find where our input lives. 193 | const char *cram_filename = getenv("CRAM_FILE"); 194 | if (!cram_filename) { 195 | if (rank == 0) { 196 | fprintf(stderr, " CRAM_FILE environment variable is not set.\n"); 197 | fprintf(stderr, " Disabling Cram and running normally instead.\n"); 198 | fprintf(stderr, "===========================================================\n"); 199 | } 200 | local_world = MPI_COMM_WORLD; 201 | return MPI_SUCCESS; 202 | } 203 | 204 | // Read the whole file in on rank 1 (it's compressed, so this should scale fairly well, 205 | // e.g. out to ~1M jobs assuming 1GB RAM per process) 206 | cram_file_t cram_file; 207 | 208 | if (rank == 0) { 209 | if (!cram_file_open(cram_filename, &cram_file)) { 210 | fprintf(stderr, "Error: Failed to open cram file '%s'.\n", cram_filename); 211 | fprintf(stderr, "%s\n", strerror(errno)); 212 | PMPI_Abort(MPI_COMM_WORLD, errno); 213 | } 214 | 215 | fprintf(stderr, " Splitting this MPI job into %d jobs.\n", cram_file.num_jobs); 216 | fprintf(stderr, " This will use %d total processes.\n", cram_file.total_procs); 217 | } 218 | 219 | // Receive our job from the root process. 220 | cram_job_t cram_job; 221 | double start_time = PMPI_Wtime(); 222 | cram_file_bcast_jobs(&cram_file, 0, &cram_job, &job_id, MPI_COMM_WORLD); 223 | double bcast_time = PMPI_Wtime(); 224 | 225 | // Use the job id to split MPI_COMM_WORLD. 226 | PMPI_Comm_split(MPI_COMM_WORLD, job_id, rank, &local_world); 227 | double split_time = PMPI_Wtime(); 228 | 229 | // Throw away unneeded ranks. 230 | if (job_id == -1) { 231 | PMPI_Barrier(MPI_COMM_WORLD); // matches barrier later. 232 | PMPI_Finalize(); 233 | exit(0); 234 | } 235 | 236 | // set up this job's environment based on the job descriptor. 237 | cram_job_setup(&cram_job, {{0}}, (const char***){{1}}); 238 | PMPI_Comm_rank(local_world, &local_rank); 239 | double setup_time = PMPI_Wtime(); 240 | 241 | cram_output_mode = get_output_mode(); 242 | char out_file_name[1024]; 243 | char err_file_name[1024]; 244 | 245 | if (cram_output_mode != cram_output_system) { 246 | sprintf(out_file_name, "/dev/null"); 247 | sprintf(err_file_name, "/dev/null"); 248 | 249 | if (cram_output_mode == cram_output_rank0) { 250 | // Redirect I/O to a separate file for each cram job. 251 | // These files will be in the job's working directory. 252 | if (local_rank == 0) { 253 | sprintf(out_file_name, "cram.%d.out", job_id); 254 | sprintf(err_file_name, "cram.%d.err", job_id); 255 | } 256 | 257 | } else if (cram_output_mode == cram_output_all) { 258 | sprintf(out_file_name, "cram.%d.%d.out", job_id, local_rank); 259 | sprintf(err_file_name, "cram.%d.%d.err", job_id, local_rank); 260 | } 261 | 262 | // don't freopen on root until after printing status. 263 | if (rank != 0) { 264 | redirect_io(out_file_name, err_file_name); 265 | } 266 | } 267 | 268 | // wait for lots of files to open. 269 | PMPI_Barrier(MPI_COMM_WORLD); 270 | double freopen_time = PMPI_Wtime(); 271 | 272 | if (rank == 0) { 273 | fprintf(stderr, "\n"); 274 | fprintf(stderr, " Successfully set up job:\n"); 275 | fprintf(stderr, " Job broadcast: %.6f sec\n", bcast_time - start_time); 276 | fprintf(stderr, " MPI_Comm_split: %.6f sec\n", split_time - bcast_time); 277 | fprintf(stderr, " Job setup: %.6f sec\n", setup_time - split_time); 278 | fprintf(stderr, " File open: %.6f sec\n", freopen_time - setup_time); 279 | fprintf(stderr, " --------------------------------------\n"); 280 | fprintf(stderr, " Total: %.6f sec\n", freopen_time - start_time); 281 | fprintf(stderr, " \n"); 282 | fprintf(stderr, "===========================================================\n"); 283 | 284 | if (cram_output_mode != cram_output_system) { 285 | // reopen *last* on the zero rank. 286 | redirect_io(out_file_name, err_file_name); 287 | } 288 | 289 | cram_file_close(&cram_file); 290 | } 291 | 292 | // Now that I/O is set up, register some handlers for crashes. 293 | setup_crash_handlers(); 294 | 295 | cram_job_free(&cram_job); 296 | }{{endfn}} 297 | 298 | // This generates interceptors that will catch every MPI routine 299 | // *except* MPI_Init. The interceptors just make sure that if 300 | // they are called with an argument of type MPI_Comm that has a 301 | // value of MPI_COMM_WORLD, they switch it to local_world. 302 | {{fnall func MPI_Init}}{ 303 | {{apply_to_type MPI_Comm swap_world}} 304 | {{callfn}} 305 | }{{endfnall}} 306 | -------------------------------------------------------------------------------- /wrap/README.md: -------------------------------------------------------------------------------- 1 | wrap.py 2 | =========================== 3 | a [PMPI](http://www.open-mpi.org/faq/?category=perftools#PMPI) wrapper generator 4 | 5 | by Todd Gamblin, tgamblin@llnl.gov, https://github.com/tgamblin/wrap 6 | 7 | Usage: wrap.py [-fgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...] 8 | Python script for creating PMPI wrappers. Roughly follows the syntax of 9 | the Argonne PMPI wrapper generator, with some enhancements. 10 | Options:" 11 | -d Just dump function declarations parsed out of mpi.h 12 | -f Generate fortran wrappers in addition to C wrappers. 13 | -g Generate reentry guards around wrapper functions. 14 | -c exe Provide name of MPI compiler (for parsing mpi.h). 15 | Default is \'mpicc\'. 16 | -s Skip writing #includes, #defines, and other 17 | front-matter (for non-C output). 18 | -i pmpi_init Specify proper binding for the fortran pmpi_init 19 | function. Default is \'pmpi_init_\'. Wrappers 20 | compiled for PIC will guess the right binding 21 | automatically (use -DPIC when you compile dynamic 22 | libs). 23 | -o file Send output to a file instead of stdout. 24 | 25 | 26 | Thanks to these people for their suggestions and contributions: 27 | 28 | * David Lecomber, Allinea 29 | * Barry Rountree, LLNL 30 | 31 | Known Bugs: 32 | 33 | * Certain fortran bindings need some bugfixes and may not work. 34 | 35 | Tutorial 36 | ----------------------------- 37 | For a thorough tutorial, look at `examples/tutorial.w`! It walks you through 38 | the process of using `wrap.py`. It is also legal `wrap.py` code, so you 39 | can run `wrap.py` on it and see the output to better understand what's 40 | going on. 41 | 42 | 43 | CMake Integration 44 | ----------------------------- 45 | `wrap.py` includes a `WrapConfig.cmake` file. You can use this in your CMake project to automatically generate rules to generate wrap.py code. 46 | 47 | Here's an example. Suppose you put `wrap.py` in a subdirectory of your project called wrap, and your project looks like this: 48 | 49 | project/ 50 | CMakeLists.txt 51 | wrap/ 52 | wrap.py 53 | WrapConfig.cmake 54 | In your top-level CMakeLists.txt file, you can now do this: 55 | 56 | # wrap.py setup -- grab the add_wrapped_file macro. 57 | set(WRAP ${PROJECT_SOURCE_DIR}/wrap/wrap.py) 58 | include(wrap/WrapConfig.cmake) 59 | 60 | If you have a wrapped source file, you can use the wrapper auto-generation like this: 61 | 62 | add_wrapped_file(wrappers.C wrappers.w) 63 | add_library(tool_library wrappers.C) 64 | 65 | The `add_wrapped_file` function takes care of the dependences and code generation for you. If you need fortran support, call it like this: 66 | 67 | add_wrapped_file(wrappers.C wrappers.w -f) 68 | 69 | And note that if you generate a header that your .C files depend on, you need to explicitly include it in a target's sources, unlike non-generated headers. e.g.: 70 | 71 | add_wrapped_file(my-header.h my-header.w) 72 | add_library(tool_library 73 | tool.C # say that this includes my-header.h 74 | my-header.h) # you need to add my-header.h here. 75 | 76 | If you don't do this, then the header dependence won't be accounted for when tool.C is built. 77 | 78 | Wrapper file syntax 79 | ----------------------------- 80 | Wrap syntax is a superset of the syntax defined in Appendix C of 81 | the MPE manual [1], but many commands from the original wrapper 82 | generator are now deprecated. 83 | 84 | 85 | The following two macros generate skeleton wrappers and allow 86 | delegation via `{{callfn}}`: 87 | 88 | * `fn` iterates over only the listed 89 | functions. 90 | * `fnall` iterates over all functions *minus* the named functions. 91 | 92 | {{fnall ... }} 93 | // code here 94 | {{endfnall}} 95 | 96 | {{fn ... }} 97 | {{endfn} 98 | 99 | {{callfn}} 100 | 101 | `callfn` expands to the call of the function being profiled. 102 | 103 | `fnall` defines a wrapper to be used on all functions except the functions named. fn is identical to fnall except that it only generates wrappers for functions named explicitly. 104 | 105 | {{fn FOO MPI_Abort}} 106 | // Do-nothing wrapper for {{FOO}} 107 | {{endfn}} 108 | 109 | generates (in part): 110 | 111 | /* ================== C Wrappers for MPI_Abort ================== */ 112 | _EXTERN_C_ int PMPI_Abort(MPI_Comm arg_0, int arg_1); 113 | _EXTERN_C_ int MPI_Abort(MPI_Comm arg_0, int arg_1) { 114 | int return_val = 0; 115 | 116 | // Do-nothing wrapper for MPI_Abort 117 | return return_val; 118 | } 119 | 120 | `foreachfn` and `forallfn` are the counterparts of `fn` and `fnall`, but they don't generate the 121 | skeletons (and therefore you can't delegate with `{{callfn}}`). However, you 122 | can use things like `fn_name` (or `foo`) and `argTypeList`, `retType`, `argList`, etc. 123 | 124 | They're not designed for making wrappers, but declarations of lots of variables and other things you need to declare per MPI function. e.g., say you wanted a static variable per MPI call for some flag. 125 | 126 | {{forallfn ... }} 127 | // code here 128 | {{endforallfn} 129 | 130 | {foreachfn ... }} 131 | // code here 132 | {{endforeachfn}} 133 | 134 | 135 | The code between {{forallfn}} and {{endforallfn}} is copied once 136 | for every function profiled, except for the functions listed. 137 | For example: 138 | 139 | {{forallfn fn_name}} 140 | static int {{fn_name}}_ncalls_{{fileno}}; 141 | {{endforallfn}} 142 | 143 | might expand to: 144 | 145 | static int MPI_Send_ncalls_1; 146 | static int MPI_Recv_ncalls_1; 147 | ... 148 | 149 | etc. 150 | 151 | * `{{get_arg }}` OR `{{}}` 152 | Arguments to the function being profiled may be referenced by 153 | number, starting with 0 and increasing. e.g., in a wrapper file: 154 | 155 | void process_argc_and_argv(int *argc, char ***argv) { 156 | // do stuff to argc and argv. 157 | } 158 | 159 | {{fn fn_name MPI_Init}} 160 | process_argc_and_argv({{0}}, {{1}}); 161 | {{callfn}} 162 | {{endfn}} 163 | Note that `{{0}}` is just a synonym for `{{get_arg 0}}` 164 | 165 | * `{{ret_val}}` 166 | ReturnVal expands to the variable that is used to hold the return 167 | value of the function being profiled. (was: `{{returnVal}}`) 168 | 169 | * `{{fn_num}}` 170 | This is a number, starting from zero. It is incremented every time 171 | it is used. 172 | 173 | * `{{ret_type}}` 174 | The return type of the function. (was: `{{retType}}`) 175 | 176 | * `{{formals}}` 177 | Essentially what would be in a formal declaration for the function. 178 | Can be used this with forallfn and foreachfn; these don't generate 179 | prototypes, they just iterate over the functions without making a 180 | skeleton. (was: `{{argTypeList}}`) 181 | 182 | * `{{args}}` 183 | Names of the arguments in a comma-separated list, e.g.: 184 | `buf, type, count, comm` 185 | 186 | * `{{argList}}` 187 | Same as `{{args}}`, but with parentheses around the list, e.g.: 188 | `(buf, type, count, comm)` 189 | 190 | * `{{applyToType }}` 191 | This macro must be nested inside either a fn or fnall block. 192 | Within the functions being wrapped by fn or fnall, this macro will 193 | apply `` to any arguments of the function with type 194 | ``. For example, you might write a wrapper file like this: 195 | 196 | #define my_macro(comm) do_something_to(comm); 197 | {{fn fn_name MPI_Send MPI_Isend MPI_Ibsend}} 198 | {{applyToType MPI_Comm my_macro}} 199 | {{callfn}} 200 | {{endfn}} 201 | 202 | Now the generated wrappers to `MPI_Send`, `MPI_Isend`, and `MPI_Ibsend` will do something like this: 203 | 204 | int MPI_Isend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { 205 | int _wrap_py_return_val = 0; 206 | my_macro(comm); 207 | PMPI_Isend(buf, count, datatype, dest, tag, comm, request); 208 | } 209 | 210 | * `{{sub }}` 211 | Declares `` in the current scope and gives it the value 212 | of `` with all instances of `` replaced with 213 | ``. You may use any valid python regexp for `` 214 | and any valid substitution value for ``. The regexps 215 | follow the same syntax as Python's re.sub(), and they may be single 216 | or double quoted (though it's not necessary unless you use spaces in 217 | the expressions). 218 | 219 | Example: 220 | 221 | {{forallfn foo}} 222 | {{sub nqjfoo foo '^MPI_' NQJ_}} 223 | {{nqjfoo}} 224 | {{endforallfn}} 225 | 226 | This will print `NQJ_xxx` instead of `MPI_xxx` for each MPI function. 227 | 228 | * `{{fileno}}` 229 | An integral index representing which wrapper file the macro 230 | came from. This is useful when decalring file-global variables 231 | to prevent name collisions. Identifiers declared outside 232 | functions should end with _{{fileno}}. For example: 233 | 234 | static double overhead_time_{{fileno}}; 235 | 236 | might expand to 237 | 238 | static double overhead_time_0; 239 | 240 | 241 | * `{{vardecl ...}}` *(not yet supported)* 242 | Declare variables within a wrapper definition. Wrap will decorate 243 | the variable name to prevent collisions. 244 | 245 | * `{{}}` *(not yet supported)* 246 | Access a variable declared by `{{vardecl}}`. 247 | 248 | Notes on the fortran wrappers 249 | ------------------------------- 250 | #if (!defined(MPICH_HAS_C2F) && defined(MPICH_NAME) && (MPICH_NAME == 1)) 251 | /* MPICH call */ 252 | return_val = MPI_Abort((MPI_Comm)(*arg_0), *arg_1); 253 | #else 254 | /* MPI-2 safe call */ 255 | return_val = MPI_Abort(MPI_Comm_f2c(*arg_0), *arg_1); 256 | #endif 257 | 258 | This is the part of the wrapper that delegates from Fortran 259 | to C. There are two ways to do that. The MPI-2 way is to 260 | call the appropriate _f2c call on the handle and pass that 261 | to the C function. The f2c/c2f calls are also available in 262 | some versions of MPICH1, but not all of them (I believe they 263 | were backported), so you can do the MPI-2 thing if 264 | `MPICH_HAS_C2F` is defined. 265 | 266 | If c2f functions are not around, then the script tries to 267 | figure out if it's dealing with MPICH1, where all the 268 | handles are ints. In that case, you can just pass the int 269 | through. 270 | 271 | Right now, if it's not *specifically* MPICH1, wrap.py does 272 | the MPI-2 thing. From what Barry was telling me, your MPI 273 | environment might have int handles, but it is not MPICH1. 274 | So you could either define all the `MPI_Foo_c2f`/`MPI_Foo_f2c` 275 | calls to identity macros, e.g.: 276 | 277 | #define MPI_File_c2f(x) (x) 278 | #define MPI_File_f2c(x) (x) 279 | 280 | or you could add something to wrap.py to force the 281 | int-passing behavior. I'm not sure if you have to care 282 | about this, but I thought I'd point it out. 283 | 284 | -s, or 'structural' mode 285 | ------------------------------- 286 | 287 | If you use the `-s` option, this skips the includes and defines used for C 288 | wrapper functions. This is useful if you want to use wrap to generate 289 | non-C files, such as XML. 290 | 291 | If you use -s, we recommend that you avoid using `{{fn}}` and `{{fnall}}`, 292 | as these generate proper wrapper functions that rely on some of the 293 | header information. Instead, use `{{foreachfn}}` and `{{forallfn}}`, as 294 | these do not generate wrappers around each iteration of the macro. 295 | 296 | e.g. if you want to generate a simple XML file with descriptions of the 297 | MPI arguments, you might write this in a wrapper file: 298 | 299 | {{forallfn fun}} 300 | 301 | {{endforallfn}} 302 | 303 | We don't disallow `{{fnall}}` or `{{fn}}` with `-s`, but If you used 304 | `{{fnall}}` here, each XML tag would have a C wrapper function around it, 305 | which is probably NOT what you want. 306 | 307 | 308 | 1. Anthony Chan, William Gropp and Weing Lusk. *User's Guide for MPE: 309 | Extensions for MPI Programs*. ANL/MCS-TM-ANL-98/xx. 310 | ftp://ftp.mcs.anl.gov/pub/mpi/mpeman.pdf 311 | 312 | 313 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![image](images/cram-logo.png) Cram 2 | ======================================= 3 | 4 | Cram runs many small MPI jobs inside one large MPI job. 5 | 6 | Released under the GNU LGPL, `LLNL-CODE-661100`. See the `LICENSE` 7 | file for details. 8 | 9 | Overview 10 | ------------------------- 11 | 12 | Suppose you have a supercomputer with 1 million cores, and you want to 13 | run 1 million 1-process jobs. If your resource manager is scalable, 14 | that is most likely an easy thing to do. You might just submit 1 15 | million jobs: 16 | 17 | srun -n 1 my_mpi_application input.1.txt 18 | srun -n 1 my_mpi_application input.2.txt 19 | ... 20 | srun -n 1 my_mpi_application input.1048576.txt 21 | 22 | Unfortunately, for some resource managers, that requires you to have 1 23 | million `srun` processes running on the front-end node of your fancy 24 | supercomputer, which quickly blows out either memory or the front-end 25 | OS's process limit. 26 | 27 | Cram fixes this problem by allowing you to lump all of those jobs into 28 | one large submission. You use Cram like this: 29 | 30 | cram pack -f cram.job -n 1 -e my_mpi_application input.1.txt 31 | cram pack -f cram.job -n 1 -e my_mpi_application input.2.txt 32 | ... 33 | cram pack -f cram.job -n 1 -e my_mpi_application input.1048576.txt 34 | 35 | This packs all those job invocations into a file called `cram.job`, 36 | and saves them for one big submission. 37 | 38 | To use the file, either: 39 | * Link your application with `libcram.a` if you are using C/C++. 40 | * Link your application with `libfcram.a` if you are using Fortran. 41 | 42 | Then, run your job like this: 43 | 44 | env CRAM_FILE=/path/to/cram.job srun -n 1048576 my_mpi_application 45 | 46 | That's it. The job launches, it splits itself into a million pieces. 47 | You'll see some output like this from your srun invocation if Cram 48 | is running properly: 49 | 50 | =========================================================== 51 | This job is running with Cram. 52 | 53 | Splitting this MPI job into 1048576 jobs. 54 | This will use total processes. 55 | 56 | Successfully set up job: 57 | Job broadcast: 0.005161 sec 58 | MPI_Comm_split: 0.001973 sec 59 | Job broadcast: 0.078768 sec 60 | File open: 0.700172 sec 61 | -------------------------------------- 62 | Total: 0.786074 sec 63 | 64 | =========================================================== 65 | 66 | Each runs independently of the others, generates its own output 67 | and error files, then terminates as it normally would. 68 | All you need to make sure of is that the final, large submitted job 69 | has at least as many processes as all the small jobs combined. 70 | 71 | When the job completes, you should see output files like this: 72 | 73 | cram.0.out cram.1.err cram.3.out cram.4.err 74 | cram.0.err cram.2.out cram.3.err cram.5.out ... etc ... 75 | cram.1.out cram.2.err cram.4.out cram.5.err 76 | 77 | If the jobs in your cram file had different working directories, then 78 | these files will appear in the working directories. 79 | 80 | **NOTE:** By default, only rank 0 in each job in your cram file will write 81 | to the output and error files. If you don't like this, Cram has other 82 | options for output handling. See **Output Options** below for more on this. 83 | 84 | 85 | 86 | Setup 87 | ------------------------- 88 | 89 | This section covers how to use Cram at LLNL. If you're at another 90 | site where Cram isn't installed yet, see **Build & Install** 91 | below to build your own. 92 | 93 | To run Cram, you need Python 2.7. Python and cram are available 94 | at LLNL through dotkits: 95 | 96 | use python-2.7.3 97 | use cram 98 | 99 | Once you use cram, the `CRAM_HOME` environment variable is set for 100 | you. You can use this to locate `libcram.a`: 101 | 102 | $ ls $CRAM_HOME/lib 103 | libcram.a python2.7/ 104 | 105 | That's the library you want to link your application with before you 106 | submit your large job. 107 | 108 | Command Reference 109 | ------------------------- 110 | 111 | Nearly everything in Cram is done through the `cram` command. Like 112 | `svn`, `git`, and various other UNIX commands, `cram` is made up of 113 | many subcommands. They're documented here. 114 | 115 | ### cram pack 116 | 117 | The most basic cram command -- this packs command line invocations 118 | into a file for batch submission. 119 | 120 | usage: cram pack [-h] -n NPROCS -f FILE [-e EXE] .. 121 | 122 | * `-n NPROCS` 123 | Number of processes this job should run with. 124 | 125 | * `-f FILE` 126 | File to store command invocation in. Default is 'cram.job' 127 | 128 | * `-e EXE` 129 | 130 | Optionally specify the executable name to use. By default, when you 131 | launch a cram job, The name of the launched binary is used 132 | (`argv[0]`). If you need to override this, you can supply -e with 133 | an argument. 134 | 135 | * `...` 136 | Command line arguments of the job to run, **not including the 137 | executable**. 138 | 139 | ### cram info 140 | 141 | This command is useful if you are trying to debug your run and you 142 | need to see what you actually submitted. 143 | 144 | Usage: cram info [-h] [-a] [-j JOB] [-n NUM_LINES] cramfile 145 | 146 | There are three modes: 147 | 148 | #### 1. Summary mode 149 | 150 | `cram info ` will show you a summary of the cram file, e.g.: 151 | 152 | $ cram info test-cram.job 153 | Name: test-cram.job 154 | Number of Jobs: 3 155 | Total Procs: 82 156 | Cram version: 1 157 | 158 | Job command lines: 159 | 0 35 procs my_app foo bar 2 2 4 160 | 1 35 procs my_app foo bar 2 4 2 161 | 2 12 procs my_app foo bar 4 2 2 162 | 163 | #### 2. Job detail 164 | 165 | `cram info -j 5 ` will show you information about the job 166 | with index 5 inside the cram file. That includes: 167 | 168 | 1. Size (number of processes) of the job 169 | 2. Its working directory 170 | 3. Its command line 171 | 4. All of its environment variables. 172 | 173 | Example: 174 | 175 | cram info -j 2 test-cram.job 176 | Job 2: 177 | Num procs: 12 178 | Working dir: /g/g21/gamblin2/test 179 | Arguments: 180 | my_app foo bar 4 2 2 181 | Environment: 182 | 'LOG_DIRECTORY' : '/p/lscratcha/my_app/output' 183 | ... etc ... 184 | 185 | #### 3. All data 186 | 187 | `cram info -a ` will print out all information for all 188 | jobs in the file. This can be very verbose, so use it carefully. 189 | 190 | ### cram test 191 | 192 | Usage: cram test [-h] [-l] [-v] [names [names ...]] 193 | 194 | Running `cram test` will run Cram's Python test suite. If you think 195 | something is wrong wtih Cram, use this command to see if it catches 196 | anything. 197 | 198 | 199 | Packing lots of jobs 200 | ------------------------- 201 | 202 | If you want to pack millions of jobs, you most likely do NOT want to 203 | call ``cram`` millions of times. Python starts up very slowly, and 204 | you don't want to pay that cost for each pack invocation. You can get 205 | around this by writing a simple Python script. Alongside the ``cram`` 206 | script, there is a ``cram-python`` script. You can use this to make 207 | an executable script like this one: 208 | 209 | #!/usr/bin/env cram-python 210 | 211 | import os 212 | from cram import * 213 | 214 | cf = CramFile('cram.job', 'w') 215 | env = os.environ 216 | 217 | # Pack cram invocations. 218 | # Usage: 219 | # cf.pack(, , , ) 220 | env["MY_SPECIAL_VAR"] = "my_value" 221 | cf.pack(1, '/home/youruser/ensemble/run-0', ['input.1.txt'], env) 222 | cf.pack(1, '/home/youruser/ensemble/run-1', ['input.2.txt'], env) 223 | # ... 224 | env["MY_SPECIAL_VAR"] = "another_value" 225 | cf.pack(1, '/home/youruser/ensemble/run-1048576', ['input.1048576.txt'], env) 226 | 227 | cf.close() 228 | 229 | This script will create a cram file, just like all those invocations 230 | of ``cram pack`` above, but it will run much faster because it runs in 231 | a single python session. Note that `CramFile.pack()` takes similar 232 | arguments as the `cram pack` command. 233 | 234 | Here's a more realistic one, for creating a million jobs with 235 | different user-specific scratch directories: 236 | 237 | #!/usr/bin/env cram-python 238 | 239 | import os 240 | import getpass 241 | from cram import * 242 | 243 | env = os.environ 244 | user = getpass.getuser() 245 | 246 | cf = CramFile('cram.job', 'w') 247 | for i in xrange(1048576): 248 | env["SCRATCH_DIR"] = "/p/lscratcha/%s/scratch-%08d" % (user, i) 249 | args = ["input.%08d" % i] 250 | cf.pack(1, '/home/%s/ensemble/run-%08d' % (user, i), args, env) 251 | cf.close() 252 | 253 | 254 | Output Options 255 | ------------------------- 256 | By default, Cram will redirect the `stdout` and `stderr` streams to 257 | `cram..out` and `cram..err` ONLY for rank 0 of each 258 | cram job. Other processes have their output and error redirected to 259 | `/dev/null`. You can control this behavior using the `CRAM_OUTPUT` 260 | environment variable. The options are: 261 | 262 | * `NONE`: No output. All processes redirecto to `/dev/null`. 263 | * `RANK0`: Default behavior. Rank 0 writes to `cram..out` 264 | and `cram..err`. All other processes write to `/dev/null`. 265 | * `ALL`: All processes write to unique files. e.g., rank 4 in job 1 writes to 266 | `cram.1.4.out` and `cram.4.1.err`. 267 | * `SYSTEM`: Ouptut is not redirected and system defaults are used. 268 | 269 | To set a particular output mode, set the `CRAM_OUTPUT` environment 270 | variable to one of the above values when you run. For example, to 271 | make every process write an output and error file, you would run like 272 | this: 273 | 274 | env CRAM_OUTPUT=ALL CRAM_FILE=/path/to/cram.job srun -n 1048576 my_mpi_application 275 | 276 | 277 | Error reporting 278 | ------------------------- 279 | You may notice that in the `NONE` and `RANK0` modes, some processes 280 | don't have a place to report errors. Cram tries its best to report 281 | errors regardless of the output mode the user has chosen. 282 | 283 | In particular, it tries to report non-zero `exit()` calls and 284 | segmentation faults (`SIGSEGV`) even if not every process has an 285 | output file. 286 | 287 | To do this, Cram will write error messages to the original error 288 | stream in all modes except `ALL`. So, if a process dies or exits, 289 | you'll see a message like this print out to the console: 290 | 291 | Rank 1 on cram job 4 died with signal 11. 292 | 293 | If you are running in `ALL` mode, Cram will print error messages like 294 | this out to the per-process `cram...err` file. 295 | 296 | 297 | Build & Install 298 | ------------------------- 299 | 300 | To build your own version of Cram, you need [CMake](http://cmake.org). 301 | On LLNL machines, you can get cmake by typing `use cmake`. 302 | 303 | ### Basic build 304 | 305 | To build, run this in the Cram root directory: 306 | 307 | cmake -D CMAKE_INSTALL_PREFIX=/path/to/install . 308 | make 309 | make install 310 | 311 | `CMAKE_INSTALL_PREFIX` is the location you want to install Cram, and 312 | `.` is the path to the top-level Cram source directory. 313 | 314 | CMake supports out-of-source builds, so if you like to have separate 315 | build directories for different machines, you can run the above 316 | command in a subdirectory. Just remember to change the path to source 317 | to `..` or similar when you do this, e.g.: 318 | 319 | mkdir $SYS_TYPE && cd $SYS_TYPE 320 | cmake -DCMAKE_INSTALL_PREFIX=/path/to/install .. 321 | 322 | ### Cross-compiling 323 | 324 | On Blue Gene/Q, you also need to supply `-DCMAKE_TOOLCHAIN_FILE` to 325 | get cross compilation working: 326 | 327 | cmake -DCMAKE_INSTALL_PREFIX=/path/to/install \ 328 | -DCMAKE_TOOLCHAIN_FILE=cmake/Toolchain/BlueGeneQ-xl.cmake . 329 | make 330 | make install 331 | 332 | If you want ot build with the GNU compilers, there is a 333 | `BlueGeneQ-gnu.cmake` toolchain file too. This package supports 334 | parallel make, so you are free to use make -j if you like. 335 | 336 | Other notes 337 | ------------------------- 338 | This tool would have been called "Clowncar", but it was decided that 339 | it should have a more serious name. Cram stands for "Clowncar Renamed 340 | to Appease Management". 341 | 342 | 343 | ### Authors 344 | 345 | Cram was written by Todd Gamblin 346 | ([tgamblin@llnl.gov](mailto:tgamblin@llnl.gov)). 347 | 348 | Many contributions, bugfixes, and testing were provided by John 349 | Gyllenhaal ([gyllen@llnl.gov](mailto:gyllen@llnl.gov)). 350 | 351 | 352 | References 353 | ------------------------- 354 | 355 | [1] J. Gyllenhaal, T. Gamblin, A. Bertsch, and R. Musselman. 356 | [Enabling High Job Throughput for Uncertainty Quantification on BG/Q](http://spscicomp.org/wordpress/pages/enabling-high-job-throughput-for-uncertainty-quantification-on-bgq/). In _IBM HPC Systems Scientific Computing User Group (ScicomP’14)_, Chicago, IL, 2014. 357 | 358 | [2] J. Gyllenhaal, T. Gamblin, A. Bertsch, and R. Musselman. ["Cramming" Sequoia Full of Jobs for Uncertainty Quantification](http://nnsa.energy.gov/sites/default/files/nnsa/09-14-inlinefiles/2014-09-12%20ASC%20eNews%20-%20June%202014.pdf). In _ASC eNews Quarterly Newsletter_. June 2014. 359 | -------------------------------------------------------------------------------- /src/python/cram/cramfile.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | # Produced at the Lawrence Livermore National Laboratory. 4 | # 5 | # This file is part of Cram. 6 | # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | # LLNL-CODE-661100 8 | # 9 | # For details, see https://github.com/scalability-llnl/cram. 10 | # Please also see the LICENSE file for our notice and the LGPL. 11 | # 12 | # This program is free software; you can redistribute it and/or modify 13 | # it under the terms of the GNU General Public License (as published by 14 | # the Free Software Foundation) version 2.1 dated February 1999. 15 | # 16 | # This program is distributed in the hope that it will be useful, but 17 | # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | # conditions of the GNU General Public License for more details. 20 | # 21 | # You should have received a copy of the GNU Lesser General Public License 22 | # along with this program; if not, write to the Free Software Foundation, 23 | # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ############################################################################## 25 | """ 26 | This module defines the CramFile class, which is used to store many of 27 | job invocations to be run later in the same MPI job. A job invocation 28 | is simply the context needed to run a single MPI job: number of 29 | processes, working directory, command-line arguments, and environment. 30 | 31 | The format is designed to be read easily from a C program, so it only 32 | writes simple ints and strings. Ints are all unsigned. Strings start 33 | with an integer length, after which all the characters are written out. 34 | 35 | CramFiles use a very simple form of compression to store each job's 36 | environment, since the environment can grow to be very large and is 37 | usually quite redundant. For each job appended to a CramFile after 38 | the first, we compare its environment to the first job's environment, 39 | and we only store the differences. 40 | 41 | We could potentially get more compression out of comparing each 42 | environment to its successor, but that would mean that you'd need to 43 | read all preceding jobs to decode one. We wanted a format that would 44 | allow scattering jobs very quickly to many MPI processes. 45 | 46 | Sample usage: 47 | 48 | cf = CramFile('file.cram', 'w') 49 | cf.pack(Job(64, # number of processes 50 | os.getcwd(), # working dir, as a string 51 | sys.argv[1:], # cmdline args, as a list 52 | os.env)) # environment, as a dict. 53 | cf.close() 54 | 55 | To read from a CramFile, use len and iterate: 56 | 57 | cf = CramFile('file.cram') 58 | num_jobs = len(cf) 59 | for job in cf: 60 | # do something with job 61 | cf.close() 62 | 63 | Like regular files, CramFiles do not support indexing. To do that, 64 | read all jobs into a list first, then index the list. 65 | 66 | Here is the CramFile format. '*' below means that the section can be 67 | repeated a variable number of times. 68 | 69 | Type Name 70 | ======================================================================== 71 | Header 72 | ------------------------------------------------------------------------ 73 | int(4) 0x6372616d ('cram') 74 | int(4) Version 75 | int(4) # of jobs 76 | int(4) # of processes 77 | int(4) Size of max job record in this file 78 | 79 | * Job records 80 | ------------------------------------------------------------------------ 81 | int(4) Size of job record in bytes 82 | int(4) Number of processes 83 | str Working dir 84 | 85 | int(4) Number of command line arguments 86 | * str Command line arguments, in original order 87 | 88 | int(4) Number of subtracted env var names (0 for first record) 89 | * str Subtracted env vars in sorted order. 90 | int(4) Number of added or changed env vars 91 | * str Names of added/changed var 92 | * str Corresponding value 93 | 94 | Env vars are stored alternating keys and values, in sorted order by key. 95 | ======================================================================== 96 | """ 97 | import os 98 | import re 99 | 100 | from collections import defaultdict 101 | from contextlib import contextmanager, closing 102 | 103 | from cram.serialization import * 104 | import llnl.util.tty as tty 105 | 106 | # Magic number goes at beginning of file. 107 | _magic = 0x6372616d 108 | 109 | # Increment this when the binary format changes (hopefully infrequent) 110 | _version = 2 111 | 112 | # Offsets of file header fields 113 | _magic_offset = 0 114 | _version_offset = 4 115 | _njobs_offset = 8 116 | _nprocs_offset = 12 117 | _max_job_offset = 16 118 | 119 | # Default name for cram executable. 120 | USE_APP_EXE = "" 121 | 122 | 123 | @contextmanager 124 | def save_position(stream): 125 | """Context for doing something while saving the current position in a 126 | file.""" 127 | pos = stream.tell() 128 | yield 129 | stream.seek(pos) 130 | 131 | 132 | def compress(base, modified): 133 | """Compare two dicts, modified and base, and return a diff consisting 134 | of two parts: 135 | 136 | 1. missing: set of keys in base but not in modified 137 | 2. changed: dict of key:value pairs either in modified but not in base, 138 | or that have different values in base and modified. 139 | 140 | This simple compression scheme is used to compress the environment 141 | in cramfiles, since that takes up the bulk of the space. 142 | """ 143 | missing = set(base.keys()).difference(modified) 144 | changed = { k:v for k,v in modified.items() 145 | if k not in base or base[k] != v } 146 | return missing, changed 147 | 148 | 149 | def decompress(base, missing, changed): 150 | """Given the base dict and the output of compress(), reconstruct the 151 | modified dict.""" 152 | d = base.copy() 153 | for k in missing: 154 | del d[k] 155 | d.update(changed) 156 | return d 157 | 158 | 159 | class Job(object): 160 | """Simple class to represent one job invocation packed into a cramfile. 161 | This contains all environmental context needed to launch the job 162 | later from within MPI. 163 | """ 164 | def __init__(self, num_procs, working_dir, args, env): 165 | """Construct a new Job object. 166 | 167 | Arguments: 168 | num_procs -- number of processes to run on. 169 | working_dir -- path to working directory for job. 170 | args -- sequence of arguments, INCLUDING the executable name. 171 | env -- dict containng environment. 172 | 173 | """ 174 | 175 | self.num_procs = num_procs 176 | self.working_dir = working_dir 177 | 178 | # Be lenient about the args. Let the user pass a string if he 179 | # wants, and split it like the shell would do. 180 | if isinstance(args, basestring): 181 | args = re.split(r'\s+', args) 182 | self.args = args 183 | 184 | self.env = env 185 | 186 | 187 | def __eq__(self, other): 188 | return (self.num_procs == other.num_procs and 189 | self.working_dir == other.working_dir and 190 | self.args == other.args and 191 | self.env == other.env) 192 | 193 | 194 | def __ne__(self, other): 195 | return not (self == other) 196 | 197 | 198 | class CramFile(object): 199 | """A CramFile compactly stores a number of Jobs, so that they can 200 | later be run within the same MPI job by cram. 201 | """ 202 | def __init__(self, filename, mode='r'): 203 | """The CramFile constructor functions much like open(). 204 | 205 | The constructor takes a filename and an I/O mode, which can 206 | be 'r', 'w', or 'a', for read, write, or append. 207 | 208 | Opening a CramFile for writing will create a file with a 209 | simple header containing no jobs. 210 | """ 211 | # Save the first job from the file. 212 | self.first_job = None 213 | 214 | self.mode = mode 215 | if mode not in ('r', 'w', 'a'): 216 | raise ValueError("Mode must be 'r', 'w', or 'a'.") 217 | 218 | if mode == 'r': 219 | if not os.path.exists(filename) or os.path.isdir(filename): 220 | tty.die("No such file: %s" % filename) 221 | 222 | self.stream = open(filename, 'rb') 223 | self._read_header() 224 | 225 | elif mode == 'w' or (mode == 'a' and not os.path.exists(filename)): 226 | self.stream = open(filename, 'wb') 227 | self.version = _version 228 | self.num_jobs = 0 229 | self.num_procs = 0 230 | self.max_job_size = 0 231 | self._write_header() 232 | 233 | elif mode == 'a': 234 | self.stream = open(filename, 'rb+') 235 | self._read_header() 236 | self.stream.seek(0, os.SEEK_END) 237 | 238 | 239 | def _read_header(self): 240 | """Jump to the beginning of the file and read the header. The cursor 241 | will be at the end of the header on completion, so you will need to 242 | save it if you want to end up somewhere else.""" 243 | self.stream.seek(0) 244 | 245 | magic = read_int(self.stream, 4) 246 | if magic != _magic: 247 | raise IOError("%s is not a Cramfile!") 248 | 249 | self.version = read_int(self.stream, 4) 250 | if self.version != _version: 251 | raise IOError( 252 | "Version mismatch: File has version %s, but this is version %s" 253 | % (self.version, _version)) 254 | 255 | self.num_jobs = read_int(self.stream, 4) 256 | self.num_procs = read_int(self.stream, 4) 257 | self.max_job_size = read_int(self.stream, 4) 258 | 259 | # read in the first job automatically if it is there, since 260 | # it is used for compression of subsequent jobs. 261 | if self.num_jobs > 0: 262 | self._read_job() 263 | 264 | 265 | def _write_header(self): 266 | """Jump to the beginning of the file and write the header.""" 267 | self.stream.seek(0) 268 | write_int(self.stream, _magic, 4) 269 | write_int(self.stream, self.version, 4) 270 | write_int(self.stream, self.num_jobs, 4) 271 | write_int(self.stream, self.num_procs, 4) 272 | write_int(self.stream, self.max_job_size, 4) 273 | 274 | 275 | def _pack(self, job): 276 | """Appends a job to a cram file, compressing the environment in the 277 | process.""" 278 | if self.mode == 'r': 279 | raise IOError("Cannot pack into CramFile opened for reading.") 280 | 281 | # Save offset and write 0 for size to start with 282 | start_offset = self.stream.tell() 283 | size = 0 284 | write_int(self.stream, 0, 4) 285 | 286 | # Number of processes 287 | size += write_int(self.stream, job.num_procs, 4) 288 | 289 | # Working directory 290 | size += write_string(self.stream, job.working_dir) 291 | 292 | # Command line arguments 293 | size += write_int(self.stream, len(job.args), 4) 294 | for arg in job.args: 295 | size += write_string(self.stream, arg) 296 | 297 | # Compress using first dict 298 | missing, changed = compress( 299 | self.first_job.env if self.first_job else {}, job.env) 300 | 301 | # Subtracted env var names 302 | size += write_int(self.stream, len(missing), 4) 303 | for key in sorted(missing): 304 | size += write_string(self.stream, key) 305 | 306 | # Changed environment variables 307 | size += write_int(self.stream, len(changed), 4) 308 | for key in sorted(changed.keys()): 309 | size += write_string(self.stream, key) 310 | size += write_string(self.stream, changed[key]) 311 | 312 | with save_position(self.stream): 313 | # Update job chunk size 314 | self.stream.seek(start_offset) 315 | write_int(self.stream, size, 4) 316 | 317 | # Update total number of jobs in file. 318 | self.num_jobs += 1 319 | self.stream.seek(_njobs_offset) 320 | write_int(self.stream, self.num_jobs, 4) 321 | 322 | # Update total number of processes in all jobs. 323 | self.num_procs += job.num_procs 324 | self.stream.seek(_nprocs_offset) 325 | write_int(self.stream, self.num_procs, 4) 326 | 327 | # Update max job size if necessary 328 | if size > self.max_job_size: 329 | self.max_job_size = size 330 | self.stream.seek(_max_job_offset) 331 | write_int(self.stream, self.max_job_size, 4) 332 | 333 | # Discard all but hte first job after writing. This conserves 334 | # memory when writing cram files. 335 | if not self.first_job: 336 | self.first_job = Job( 337 | job.num_procs, job.working_dir, list(job.args), job.env.copy()) 338 | 339 | 340 | def pack(self, *args, **kwargs): 341 | """Pack a Job into a cram file. 342 | 343 | Takes either a Job object, or Job constructor params. 344 | """ 345 | if len(args) == 1: 346 | job = args[0] 347 | if not isinstance(job, Job): 348 | raise TypeError( 349 | "Must pass a Job object, or (nprocs, working_dir, args, env)") 350 | if kwargs: 351 | raise TypeError("Cannot pass kwargs with cramfile.pack(Job)") 352 | self._pack(job) 353 | 354 | elif len(args) == 4: 355 | nprocs, working_dir, args, env = args 356 | 357 | # Split strings for the caller. 358 | if isinstance(args, basestring): 359 | args = re.split(r'\s+', args) 360 | 361 | # By default, cram takes app's exe name. 362 | exe = kwargs.pop('exe', USE_APP_EXE) 363 | if kwargs: 364 | raise ValueError("%s is an invalid keyword arg for this function!" 365 | % next(iter(kwargs.keys()))) 366 | 367 | args = [exe] + list(args) 368 | self._pack(Job(nprocs, working_dir, args, env)) 369 | 370 | 371 | def _read_job(self): 372 | """Read the next job out of the CramFile. 373 | 374 | This is an internal method because it's used to load stuff 375 | that isn't already in memory. Client code should use 376 | len(), [], or iterate to read jobs from CramFiles. 377 | """ 378 | # Size of job record 379 | job_bytes = read_int(self.stream, 4) 380 | start_pos = self.stream.tell() 381 | 382 | # Number of processes 383 | num_procs = read_int(self.stream, 4) 384 | 385 | # Working directory 386 | working_dir = read_string(self.stream) 387 | 388 | # Command line arguments 389 | num_args = read_int(self.stream, 4) 390 | args = [] 391 | for i in xrange(num_args): 392 | args.append(read_string(self.stream)) 393 | 394 | # Subtracted environment variables 395 | num_missing = read_int(self.stream, 4) 396 | missing = [] 397 | for i in xrange(num_missing): 398 | missing.append(read_string(self.stream)) 399 | 400 | # Changed environment variables 401 | num_changed = read_int(self.stream, 4) 402 | changed = {} 403 | for i in xrange(num_changed): 404 | key = read_string(self.stream) 405 | val = read_string(self.stream) 406 | changed[key] = val 407 | 408 | # validate job record size 409 | actual_size = self.stream.tell() - start_pos 410 | if actual_size != job_bytes: 411 | raise Exception("Cram file job record size is invalid! "+ 412 | "Expected %d, found %d" % (job_bytes, actual_size)) 413 | 414 | # Decompress using first dictionary 415 | env = decompress(self.first_job.env if self.first_job else {}, 416 | missing, changed) 417 | 418 | job = Job(num_procs, working_dir, args, env) 419 | if not self.first_job: 420 | self.first_job = job 421 | return job 422 | 423 | 424 | def __iter__(self): 425 | """Iterate over all jobs in the CramFile.""" 426 | if self.mode != 'r': 427 | raise IOError("Cramfile is not opened for reading.") 428 | 429 | yield self.first_job 430 | for i in xrange(1, self.num_jobs): 431 | yield self._read_job() 432 | 433 | 434 | def __len__(self): 435 | """Number of jobs in the file.""" 436 | return self.num_jobs 437 | 438 | 439 | def close(self): 440 | """Close the underlying file stream.""" 441 | self.stream.close() 442 | -------------------------------------------------------------------------------- /src/c/libcram/cram_file.c: -------------------------------------------------------------------------------- 1 | ////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2014, Lawrence Livermore National Security, LLC. 3 | // Produced at the Lawrence Livermore National Laboratory. 4 | // 5 | // This file is part of Cram. 6 | // Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. 7 | // LLNL-CODE-661100 8 | // 9 | // For details, see https://github.com/scalability-llnl/cram. 10 | // Please also see the LICENSE file for our notice and the LGPL. 11 | // 12 | // This program is free software; you can redistribute it and/or modify 13 | // it under the terms of the GNU General Public License (as published by 14 | // the Free Software Foundation) version 2.1 dated February 1999. 15 | // 16 | // This program is distributed in the hope that it will be useful, but 17 | // WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF 18 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and 19 | // conditions of the GNU General Public License for more details. 20 | // 21 | // You should have received a copy of the GNU Lesser General Public License 22 | // along with this program; if not, write to the Free Software Foundation, 23 | // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 24 | ////////////////////////////////////////////////////////////////////////////// 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "cram_file.h" 35 | 36 | // Magic number goes at beginning of file 37 | #define MAGIC 0x6372616d 38 | 39 | // Tag for cram messages 40 | #define CRAM_TAG 7675 41 | 42 | // Offsets of file header fields 43 | #define MAGIC_OFFSET 0 44 | #define VERSION_OFFSET 4 45 | #define NJOBS_OFFSET 8 46 | #define NPROCS_OFFSET 12 47 | #define MAX_JOB_OFFSET 16 48 | 49 | // offset of first job record 50 | #define JOB_RECORD_OFFSET 20 51 | 52 | // max concurrent ranks to send job records to at once. 53 | #define MAX_CONCURRENT_PEERS 512 54 | 55 | // Ideal number of bytes to use for Lustre read buffers: 2MB. 56 | #define LUSTRE_BUFFER_SIZE 2097152 57 | 58 | // Default cram executable name: means we should use argv[0] for exe. 59 | #define CRAM_DEFAULT_EXE "" 60 | 61 | 62 | // ------------------------------------------------------------------------ 63 | // Globals used by fortran arg routines 64 | // ------------------------------------------------------------------------ 65 | int cram_argc = 0; 66 | const char **cram_argv = NULL; 67 | 68 | 69 | // ------------------------------------------------------------------------ 70 | // Utility functions 71 | // ------------------------------------------------------------------------ 72 | 73 | /// 74 | /// Find strings efficiently in a sorted array using bsearch. 75 | /// 76 | static inline size_t index_of(size_t num_elts, 77 | const char **sorted_array, 78 | const char *string) { 79 | const char **pos = bsearch(string, sorted_array, num_elts, sizeof(char*), 80 | (int(*)(const void*, const void*))strcmp); 81 | return (pos - sorted_array); 82 | } 83 | 84 | 85 | /// 86 | /// Free all the strings in a char**, then free the array itself. 87 | /// 88 | static void free_string_array(int num_elts, const char **arr) { 89 | for (int i=0; i < num_elts; i++) { 90 | free((char*)arr[i]); 91 | } 92 | free(arr); 93 | } 94 | 95 | 96 | /// 97 | /// Create a copy of an entire string array. 98 | /// 99 | static const char **dup_string_array(int num_elts, const char **src) { 100 | const char **arr = malloc(sizeof(char*) * num_elts); 101 | for (int i=0; i < num_elts; i++) { 102 | arr[i] = strdup(src[i]); 103 | } 104 | return arr; 105 | } 106 | 107 | 108 | /// 109 | /// Read a cram int from a FILE*. 110 | /// 111 | static int file_read_int(const cram_file_t *file) { 112 | int buf; 113 | size_t ints = fread(&buf, sizeof(int), 1, file->fd); 114 | if (ints != 1) { 115 | fprintf(stderr, "Error reading cram file. " 116 | "Expected one int but got read %zd ints.\n", ints); 117 | exit(1); 118 | } 119 | return ntohl(buf); 120 | } 121 | 122 | 123 | /// 124 | /// Read a cram int from a buffer 125 | /// 126 | static int buf_read_int(const char *buf, size_t *offset) { 127 | const int *int_ptr = (int*)&buf[*offset]; 128 | *offset += sizeof(int); 129 | return ntohl(*int_ptr); 130 | } 131 | 132 | 133 | /// 134 | /// Read a cram string from a buffer 135 | /// 136 | static char *buf_read_string(const char *buf, size_t *offset) { 137 | size_t size = buf_read_int(buf, offset); 138 | char *string = strndup(&buf[*offset], size); 139 | *offset += size; 140 | return string; 141 | } 142 | 143 | 144 | static size_t get_cram_buffer_size() { 145 | const char *bufsize_string = getenv("CRAM_BUFFER_SIZE"); 146 | if (!bufsize_string) { 147 | return LUSTRE_BUFFER_SIZE; 148 | } 149 | 150 | int rank; 151 | PMPI_Comm_rank(MPI_COMM_WORLD, &rank); 152 | 153 | char *endptr; 154 | size_t bufsize = strtoll(bufsize_string, &endptr, 10); 155 | if (*bufsize_string && *endptr == '\0') { 156 | if (rank == 0) { 157 | fprintf(stderr, "Using CRAM_BUFFER_SIZE=%d.\n", bufsize); 158 | } 159 | return bufsize; 160 | } else { 161 | if (rank == 0) { 162 | fprintf(stderr, "Warning: Invalid value for CRAM_BUFFER_SIZE: %s. " 163 | "Using default of %d", bufsize_string, LUSTRE_BUFFER_SIZE); 164 | } 165 | return LUSTRE_BUFFER_SIZE; 166 | } 167 | } 168 | 169 | 170 | // ------------------------------------------------------------------------ 171 | // Public cram file interface 172 | // ------------------------------------------------------------------------ 173 | 174 | bool cram_file_open(const char *filename, cram_file_t *file) { 175 | file->fd = fopen(filename, "r"); 176 | if (file->fd == NULL) { 177 | return false; 178 | } 179 | 180 | // Try to use a large buffer to read the file fast. 181 | setvbuf(file->fd, NULL, _IOFBF, get_cram_buffer_size()); 182 | 183 | // check magic number at start of header 184 | int magic = file_read_int(file); 185 | if (magic != MAGIC) { 186 | fprintf(stderr, "Error: %s is not a cram file!", filename); 187 | return false; 188 | } 189 | 190 | // read rest of header after magic check. 191 | file->version = file_read_int(file); 192 | file->num_jobs = file_read_int(file); 193 | file->total_procs = file_read_int(file); 194 | file->max_job_size = file_read_int(file); 195 | 196 | file->cur_job_record_size = 0; 197 | file->cur_job_procs = 0; 198 | file->cur_job_id = -1; 199 | 200 | return true; 201 | } 202 | 203 | 204 | void cram_file_close(const cram_file_t *file) { 205 | fclose(file->fd); 206 | } 207 | 208 | 209 | bool cram_file_has_more_jobs(const cram_file_t *file) { 210 | return file->cur_job_id < (file->num_jobs - 1); 211 | } 212 | 213 | 214 | /// 215 | /// Helper for cram_job_decompress -- does the real work. 216 | /// 217 | static inline void decompress(const cram_job_t *base, 218 | int num_missing, const char **missing, 219 | int num_changed, const char **changed_keys, 220 | const char **changed_vals, 221 | cram_job_t *job) { 222 | // Figure out how many environemnt variables overlap with base 223 | int num_overlap = 0; 224 | for (int i=0; i < num_changed; i++) { 225 | if (index_of(base->num_env_vars, base->keys, changed_keys[i])) { 226 | num_overlap++; 227 | } 228 | } 229 | 230 | // Allocate space for new vars. 231 | job->num_env_vars = base->num_env_vars 232 | + num_changed - num_overlap - num_missing; 233 | job->keys = malloc(job->num_env_vars * sizeof(const char*)); 234 | job->values = malloc(job->num_env_vars * sizeof(const char*)); 235 | 236 | // Merge base and changed values into job, removing missing keys. 237 | // These arrays are assumed to be sorted, so we can march through 238 | // them in O(n) time. 239 | size_t bx=0, cx=0, mx=0, jx=0; 240 | while (jx < job->num_env_vars) { 241 | int cmp = -1; 242 | if (cx < num_changed) { 243 | cmp = strcmp(base->keys[bx], changed_keys[cx]); 244 | } 245 | 246 | if (cmp == 0) { 247 | // base & changed are the same, take the changed value and skip base. 248 | job->keys[jx] = strdup(changed_keys[cx]); 249 | job->values[jx] = strdup(changed_vals[cx]); 250 | bx++; cx++; jx++; 251 | 252 | } else if (cmp < 0) { 253 | if (mx < num_missing && strcmp(base->keys[bx], missing[mx]) == 0) { 254 | // This key is missing in job; skip it. 255 | bx++; mx++; 256 | 257 | } else { 258 | // base < changed: this key is preserved in the new job. 259 | job->keys[jx] = strdup(base->keys[bx]); 260 | job->values[jx] = strdup(base->values[bx]); 261 | bx++; jx++; 262 | } 263 | 264 | } else { /* cmp > 0 */ 265 | // This key was added, and isn't in base. Take it from changed. 266 | job->keys[jx] = strdup(changed_keys[cx]); 267 | job->values[jx] = strdup(changed_vals[cx]); 268 | cx++; jx++; 269 | } 270 | } 271 | } 272 | 273 | 274 | void cram_job_decompress(const char *job_record, 275 | const cram_job_t *base, cram_job_t *job) { 276 | // start at beginning of job record. 277 | size_t offset = 0; 278 | 279 | // num_procs 280 | job->num_procs = buf_read_int(job_record, &offset); 281 | 282 | // working directory 283 | job->working_dir = buf_read_string(job_record, &offset); 284 | 285 | // command line arguments 286 | int num_args = buf_read_int(job_record, &offset); 287 | job->num_args = num_args; 288 | job->args = (const char**) malloc(num_args * sizeof(char*)); 289 | 290 | for (int i=0; i < num_args; i++) { 291 | job->args[i] = buf_read_string(job_record, &offset); 292 | } 293 | 294 | // Subtracted environment variables are not in this job but are 295 | // in the base job. 296 | int num_subtracted = buf_read_int(job_record, &offset); 297 | 298 | const char **subtracted_env_vars = NULL; 299 | if (num_subtracted) { 300 | // If there is no base job, then there can't be any subtracted vars. 301 | if (!base) { 302 | fprintf(stderr, "Cannot decompress this job without a base job!\n"); 303 | PMPI_Abort(MPI_COMM_WORLD, 1); 304 | } 305 | 306 | subtracted_env_vars = malloc(num_subtracted * sizeof(const char*)); 307 | for (int i=0; i < num_subtracted; i++) { 308 | subtracted_env_vars[i] = buf_read_string(job_record, &offset); 309 | } 310 | } 311 | 312 | // Changed environemnt vars were either added to the base or they're different 313 | // in job from in base. 314 | int num_changed = buf_read_int(job_record, &offset); 315 | 316 | const char **changed_keys = malloc(num_changed * sizeof(const char*)); 317 | const char **changed_vals = malloc(num_changed * sizeof(const char*)); 318 | for (int i=0; i < num_changed; i++) { 319 | changed_keys[i] = buf_read_string(job_record, &offset); 320 | 321 | changed_vals[i] = buf_read_string(job_record, &offset); 322 | } 323 | 324 | if (base) { 325 | // Apply diffs to the base job and write result into job. 326 | decompress(base, num_subtracted, subtracted_env_vars, 327 | num_changed, changed_keys, changed_vals, job); 328 | 329 | // free temporaries 330 | free_string_array(num_subtracted, subtracted_env_vars); 331 | free_string_array(num_changed, changed_keys); 332 | free_string_array(num_changed, changed_vals); 333 | 334 | } else { 335 | // If there is no base, we just copy the changed vals (first job) 336 | job->num_env_vars = num_changed; 337 | job->keys = changed_keys; 338 | job->values = changed_vals; 339 | } 340 | } 341 | 342 | 343 | bool cram_file_next_job(cram_file_t *file, char *job_record) { 344 | int job_record_size = file_read_int(file); 345 | if (job_record_size > file->max_job_size) { 346 | fprintf(stderr, "Error: Invalid job record size: %d > %d", 347 | job_record_size, file->max_job_size); 348 | return false; 349 | } 350 | 351 | file->cur_job_record_size = job_record_size; 352 | size_t bytes = fread(job_record, 1, job_record_size, file->fd); 353 | if (bytes != job_record_size) { 354 | fprintf(stderr, "Error: Expected to read %d bytes, but got %zd\n", 355 | job_record_size, bytes); 356 | return false; 357 | } 358 | 359 | size_t offset = 0; 360 | file->cur_job_procs = buf_read_int(job_record, &offset); 361 | file->cur_job_id++; 362 | 363 | return true; 364 | } 365 | 366 | 367 | void cram_file_bcast_jobs(cram_file_t *file, int root, cram_job_t *job, int *id, 368 | MPI_Comm comm) { 369 | int rank, size; 370 | PMPI_Comm_rank(comm, &rank); 371 | PMPI_Comm_size(comm, &size); 372 | 373 | // check total procs and grab the max job size. 374 | int max_job_size; 375 | if (rank == root) { 376 | if (file->total_procs > size) { 377 | fprintf(stderr, "Error: This cram file requires %d processes, " 378 | "but this communicator has only %d.\n", file->total_procs, size); 379 | PMPI_Abort(comm, 1); 380 | } 381 | max_job_size = file->max_job_size; 382 | } 383 | 384 | // bcast max job size 385 | PMPI_Bcast(&max_job_size, 1, MPI_INT, root, comm); 386 | char *job_record = malloc(max_job_size); 387 | 388 | // read in compressed data for first job record 389 | if (rank == root) { 390 | if (!cram_file_next_job(file, job_record)) { 391 | fprintf(stderr, "Error reading job %d from cram file on rank %d\n", 392 | 0, root); 393 | PMPI_Abort(comm, 1); 394 | } 395 | } 396 | 397 | // Bcast and decompress first job. 398 | cram_job_t first_job; 399 | PMPI_Bcast(job_record, max_job_size, MPI_CHAR, root, comm); 400 | cram_job_decompress(job_record, NULL, &first_job); 401 | 402 | // start by sending to the first rank in the second job. 403 | int cur_rank = first_job.num_procs; 404 | bool in_first_job = rank < cur_rank; 405 | 406 | if (rank == root) { 407 | // Root needs to send to all the other jobs 408 | // Start by iterating through remaining jobs in the file. 409 | while (cram_file_has_more_jobs(file)) { 410 | if (!cram_file_next_job(file, job_record)) { 411 | fprintf(stderr, "Error reading job %d from cram file on rank %d\n", 412 | file->cur_job_id + 1, root); 413 | PMPI_Abort(comm, 1); 414 | } 415 | 416 | // array of requests for all sends we'll do 417 | // max concurrent peers max number of ranks we'll send to at once. 418 | int max_requests = MAX_CONCURRENT_PEERS * 2; 419 | MPI_Request requests[max_requests]; 420 | 421 | // iterate through all ranks in this job, incrementing first_rank as we go. 422 | int end_rank = cur_rank + file->cur_job_procs; 423 | while (cur_rank < end_rank) { 424 | int r = 0; 425 | while (r < max_requests && cur_rank < end_rank) { 426 | PMPI_Isend(&file->cur_job_id, 1, MPI_INT, cur_rank, 427 | CRAM_TAG, comm, &requests[r++]); 428 | PMPI_Isend(job_record, file->cur_job_record_size, MPI_CHAR, cur_rank, 429 | CRAM_TAG, comm, &requests[r++]); 430 | cur_rank++; 431 | } 432 | PMPI_Waitall(r, requests, MPI_STATUSES_IGNORE); 433 | } 434 | } 435 | 436 | // send a job id of -1 to any inactive ranks 437 | int inactive_rank_id = -1; 438 | for (; cur_rank < size; cur_rank++) { 439 | PMPI_Send(&inactive_rank_id, 1, MPI_INT, cur_rank, CRAM_TAG, comm); 440 | } 441 | 442 | } else if (!in_first_job) { 443 | // Ranks NOT in the first job need to receive their actual job record. 444 | // Ranks in the first job already have their job record. 445 | PMPI_Recv(id, 1, MPI_INT, root, CRAM_TAG, comm, MPI_STATUS_IGNORE); 446 | if (*id >= 0) { 447 | PMPI_Recv(job_record, max_job_size, MPI_CHAR, root, CRAM_TAG, comm, 448 | MPI_STATUS_IGNORE); 449 | } 450 | cram_job_decompress(job_record, &first_job, job); 451 | } 452 | 453 | // If this rank is in the first job, then just copy the first job we 454 | // got from the bcast. And set the id to zero. 455 | if (in_first_job) { 456 | cram_job_copy(&first_job, job); 457 | *id = 0; 458 | } 459 | 460 | // Can free the first job now b/c we don't need it. 461 | cram_job_free(&first_job); 462 | } 463 | 464 | 465 | static void arg_copy(const cram_job_t *job, 466 | int *dest_argc, const char ***dest_argv) { 467 | 468 | *dest_argc = job->num_args; 469 | *dest_argv = malloc(job->num_args * sizeof(char**)); 470 | for (int i=0; i < job->num_args; i++) { 471 | (*dest_argv)[i] = strdup(job->args[i]); 472 | } 473 | } 474 | 475 | 476 | void cram_job_setup(const cram_job_t *job, int *argc, const char ***argv) { 477 | // change working directory 478 | chdir(job->working_dir); 479 | 480 | // save argv[0] so that we can use it for the first arg. 481 | const char *exe_name = NULL; 482 | if (*argc > 0 && *argv) { 483 | exe_name = (*argv)[0]; 484 | } 485 | 486 | // Replace command line arguments with those of the job. 487 | arg_copy(job, argc, argv); 488 | 489 | // set argv[0] to the actual exe name 490 | if (strcmp(job->args[0], CRAM_DEFAULT_EXE) == 0) { 491 | if (exe_name) { 492 | free((void*)(*argv)[0]); 493 | (*argv)[0] = strdup(exe_name); 494 | } 495 | } 496 | 497 | // Also copy arguments into globals for Fortran arg interceptors to access. 498 | arg_copy(job, &cram_argc, &cram_argv); 499 | 500 | // Set environment variables based on the job's key/val pairs. 501 | for (int i=0; i < job->num_env_vars; i++) { 502 | setenv(job->keys[i], job->values[i], 1); 503 | } 504 | } 505 | 506 | 507 | void cram_job_free(cram_job_t *job) { 508 | free((char*)job->working_dir); 509 | free_string_array(job->num_args, job->args); 510 | free_string_array(job->num_env_vars, job->keys); 511 | free_string_array(job->num_env_vars, job->values); 512 | } 513 | 514 | 515 | void cram_job_copy(const cram_job_t *src, cram_job_t *dest) { 516 | dest->num_procs = src->num_procs; 517 | dest->working_dir = strdup(src->working_dir); 518 | 519 | dest->num_args = src->num_args; 520 | dest->args = dup_string_array(src->num_args, src->args); 521 | 522 | dest->num_env_vars = src->num_env_vars; 523 | dest->keys = dup_string_array(src->num_env_vars, src->keys); 524 | dest->values = dup_string_array(src->num_env_vars, src->values); 525 | } 526 | 527 | 528 | void cram_job_print(const cram_job_t *job) { 529 | printf(" Num procs: %d\n", job->num_procs); 530 | printf(" Working dir: %s\n", job->working_dir); 531 | printf(" Arguments:\n"); 532 | 533 | printf(" "); 534 | for (int i=0; i < job->num_args; i++) { 535 | if (i > 0) printf(" "); 536 | printf("%s", job->args[i]); 537 | } 538 | printf("\n"); 539 | 540 | printf(" Environment:\n"); 541 | for (int i=0; i < job->num_env_vars; i++) { 542 | printf(" '%s' : '%s'\n", job->keys[i], job->values[i]); 543 | } 544 | } 545 | 546 | 547 | void cram_file_cat(cram_file_t *file) { 548 | printf("Number of Jobs: %12d\n", file->num_jobs); 549 | printf("Total Procs: %12d\n", file->total_procs); 550 | printf("Cram version: %12d\n", file->version); 551 | printf("Max job record: %12d\n", file->max_job_size); 552 | printf("\n"); 553 | printf("Job information:\n"); 554 | 555 | if (!cram_file_has_more_jobs(file)) { 556 | return; 557 | } 558 | 559 | // space for raw, compressed job record. 560 | char *job_record = malloc(file->max_job_size); 561 | 562 | // First job is special because we don't have to decompress 563 | cram_job_t first_job; 564 | cram_file_next_job(file, job_record); 565 | cram_job_decompress(job_record, NULL, &first_job); 566 | 567 | // print first job 568 | printf("Job %d:\n", file->cur_job_id); 569 | cram_job_print(&first_job); 570 | 571 | // Rest of jobs are based on first job. 572 | cram_job_t job; 573 | while (cram_file_has_more_jobs(file)) { 574 | cram_file_next_job(file, job_record); 575 | cram_job_decompress(job_record, &first_job, &job); 576 | 577 | // print each subsequent job 578 | printf("Job %d:\n", file->cur_job_id); 579 | cram_job_print(&job); 580 | 581 | cram_job_free(&job); 582 | } 583 | 584 | free(job_record); 585 | cram_job_free(&first_job); 586 | } 587 | --------------------------------------------------------------------------------