├── .github └── workflows │ └── cmake.yml ├── .gitignore ├── .gitmodules ├── .readthedocs.yaml ├── CMake ├── recorder-config-version.cmake.in ├── recorder-config.cmake.build.in └── recorder-config.cmake.install.in ├── CMakeLists.txt ├── LICENSE ├── README.md ├── docs ├── make.bat ├── requirements.txt └── source │ ├── build.rst │ ├── changes.rst │ ├── conf.py │ ├── features.rst │ ├── index.rst │ ├── postprocessing.rst │ ├── quickstart.rst │ └── usage.rst ├── exclusion_prefix.txt ├── include ├── recorder-cuda-profiler.h ├── recorder-gotcha.h ├── recorder-logger.h ├── recorder-pattern-recognition.h ├── recorder-sequitur.h ├── recorder-timestamps.h ├── recorder-utils.h ├── recorder.h ├── uthash.h └── utlist.h ├── lib ├── CMakeLists.txt ├── recorder-cst-cfg.c ├── recorder-cuda-profiler.c ├── recorder-function-profiler.c ├── recorder-gotcha.c ├── recorder-hdf5.c ├── recorder-init-finalize.c ├── recorder-logger.c ├── recorder-mpi.c ├── recorder-netcdf.c ├── recorder-pattern-recognition.c ├── recorder-pnetcdf.c ├── recorder-posix.c ├── recorder-sequitur-digram.c ├── recorder-sequitur-logger.c ├── recorder-sequitur-symbol.c ├── recorder-sequitur-utils.c ├── recorder-sequitur.c ├── recorder-timestamps.c └── recorder-utils.c ├── m4 └── README ├── test ├── test_hdf5.c ├── test_hybrid.c ├── test_iopr.c ├── test_mpi.c ├── test_phdf5.c ├── test_posix.c ├── test_signal.c └── vec.cu └── tools ├── CMakeLists.txt ├── build-offset-intervals.cpp ├── conflict-detector.cpp ├── generator ├── clean.py ├── generator.py ├── hdf5-1.14.4.2-funcs.txt ├── hdf5-1.14.4.2-funcs.txt2 ├── hdf5-new-funcs.txt ├── netcdf-4.9.2-funcs.txt └── pnetcdf-1.13.0-funcs.txt ├── meta-ops-checker.c ├── reader-cst-cfg.c ├── reader-private.h ├── reader.c ├── reader.h ├── recorder-filter.cpp ├── recorder-summary.c ├── recorder2parquet.cpp ├── recorder2text.c ├── recorder2timeline.cpp ├── reporter ├── CMakeLists.txt ├── exp_plot.py └── reporter.py └── verifyio ├── README.md ├── match_mpi.py ├── read_nodes.py ├── recorder_reader.py ├── vector_clock.py ├── verifyio.py ├── verifyio_graph.py └── visualize.py /.github/workflows/cmake.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: [ dev ] 6 | pull_request: 7 | branches: [ dev ] 8 | 9 | env: 10 | # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) 11 | BUILD_TYPE: Release 12 | 13 | jobs: 14 | build: 15 | # The CMake configure and build commands are platform agnostic and should work equally 16 | # well on Windows or Mac. You can convert this to a matrix build if you need 17 | # cross-platform coverage. 18 | # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix 19 | runs-on: ubuntu-latest 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | with: 24 | submodules: 'true' 25 | 26 | - name: Install HDF5 and MPI 27 | run: | 28 | sudo apt-get update 29 | sudo apt-get install libhdf5-mpi-dev 30 | 31 | - name: Install Apache Arrow and Parquet 32 | run: | 33 | sudo apt update 34 | sudo apt install -y -V ca-certificates lsb-release wget 35 | wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb 36 | sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb 37 | sudo apt update 38 | sudo apt install -y -V libarrow-dev 39 | sudo apt install -y -V libparquet-dev 40 | 41 | - name: Configure CMake 42 | # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. 43 | # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type 44 | run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} 45 | 46 | - name: Build 47 | # Build your program with the given configuration 48 | run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} 49 | 50 | #- name: Test 51 | # working-directory: ${{github.workspace}}/build 52 | # Execute tests defined by the CMake configuration. 53 | # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail 54 | #run: ctest -C ${{env.BUILD_TYPE}} 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Libraries 12 | *.lib 13 | *.a 14 | *.la 15 | *.lo 16 | 17 | # Shared objects (inc. Windows DLLs) 18 | *.dll 19 | *.so 20 | *.so.* 21 | *.dylib 22 | 23 | # Executables 24 | *.exe 25 | *.out 26 | *.app 27 | *.i*86 28 | *.x86_64 29 | *.hex 30 | 31 | *.po 32 | config.inc 33 | .DS_Store 34 | .vscode/ 35 | 36 | # Autotools files 37 | Makefile.in 38 | /ar-lib 39 | /mdate-sh 40 | /py-compile 41 | /test-driver 42 | /ylwrap 43 | .deps/ 44 | .dirstamp 45 | .libs 46 | # http://www.gnu.org/software/autoconf 47 | autom4te.cache 48 | /autoscan.log 49 | /autoscan-*.log 50 | /aclocal.m4 51 | /compile 52 | /config.guess 53 | /config.h.in 54 | /config.log 55 | /config.status 56 | /config.sub 57 | /configure 58 | /configure.scan 59 | /depcomp 60 | /install-sh 61 | /missing 62 | /stamp-h1 63 | # https://www.gnu.org/software/libtool/ 64 | /libtool 65 | /ltmain.sh 66 | # http://www.gnu.org/software/texinfo 67 | /texinfo.tex 68 | # http://www.gnu.org/software/m4/ 69 | m4/libtool.m4 70 | m4/ltoptions.m4 71 | m4/ltsugar.m4 72 | m4/ltversion.m4 73 | m4/lt~obsolete.m4 74 | 75 | # Generated Makefile 76 | # (meta build system like autotools, 77 | # can automatically generate from config.status script 78 | # (which is called by configure script)) 79 | Makefile 80 | 81 | 82 | # Configuration file 83 | *.inc 84 | 85 | # Python 86 | __pycache__/ 87 | *.py[cod] 88 | *$py.class 89 | 90 | 91 | # App-specific 92 | logs/ 93 | recorder-logs/ 94 | tools/otf2-converter 95 | tools/conflict_detector 96 | tools/recorder2text 97 | tools/metaops_checker 98 | tools/figures/ 99 | install 100 | *.html 101 | 102 | /build/ 103 | .cache 104 | 105 | # IDE-specific 106 | .idea/ 107 | /cmake-build-debug/ 108 | 109 | # clangd database 110 | compile_commands.json 111 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/GOTCHA"] 2 | path = deps/GOTCHA 3 | url = https://github.com/LLNL/GOTCHA.git 4 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.12" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Build documentation in the "docs/" directory with Sphinx 18 | sphinx: 19 | configuration: docs/source/conf.py 20 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 21 | # builder: "dirhtml" 22 | # Fail on all warnings to avoid broken references 23 | # fail_on_warning: true 24 | 25 | # Optionally build your docs in additional formats such as PDF and ePub 26 | # formats: 27 | # - pdf 28 | # - epub 29 | 30 | # Optional but recommended, declare the Python requirements required 31 | # to build your documentation 32 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 33 | python: 34 | install: 35 | - requirements: docs/requirements.txt 36 | -------------------------------------------------------------------------------- /CMake/recorder-config-version.cmake.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Version file for install directory 3 | #----------------------------------------------------------------------------- 4 | set(PACKAGE_VERSION @RECORDER_PACKAGE_VERSION@) 5 | 6 | if("${PACKAGE_FIND_VERSION_MAJOR}" EQUAL @RECORDER_VERSION_MAJOR@) 7 | if("${PACKAGE_FIND_VERSION_MINOR}" EQUAL @RECORDER_VERSION_MINOR@) 8 | set(PACKAGE_VERSION_COMPATIBLE 1) 9 | if("${PACKAGE_FIND_VERSION_PATCH}" EQUAL @RECORDER_VERSION_PATCH@) 10 | set(PACKAGE_VERSION_EXACT 1) 11 | endif() 12 | endif() 13 | endif() -------------------------------------------------------------------------------- /CMake/recorder-config.cmake.build.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Config file for compiling against the build directory 3 | #----------------------------------------------------------------------------- 4 | get_filename_component(SELF_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 5 | 6 | #----------------------------------------------------------------------------- 7 | # Directories 8 | #----------------------------------------------------------------------------- 9 | set(RECORDER_INCLUDE_DIR "@RECORDER_INCLUDES_BUILD_TIME@") 10 | 11 | #----------------------------------------------------------------------------- 12 | # Version Strings 13 | #----------------------------------------------------------------------------- 14 | set(RECORDER_VERSION_STRING @RECORDER_PACKAGE_VERSION@) 15 | set(RECORDER_VERSION_MAJOR @RECORDER_VERSION_MAJOR@) 16 | set(RECORDER_VERSION_MINOR @RECORDER_VERSION_MINOR@) 17 | set(RECORDER_VERSION_PATCH @RECORDER_VERSION_PATCH@) -------------------------------------------------------------------------------- /CMake/recorder-config.cmake.install.in: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------------- 2 | # Config file for compiling against the build directory 3 | #----------------------------------------------------------------------------- 4 | get_filename_component(SELF_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 5 | 6 | #----------------------------------------------------------------------------- 7 | # Directories 8 | #----------------------------------------------------------------------------- 9 | set(RECORDER_INCLUDE_DIR "@RECORDER_INCLUDES_INSTALL_TIME@") 10 | 11 | #----------------------------------------------------------------------------- 12 | # Version Strings 13 | #----------------------------------------------------------------------------- 14 | set(RECORDER_VERSION_STRING @RECORDER_PACKAGE_VERSION@) 15 | set(RECORDER_VERSION_MAJOR @RECORDER_VERSION_MAJOR@) 16 | set(RECORDER_VERSION_MINOR @RECORDER_VERSION_MINOR@) 17 | set(RECORDER_VERSION_PATCH @RECORDER_VERSION_PATCH@) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 University of Illinois Board of Trustees. All Rights Reserved. 2 | Developed by: Chen Wang, chenw5@illinois.edu 3 | 4 | High-Performance Computing Group 5 | University of Illinois at Urbana-Chamapign 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy of 8 | this software and associated documentation files (the "Software"), to deal with 9 | the Software without restriction, including without limitation the rights to use, 10 | copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the 11 | Software, and to permit persons to whom the Software is furnished to do so, subject 12 | to the following conditions: 13 | 14 | Redistributions of source code must retain the above copyright notice, this list of 15 | conditions and the following disclaimers. 16 | Redistributions in binary form must reproduce the above copyright notice, this list 17 | of conditions and the following disclaimers in the documentation and/or other materials 18 | provided with the distribution. Neither the names of High-Performance Computing Group, 19 | University of Illinois, nor the names of its contributors may be used to endorse or 20 | promote products derived from this Software without specific prior written permission. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 23 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 24 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE 25 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT 26 | OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 | DEALINGS WITH THE SOFTWARE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![build](https://github.com/uiuc-hpc/Recorder/actions/workflows/cmake.yml/badge.svg)](https://github.com/uiuc-hpc/Recorder/actions/workflows/cmake.yml) 2 | 3 | # Recorder 4 | 5 | **Comprehensive Parallel I/O Tracing and Analysis** 6 | 7 | Recorder is a multi-level I/O tracing framework that can capture I/O function 8 | calls at multiple levels of the I/O stack, including HDF5, PnetCDF, NetCDF, MPI-IO, 9 | and POSIX I/O. Recorder requires no modification or recompilation of the application and 10 | users can control what levels are traced. 11 | 12 | 13 | ### Quickstart 14 | 15 | Build: 16 | 17 | ```bash 18 | git clone https://github.com/uiuc-hpc/Recorder.git 19 | cd Recorder 20 | export RECORDER_INSTALL_PATH=`pwd`/install 21 | git submodule update --init --recursive 22 | mkdir build && cd build 23 | cmake .. -DCMAKE_INSTALL_PREFIX=$RECORDER_INSTALL_PATH 24 | make && make install 25 | ``` 26 | 27 | Run: 28 | ```bash 29 | mpirun -np N -env LD_PRELOAD $RECORDER_INSTALL_PATH/lib/librecorder.so ./your-app 30 | 31 | # On HPC systems, you may need to use srun or 32 | # other job schedulers to replace mpirun, e.g., 33 | srun -n4 -N1 --overlap --export=ALL,LD_PRELOAD=$RECORDER_INSTALL_PATH/lib/librecorder.so ./your-app 34 | flux run -n 4 --env LD_PRELOAD=$RECORDER_INSTALL_PATH/ilb/librecorder.so ./your-app 35 | ``` 36 | 37 | 38 | ### Documentation 39 | 40 | Recorder documentation is at [https://recorder.readthedocs.io/](https://recorder.readthedocs.io/). 41 | 42 | 43 | ### Citation 44 | 45 | *Wang, Chen, Jinghan Sun, Marc Snir, Kathryn Mohror, and Elsa Gonsiorowski. “Recorder 2.0: Efficient Parallel I/O Tracing and Analysis.” In IEEE International Workshop on High-Performance Storage (HPS), 2020.* 46 | [https://ieeexplore.ieee.org/abstract/document/9150354](https://ieeexplore.ieee.org/abstract/document/9150354) 47 | 48 | Recorder has undergone substantial updates since the publication of the 'Recorder 2.0' paper. These include the integration of a new pattern-based compression algorithm, expanded support for the complete set of HDF5, NetCDF, and PnetCDF calls, as well as numerous additional features. For detailed information, please refer to and cite the latest version here: [https://arxiv.org/abs/2501.04654](https://arxiv.org/abs/2501.04654). 49 | 50 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==7.1.2 2 | sphinx-rtd-theme==1.3.0rc1 3 | furo 4 | myst-parser 5 | -------------------------------------------------------------------------------- /docs/source/build.rst: -------------------------------------------------------------------------------- 1 | Build Recorder 2 | ----------------- 3 | 4 | Building Recorder with CMake 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | **Dependencies** 8 | 9 | - MPI (required) 10 | - HDF5 (required) 11 | - NetCDF (optional) - Needed for NetCDF tracing 12 | - PnetCDF (optional) - Needed for PnetCDF tracing 13 | - Arrow (optional) > 5.0.0 - Needed for building Parquet convertor. 14 | - CUDA (optional) - Needed for CUDA kernels interception. 15 | 16 | 17 | *Note that Recorder and the applications you intend to trace must be 18 | compiled with the same version of HDF5 and MPI.* 19 | 20 | **Build & Install** 21 | 22 | .. code:: bash 23 | 24 | git clone https://github.com/uiuc-hpc/Recorder.git 25 | git submodule update --init --recursive 26 | cd Recorder 27 | mkdir build 28 | cd build 29 | cmake .. -DCMAKE_INSTALL_PREFIX=[install location] 30 | make 31 | make install 32 | 33 | **CMake Options** 34 | 35 | (1) Dependencies install location 36 | 37 | If MPI, HDF5, or arrow is not installed in standard locations, you may 38 | need to use ``-DCMAKE_PREFIX_PATH`` to indicate their locations: 39 | 40 | .. code:: bash 41 | 42 | cmake .. \ 43 | -DCMAKE_INSTALL_PREFIX=[install location] \ 44 | -DCMAKE_PREFIX_PATH=[semicolon separated depedencies dir] 45 | 46 | Recorder can also be built with PnetCDF to trace PnetCDF calls, use `-DRECORDER_WITH_PNETCDF=/path/to/pnetcdf/install` 47 | to build Recorder with the specified PnetCDF. 48 | 49 | (2) Intercepting ``fcntl()`` call: 50 | 51 | Since v2.1.7, ``fcntl(int fd, int cmd, ...)`` is intercepted. The 52 | commands (2nd argument) defined in POSIX standard are supported. If 53 | non-POSIX commands were used, please disable fcntl tracing at configure 54 | time with ``-DRECORDER_ENABLE_FCNTL_TRACE=OFF``. 55 | 56 | (3) Intercepting CUDA kernels: 57 | 58 | add ``-DRECORDER_ENABLE_CUDA_TRACE=ON`` to cmake to allow tracing CUDA 59 | kernels. 60 | 61 | (4) Parquet Converter 62 | 63 | add ``-DRECORDER_ENABLE_PARQUET=ON`` to cmake to build the Parquet 64 | format converter 65 | 66 | 67 | Building Recorder with Spack 68 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 69 | 70 | *NOTE: please do not use Spack to install Recorder for now. The version 71 | there is outdated, we will update it soon.* 72 | 73 | For now, building Recorder with Spack provides less flexibility. We will 74 | add the CMake options for spack as well. 75 | 76 | .. code:: bash 77 | spack install recorder 78 | 79 | By default Recorder generates traces from all levels, you can use **~** 80 | to disable a specific level. 81 | 82 | E.g., the following command will install Recorder with HDF5 and MPI 83 | tracing disabled. 84 | 85 | .. code:: bash 86 | 87 | spack install recorder~hdf5~mpi 88 | -------------------------------------------------------------------------------- /docs/source/changes.rst: -------------------------------------------------------------------------------- 1 | Change Log 2 | ========== 3 | 4 | **Recorder 3.0.0** Dec 27, 2024 5 | 6 | 1. Automatic Tracing Code Generation: Introduced a new tracing mechanism that employs a three-phase generic tracing wrapper for each function of interest. These tracing wrappers can be generated automatically and compiled as plugins, making it extremely easy to extend support for additional functions and I/O libraries. Added support for nearly all functions of PnetCDF and NetCDF. 7 | 2. Supporting Diverse Programs: Extend Recorder to trace a wide range of program types, including single-threaded, multi-threaded, MPI, non-MPI programs, and CUDA kernels. Recorder also captures rich metadata, such as thread IDs and function call depth, offering a deeper insight into the traced data. These features enable the tracing and analysis of I/O operations across a broader range of HPC scenarios. 8 | 3. Pattern-Recognition-Based Compression: Implemented a compression algorithm based on pattern recognition, capable of identifying and compressing recurring code patterns and common I/O patterns. This algorithm also includes an inter-process compression stage, effectively reducing redundancy across processes. 9 | 10 | **Recorder 2.4.0** Mar 24, 2023 11 | 12 | 1. Implement inter-process compression using offset pattern detection 13 | 2. Clean up code and simplify post-processing APIs 14 | 15 | **Recorder 2.3.3** Jan 21, 2022 16 | 17 | 1. Still require a RECORDER_WITH_NON_MPI hint for non-mpi programs. 18 | 2. Add a singal handler to intercept SIGTERM and SIGINT. 19 | 3. Allow setting buffer size 4. Fix timestamps in Chrome trace conversion 20 | 21 | **Recorder 2.3.2** Jan 18, 2022 22 | 23 | 1. Can handle both MPI and non-MPI programs without user hint. 24 | 2. Can handle fork() + exec() workflows. 25 | 26 | **Recorder 2.3.1** Nov 30, 2021 27 | 28 | 1. Separate MPI and MPI-IO. 29 | 2. Updated conflict detector to use the latest reader code. 30 | 31 | **Recorder 2.3.0** Sep 15, 2021 32 | 33 | 1. Adopt pilgrim copmerssion algorithm. 34 | 2. Implemented a new reader and decorder interface. 35 | 3. Support GNU ftrace functionality. 36 | 4. Support multi-threaded programs. Record has a thread id field. 37 | 5. Store records in starting timestamp order. 38 | 6. Store level information for each record. 39 | 7. Add APIs to retrive function type and function name, etc. 40 | 41 | **Recorder 2.2.1** Aug 25, 2021 42 | 43 | 1. Include the code for verifying I/O synchronizations (tools/verifyio). 44 | 2. Add support for multi-threaded programs. 45 | 46 | **Recorder 2.2.0** Jan 25, 2021 47 | 48 | 1. Add support for MPI_Cart_sub, MPI_Comm_split_type, etc. 49 | 2. Assign each MPI_Comm object a globally unique id. 50 | 51 | **Recorder 2.1.9** Jan 14, 2021 52 | 53 | 1. Clean up the code 54 | 2. Fixed a few memory leak issues 55 | 3. Add support for fseeko and ftello 56 | 57 | **Recorder 2.1.8** Dec 18, 2020 58 | 59 | 1. Add MPI_Test, MPI_Testany, MPI_Testsome and MPI_Testall 60 | 2. Add MPI_Ireduce, MPI_Iscatter, MPI_Igather and MPI_Ialltoall 61 | 3. Do not log pointers by default as it delivers not so much information 62 | 63 | **Recorder 2.1.7** Nov 11, 2020 64 | 65 | 1. Add fcntl() support. Only support commands defined in `POSIX standard `__. 66 | 2. Add support for MPI_Ibcast() 67 | 68 | **Recorder 2.1.6** Nov 05, 2020 69 | 70 | 1. Generate unique id for communicators 71 | 2. Fix bus caused by MPI_COMM_NULL 72 | 3. Add support for MPI_File_get_size 73 | 74 | **Recorder 2.1.5** Aug 27, 2020 75 | 76 | 1. Add MPI_File_seek and MPI_File_seek_shared 77 | 2. Add documentation on how to install using `Spack `__. 78 | 79 | **Recorder 2.1.4** Aug 26, 2020 80 | 81 | 1. Update LICENSE 82 | 2. Update automake/autotools files to make it ready for Spack 83 | 84 | **Recorder 2.1.3** Aug 24, 2020 85 | 86 | 1. Use autotools and automake for compilation. 87 | 2. Add support for MPI_Comm_split/MPI_Comm_dup/MPI_Comm_create 88 | 3. Store the value of MPI_Status 89 | 90 | **Recorder 2.1.2** Aug 06, 2020 91 | 92 | 1. Rewrite the reader program with C. 93 | 2. Add Python bindings to call C functions. 94 | 3. Add support for MPI_Waitall/Waitany/Waitsome and MPI_Ssend 95 | 4. Remove oft2 converter. 96 | 5. Clean up the Makefile. 97 | 98 | **Recorder 2.1.1** Jun 28, 2020 99 | 100 | 1. Use `uthash `__ library to replace the original hash map implementation 101 | 2. Remove zlib support 102 | 103 | **Recorder 2.1** May 15, 2020 104 | 105 | 1. Dump a VERSION file for the reader script to decide the trace format. 106 | 2. Include the return value in each record. 107 | 3. Implement conflict detection algorithm for commit semantics and session semantics. 108 | 109 | **Recorder 2.0.1** Nov 7, 2019 110 | 111 | 1. Implement compressed peephole encoding schema. 112 | 2. Intergrat zlib as another compression choice. 113 | 3. Users can choose compression modes by setting environment variables. 114 | 115 | **Recorder 2.0** Jul 19, 2019 116 | 117 | 1. Add the binary format output. 118 | 2. Implement a converter that can output OTF2 trace format. 119 | 3. Write a separate log unit to provide an uniform logging interface. Re-write most of the code to use this new log unit. 120 | 4. Ignore files (e.g. /sockets) that are not used by the application itself. 121 | 5. Add a built-in hashmap to support mappings from function name and filename to integers. 122 | 6. Put all function (that we plan to intercept) signatures in the same header file 123 | 124 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | 3 | # -- Project information 4 | 5 | project = 'Recorder' 6 | #copyright = '2024, Chen Wang' 7 | #author = 'Chen Wang' 8 | 9 | #release = '0.1' 10 | #version = '0.1.0' 11 | 12 | # -- General configuration 13 | 14 | extensions = [ 15 | 'sphinx.ext.duration', 16 | 'sphinx.ext.doctest', 17 | 'sphinx.ext.autodoc', 18 | 'sphinx.ext.autosummary', 19 | 'sphinx.ext.intersphinx', 20 | 'myst_parser' 21 | ] 22 | 23 | intersphinx_mapping = { 24 | 'python': ('https://docs.python.org/3/', None), 25 | 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), 26 | } 27 | intersphinx_disabled_domains = ['std'] 28 | 29 | templates_path = ['_templates'] 30 | 31 | # -- Options for HTML output 32 | 33 | #html_theme = 'sphinx_rtd_theme' 34 | html_theme = 'furo' 35 | 36 | # -- Options for EPUB output 37 | epub_show_urls = 'footnote' 38 | -------------------------------------------------------------------------------- /docs/source/features.rst: -------------------------------------------------------------------------------- 1 | Other Features 2 | =========================================== 3 | 4 | Inclusion/Exclusion prefix list 5 | ------------------------------- 6 | 7 | Many POSIX calls intercepted are made by system libraries, job 8 | schedulers, etc. Their I/O accesses are less interesting as they 9 | operates on file locations such as ``/dev, /sys, /usr/lib, etc``. To 10 | ignore those calls, you can specifiy a file that contains prefixes that 11 | you want to exclude: 12 | 13 | .. code:: bash 14 | 15 | export RECORDER_EXCLUSION_FILE=/path/to/your/exclusion_prefix_file 16 | 17 | # This file contains one prefix each line. 18 | # An example exclusion prefix file is included with Recorder: 19 | 20 | Recorder$ cat ./exclusion_prefix.txt 21 | /dev 22 | /proc 23 | /sys 24 | /etc 25 | /usr/tce/packages 26 | pipe:[ 27 | anon_inode: 28 | socket:[ 29 | 30 | Similarly, you can set ``RECORDER_INCLUSION_FILE`` to specify the 31 | inclusion prefixes, so only the POSIX calls that match those prefixes 32 | will be recorded. 33 | 34 | Note that this feature only applies to POSIX calls. MPI and HDF5 calls 35 | are always recorded when enabled. 36 | 37 | Storing pointers 38 | ---------------- 39 | 40 | Recorder by default does not log the pointers (memory addresses) as they 41 | provide little information yet cost a lot of space to store. However, 42 | you can change this behaviour by setting the enviroment variable 43 | ``RECORDER_LOG_POINTER`` to 1. 44 | 45 | Storing thread ids 46 | ------------------ 47 | 48 | Use ``RECORDER_LOG_TID``\ (0 or 1) to control whether to store thread 49 | id. Default is 0. 50 | 51 | Storing call levels 52 | ------------------- 53 | 54 | Use ``RECORDER_LOG_LEVEL`` (0 or 1) to control whether to store call 55 | levels. Default is 1. 56 | 57 | Traces location 58 | --------------- 59 | 60 | By default Recorder will output the traces to the current working 61 | directory. You can use the enviroment variable ``RECORDER_TRACES_DIR`` 62 | to specifiy the path where you want the traces stored. Make sure that 63 | every process has the persmission to write to that directory. 64 | 65 | Timestamp buffer size 66 | ----------- 67 | 68 | Timestamps are buffered internally to avoid frequent disk I/O. Use 69 | ``RECORDER_BUFFER_SIZE`` (in MB) to set the size of this buffer. The 70 | default value is 1MB. 71 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | |build| 2 | 3 | Overview 4 | ======== 5 | 6 | Recorder is a multi-level I/O tracing framework that can capture I/O function calls at multiple levels of the I/O stack, including HDF5, PnetCDF, NetCDF, MPI-IO, and POSIX I/O. Recorder requires no modification or recompilation of the application and users can control what levels are traced. 7 | 8 | Publications 9 | ------------ 10 | 11 | Recorder has undergone significant changes since the last 12 | "Recorder 2.0" paper. We have incorporated a new pattern-based 13 | compression algorithm, along with many new features. 14 | We have submitted a journal paper describing all these 15 | changes in detail. The draft version is available on `arXiv `__. 16 | 17 | `Wang, Chen, Jinghan Sun, Marc Snir, Kathryn Mohror, and Elsa 18 | Gonsiorowski. “Recorder 2.0: Efficient Parallel I/O Tracing and 19 | Analysis.” In IEEE International Workshop on High-Performance Storage 20 | (HPS), 2020. `__ 21 | 22 | 23 | I/O Patterns and consistency requirements 24 | ----------------------------------------- 25 | 26 | `Traces from 17 HPC applications `__ 27 | 28 | The traces were collected using an old version of Recorder. The current 29 | version uses a different trace format. To read those traces please use 30 | Recorder 2.2.1 from the 31 | `release `__ 32 | page. 33 | 34 | The following paper presents a comprehensive study on the I/O patterns 35 | of these applications. Additionally, it investigates the consistency 36 | requirements of these applications, confirming experimentally that 37 | POSIX consistency is rarely required by HPC applications. 38 | 39 | `Wang, Chen, Kathryn Mohror, and Marc Snir. “File System Semantics 40 | Requirements of HPC Applications.” Proceedings of the 30th International 41 | Symposium on High-Performance Parallel and Distributed Computing (HPDC). 42 | 2021. `__ 43 | 44 | 45 | .. toctree:: 46 | :hidden 47 | quickstart 48 | build 49 | usage 50 | features 51 | postprocessing 52 | changes 53 | 54 | 55 | .. |build| image:: https://github.com/uiuc-hpc/Recorder/actions/workflows/cmake.yml/badge.svg 56 | :target: https://github.com/uiuc-hpc/Recorder/actions/workflows/cmake.yml 57 | -------------------------------------------------------------------------------- /docs/source/postprocessing.rst: -------------------------------------------------------------------------------- 1 | Post-processing and Visualization 2 | ================================= 3 | 4 | 1. recorder-viz 5 | --------------- 6 | 7 | We developed a Python library, 8 | `recorder-viz `__, for 9 | post-processing and visualizations. 10 | 11 | 12 | recorder-viz can be installed by using `pip`. 13 | It relies on a few python libraries for visualization. Please install them first. 14 | 15 | .. code:: bash 16 | 17 | pip install pandas prettytable bokeh --user 18 | pip install recorder-viz --user 19 | 20 | This will install recorder-viz locally. You may need to include 21 | ``~/.local/bin`` in your ``$PATH`` enviroment. 22 | 23 | .. code:: bash 24 | 25 | export PATH=$HOME/.local/bin:$PATH 26 | 27 | recorder-viz relies on Recorder to run. Make sure ``$RECORDER_INSTALL_PATH`` 28 | points to the location where you installed Recorder. 29 | 30 | To generate a visualization report. Run: 31 | 32 | .. code:: bash 33 | 34 | recorder-report /path/to/your_trace_folder/ 35 | 36 | This will generate an html file under the current directory. Simply open 37 | it with your web broswer. Cheers! 38 | 39 | 40 | 2. Format Converters 41 | -------------------- 42 | 43 | We also provide two format converters ``recorder2parquet`` and 44 | ``recorder2timeline``. They will be placed under $RECORDER_ROOT/bin 45 | directory after installation. 46 | 47 | - ``recorder2parquet`` will convert Recorder traces into a single 48 | `Parquet `__ formata file. The Apache 49 | Parquet format is a well-known format that is supported by many 50 | analysis tools. 51 | 52 | - ``recorder2timeline`` will conver Recorder traces into 53 | `Chromium `__ 54 | trace format files. You can upload them to https://ui.perfetto.dev 55 | for an interactive visualization. 56 | 57 | 3. APIs 58 | --------- 59 | 60 | TODO: we have C APIs (tools/reader.h). Need to doc them. 61 | 62 | -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ---------- 3 | 4 | **1. Build** 5 | 6 | Recorder requires MPI and HDF5 to build. Please make sure they are installed 7 | before building Recorder. 8 | 9 | Run the following to fetch, build, and install Recorder. 10 | In this example, we will install Recorder under its srouce directory. 11 | You can change the install localtion by modifying ``$RECORDER_INSTALL_PATH`` below. 12 | 13 | .. code:: bash 14 | 15 | git clone https://github.com/uiuc-hpc/Recorder.git 16 | cd Recorder 17 | export RECORDER_INSTALL_PATH=`pwd`/install 18 | git submodule update --init --recursive 19 | mkdir build 20 | cd build 21 | cmake .. -DCMAKE_INSTALL_PREFIX=$RECORDER_INSTALL_PATH 22 | make 23 | make install 24 | 25 | 26 | **2. Trace an application** 27 | 28 | Make sure ``$RECORDER_INSTALL_PATH`` points to the location where you installed Recorder. 29 | Now let's run an application with Recorder to collect it traces. 30 | We will use an example MPI program as our targeting application. 31 | Go to the Recorder's source directory and run the following. 32 | 33 | .. code:: bash 34 | 35 | cd test && mpicc test_mpi.c -o test_mpi 36 | mpirun -np N -env LD_PRELOAD $RECORDER_INSTALL_PATH/lib/librecorder.so ./test_mpi 37 | 38 | # On HPC systems, you may need to use srun or 39 | # other job schedulers to replace mpirun, e.g., 40 | srun -n4 -N1 --overlap --export=ALL,LD_PRELOAD=$RECORDER_INSTALL_PATH/lib/librecorder.so ./test_mpi 41 | 42 | Once completed, you will see a folder named ``recorder-yyyymmdd``. This is a parent folder 43 | that contains all traces folders generated on *yyyymmdd*. 44 | Inside it, you will find the actual trace forlder of the application you just run. 45 | The folder name follows the format of ``HHmmSS.ff-hostname-username-appname-pid``. 46 | 47 | **3. Examine the traces** 48 | 49 | Recorder provides several useful tools under ``$RECORDER_INSTALL_PATH$/bin`` for analysis purposes. 50 | 51 | *recorder-summary* can be used to reports high-level statistics: 52 | 53 | .. code:: bash 54 | 55 | $RECORDER_INSTALL_PATH/bin/recorder-summary /path/to/your_trace_folder/ 56 | 57 | *recorder2text* is used to convert the Recorder-format traces to plain text files. 58 | 59 | .. code:: bash 60 | 61 | $RECORDER_INSTALL_PATH/bin/recorder2text /path/to/your_trace_folder/ 62 | 63 | This will generate text fomart traces under ``/path/to/your_trace_folder/_text``. 64 | You will see *N* [pid].txt files, where *N* is the number processors you run your application. 65 | Each of these txt files contains the traces generated by one processor indicated by the [pid]. 66 | -------------------------------------------------------------------------------- /docs/source/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | 5 | Assume ``$RECORDER_INSTALL_PATH`` is the location where you installed Recorder. 6 | 7 | Generate traces 8 | ------------------ 9 | 10 | .. code:: bash 11 | 12 | # For MPI programs 13 | mpirun -np $nprocs -env LD_PRELOAD $RECORDER_INSTALL_PATH/lib/librecorder.so ./your_app 14 | 15 | # With srun 16 | srun -n $nprocs --export=ALL,LD_PRELOAD=$RECORDER_INSTALL_PATH/lib/librecorder.so ./your_app 17 | 18 | # With FLUX 19 | flux run $nprocs --env LD_PRELOAD=RECORDER_INSTALL_PATH/lib/librecorder.so ./your_app 20 | 21 | # For non-MPI programs or programs that may spwan non-mpi children programs 22 | RECORDER_WITH_NON_MPI=1 LD_PRELOAD=$RECORDER_INSTALL_PATH/lib/librecorder.so ./your_app 23 | 24 | The trace files will be written to the current directory under a folder 25 | named ``hostname-username-appname-pid-starttime``. 26 | 27 | *Note: In some systems (e.g., Quartz at LLNL), Darshan is deployed 28 | system-widely. Recorder does not work with Darshan. Please make sure 29 | darhsn is disabled and your application is not linked with the darshan 30 | library (use ldd to check).* 31 | 32 | Configure tracing layers 33 | ------------------------ 34 | 35 | Currently, Recorder is capable of tracing POSIX, MPI, MPI-IO, HDF5, NetCDF and PnetCDF calls. 36 | By default, all supported I/O layers are enabled. 37 | At runtime (generally before running your application), you can set 38 | the following environment variables to dynamically enable/disable 39 | the tracing of certain layers. 40 | 41 | 1 = enable; 0 = disable. 42 | 43 | * export RECORDER_POSIX_TRACING=[1|0] (default: 1) 44 | 45 | * export RECORDER_MPIIO_TRACING=[1|0] (default: 1) 46 | 47 | * export RECORDER_HDF5_TRACING=[1|0] (default: 1) 48 | 49 | * export RECORDER_NETCDF_TRACING=[1|0] (default: 1) 50 | 51 | * export RECORDER_PNETCDF_TRACING=[1|0] (default: 1) 52 | 53 | * export RECORDER_MPI_TRACING=[1|0] (default: 0) 54 | 55 | 56 | Human-readable traces 57 | ------------------------ 58 | 59 | Recorder uses its own binary tracing format to compress and store 60 | traces. 61 | 62 | We provide a tool (recorder2text) that can convert the recorder format 63 | traces to plain text format. 64 | 65 | .. code:: bash 66 | 67 | $RECORDER_INSTALL_PATH/bin/recorder2text /path/to/your_trace_folder/ 68 | 69 | This will generate text fomart traces under 70 | ``/path/to/your_trace_folder/_text``. 71 | -------------------------------------------------------------------------------- /exclusion_prefix.txt: -------------------------------------------------------------------------------- 1 | /dev 2 | /proc 3 | /sys 4 | /etc 5 | /usr/tce/packages 6 | pipe:[ 7 | anon_inode: 8 | socket:[ 9 | -------------------------------------------------------------------------------- /include/recorder-cuda-profiler.h: -------------------------------------------------------------------------------- 1 | #ifndef __RECORDER_CUDA_PROFILER_H_ 2 | #define __RECORDER_CUDA_PROFILER_H_ 3 | 4 | void cuda_profiler_init(); 5 | void cuda_profiler_exit(); 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /include/recorder-pattern-recognition.h: -------------------------------------------------------------------------------- 1 | #ifndef __RECORDER_PATTERN_RECOGNITION_H_ 2 | #define __RECORDER_PATTERN_RECOGNITION_H_ 3 | #include "uthash.h" 4 | #include "recorder-logger.h" 5 | 6 | #ifndef HAVE_OFF64_T 7 | typedef int64_t off64_t; 8 | #endif 9 | 10 | typedef struct offset_map { 11 | char* func; // key 12 | off64_t offset; // value, this is the previous offset 13 | UT_hash_handle hh; 14 | } offset_map_t; 15 | 16 | /* all functions in this unit starts with iopr_ 17 | * (I/O Pattern Recognition) 18 | * 19 | */ 20 | 21 | // intraprocess 22 | off64_t iopr_intraprocess(const char* func, off64_t offset); 23 | 24 | // interprocess 25 | void iopr_interprocess(RecorderLogger* logger); 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /include/recorder-sequitur.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) by Argonne National Laboratory 3 | * See COPYRIGHT in top-level directory 4 | */ 5 | 6 | #ifndef _RECORDER_SEQUITUR_H_ 7 | #define _RECORDER_SEQUITUR_H_ 8 | 9 | #include 10 | #include "utlist.h" 11 | #include "uthash.h" 12 | 13 | #define IS_RULE_HEAD(sym) ((!(sym)->terminal) && ((sym)->rule_head==NULL)) 14 | #define IS_NONTERMINAL(sym) ((!(sym)->terminal) && ((sym)->rule_head!=NULL)) 15 | #define IS_TERMINAL(sym) ((sym)->terminal) 16 | 17 | #define ERROR_ABORT(msg) {fprintf(stderr, msg);abort();} 18 | 19 | 20 | 21 | /** 22 | * There are three types of Symbols 23 | * 24 | * 1. Terminal: 25 | * `rule` filed is the rule (rule head) it blongs to 26 | * `rule_head`, `ref_body` and `ref` are ignored 27 | * 28 | * 2. Non-terminal: 29 | * `rule` filed is the rule (rule head) it blongs to 30 | * `rule_head` points to the rule_head node 31 | * `ref_body` and `ref` are ignored 32 | * 33 | * Terminals and Non-terminals are both stored in rule_body list. 34 | * 35 | * 3. Rule head: 36 | * Represents a rule. This is a special type that are stored in the rule list 37 | * It will never be inserted into the rules body. 38 | * `rule_body` is the right hand side 39 | * `ref` is the number of usages 40 | * `rule` and `rule_head` filed are ignored 41 | * 42 | */ 43 | typedef struct Symbol_t { // utlist node, sizeof(Symbol) = 56 44 | int val; 45 | int exp; 46 | bool terminal; 47 | 48 | // For terminal and non-termial this field 49 | // remembers the rule (Symbol of Rule Head type) they belong to 50 | struct Symbol_t *rule; 51 | 52 | // Only used by non-terminals, points to the rule (Symbol of Rule Head type) it represents 53 | struct Symbol_t *rule_head; 54 | 55 | // if this is a rule (Rule Head type) 56 | // rule_body will be a list of symbols this rule represent 57 | // ref will be the number of usages of this rule 58 | struct Symbol_t *rule_body; 59 | int ref; 60 | 61 | struct Symbol_t *prev, *next; 62 | } Symbol; 63 | 64 | 65 | typedef struct Digram_t { // uthash node, sizesof(Digram) = 72 66 | void *key; // the key is composed of two symbol values (sym->val) 67 | Symbol *symbol; // first symbol of the digram 68 | UT_hash_handle hh; 69 | } Digram; 70 | 71 | typedef struct Grammar_t { 72 | Symbol *rules; 73 | Digram *digram_table; 74 | int start_rule_id; // first rule id, normally is -1 75 | int rule_id; // current_rule id, a negative number start from 'start_rule_id' 76 | bool twins_removal; // if or not we will apply the twins-removal rule 77 | } Grammar; 78 | 79 | 80 | /* Only these five functions should be exposed 81 | * to the recorder looger code. 82 | * Alls the rest are used internally for the Sequitur 83 | * algorithm implementation. 84 | */ 85 | Symbol* append_terminal(Grammar *grammar, int val, int exp); 86 | void sequitur_init(Grammar *grammar); 87 | void sequitur_init_rule_id(Grammar *grammar, int start_rule_id, bool twins_removal); 88 | void sequitur_update(Grammar *grammar, int *update_terminal_id); 89 | void sequitur_cleanup(Grammar *grammar); 90 | 91 | 92 | /* recorder_sequitur_symbol.c */ 93 | Symbol* new_symbol(int val, int exp, bool terminal, Symbol* rule_head); 94 | void symbol_put(Symbol *rule, Symbol *pos, Symbol *sym); 95 | void symbol_delete(Symbol *rule, Symbol *sym, bool deref); 96 | 97 | Symbol* new_rule(Grammar *grammar); 98 | void rule_put(Symbol **rules_head, Symbol *rule); 99 | void rule_delete(Symbol **rules_head, Symbol *rule); 100 | void rule_ref(Symbol *rule); 101 | void rule_deref(Symbol *rule); 102 | 103 | 104 | 105 | /* recorder_sequitur_digram.c */ 106 | #define DIGRAM_KEY_LEN sizeof(int)*4 107 | Symbol* digram_get(Digram *digram_table, Symbol* sym1, Symbol* sym2); 108 | int digram_put(Digram **digram_table, Symbol *symbol); 109 | int digram_delete(Digram **digram_table, Symbol *symbol); 110 | 111 | 112 | /* recorder_sequitur_logger.c */ 113 | int* serialize_grammar(Grammar *grammar, int* serialized_integers); 114 | void sequitur_save_unique_grammars(const char* path, Grammar* lg, int mpi_rank, int mpi_size); 115 | 116 | /* recorder_sequitur_utils.c */ 117 | void sequitur_print_rules(Grammar *grammar); 118 | void sequitur_print_digrams(Grammar *grammar); 119 | 120 | 121 | #endif 122 | -------------------------------------------------------------------------------- /include/recorder-timestamps.h: -------------------------------------------------------------------------------- 1 | #ifndef __RECORDER_TIMESTAMPS_H_ 2 | #define __RECORDER_TIMESTAMPS_H_ 3 | #include 4 | #include 5 | #include "recorder-logger.h" 6 | 7 | /* 8 | * get the per-rank timestamp filename 9 | */ 10 | void ts_get_filename(RecorderLogger* logger, char* ts_filename); 11 | 12 | /* 13 | * write out the current buffer to the per-rank timestamp filename 14 | */ 15 | void ts_write_out(RecorderLogger* logger); 16 | 17 | /* 18 | * merge per-rank timestamp files into a single file 19 | */ 20 | void ts_merge_files(RecorderLogger* logger); 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /include/recorder-utils.h: -------------------------------------------------------------------------------- 1 | #ifndef __RECORDER_UTILS_H_ 2 | #define __RECORDER_UTILS_H_ 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | void utils_init(); 9 | void utils_finalize(); 10 | void* recorder_malloc(size_t size); 11 | void recorder_free(void* ptr, size_t size); 12 | pthread_t recorder_gettid(void); 13 | long get_file_size(const char *filename); // return the size of a file 14 | int accept_filename(const char *filename); // if include the file in trace 15 | double recorder_wtime(void); // return the timestamp 16 | char* itoa(off64_t val); // convert an integer to string 17 | char* ftoa(double val); // convert a float to string 18 | char* ptoa(const void* ptr); // convert a pointer to string 19 | char* strtoa(const char* ptr); // convert a char* to string (safer strdup) 20 | char* arrtoa(size_t arr[], int count); // convert an array of size_t to a string 21 | char** assemble_args_list(int arg_count, ...); 22 | const char* get_function_name_by_id(int id); 23 | int get_function_id_by_name(const char* name); 24 | char* realrealpath(const char* path); // return the absolute path (mapped to id in string) 25 | int mkpath(char* file_path, mode_t mode); // recursive mkdir() 26 | 27 | 28 | // recorder send/recv/bcast only handles MPI_BYTE stream 29 | void recorder_send( void *buf, size_t count, int dst, int tag, MPI_Comm comm); 30 | void recorder_recv( void *buf, size_t count, int src, int tag, MPI_Comm comm); 31 | void recorder_bcast(void *buf, size_t count, int root, MPI_Comm comm); 32 | void recorder_barrier(MPI_Comm comm); 33 | 34 | int min_in_array(int* arr, size_t len); 35 | double recorder_log2(int val); 36 | int recorder_ceil(double val); 37 | /* 38 | * compress buf using zlib and then write to the output file 39 | * the file stream must has been opened with write permission. 40 | */ 41 | void recorder_write_zlib(unsigned char* buf, size_t buf_size, FILE* out_file); 42 | int recorder_debug_level(); 43 | 44 | #define RECORDER_LOG(level, ...) \ 45 | do { \ 46 | int debug_level = recorder_debug_level(); \ 47 | if (level <= debug_level) { \ 48 | /* we do not intercept fprintf */ \ 49 | fprintf(stderr, __VA_ARGS__); \ 50 | } \ 51 | } while (0) 52 | 53 | #define RECORDER_LOGERR(...) RECORDER_LOG(1, __VA_ARGS__) 54 | #define RECORDER_LOGINFO(...) RECORDER_LOG(2, __VA_ARGS__) 55 | #define RECORDER_LOGDBG(...) RECORDER_LOG(3, __VA_ARGS__) 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /include/recorder.h: -------------------------------------------------------------------------------- 1 | #ifndef __RECORDER_H 2 | #define __RECORDER_H 3 | 4 | #include "recorder-utils.h" 5 | #include "recorder-logger.h" 6 | #include "recorder-gotcha.h" 7 | #include "recorder-pattern-recognition.h" 8 | #include "recorder-timestamps.h" 9 | 10 | /* List of runtime environment variables */ 11 | #define RECORDER_WITH_NON_MPI "RECORDER_WITH_NON_MPI" 12 | #define RECORDER_TRACES_DIR "RECORDER_TRACES_DIR" 13 | #define RECORDER_TIME_RESOLUTION "RECORDER_TIME_RESOLUTION" 14 | #define RECORDER_TIME_COMPRESSION "RECORDER_TIME_COMPRESSION" 15 | #define RECORDER_STORE_POINTER "RECORDER_STORE_POINTER" 16 | #define RECORDER_STORE_TID "RECORDER_STORE_TID" 17 | #define RECORDER_STORE_CALL_DEPTH "RECORDER_STORE_CALL_DEPTH" 18 | #define RECORDER_INTERPROCESS_COMPRESSION "RECORDER_INTERPROCESS_COMPRESSION" 19 | #define RECORDER_INTERPROCESS_PATTERN_RECOGNITION "RECORDER_INTERPROCESS_PATTERN_RECOGNITION" 20 | #define RECORDER_INTRAPROCESS_PATTERN_RECOGNITION "RECORDER_INTRAPROCESS_PATTERN_RECOGNITION" 21 | #define RECORDER_EXCLUSION_FILE "RECORDER_EXCLUSION_FILE" 22 | #define RECORDER_INCLUSION_FILE "RECORDER_INCLUSION_FILE" 23 | #define RECORDER_DEBUG_LEVEL "RECORDER_DEBUG_LEVEL" 24 | 25 | /* 26 | * Allowing users to exclude the interception 27 | * of certain layers at runtime. 28 | * 29 | * e.g., export RECORDER_MPIIO_TRACING=0 30 | */ 31 | #define RECORDER_POSIX_TRACING "RECORDER_POSIX_TRACING" 32 | #define RECORDER_MPIIO_TRACING "RECORDER_MPIIO_TRACING" 33 | #define RECORDER_MPI_TRACING "RECORDER_MPI_TRACING" 34 | #define RECORDER_HDF5_TRACING "RECORDER_HDF5_TRACING" 35 | #define RECORDER_PNETCDF_TRACING "RECORDER_PNETCDF_TRACING" 36 | #define RECORDER_NETCDF_TRACING "RECORDER_NETCDF_TRACING" 37 | 38 | 39 | /** 40 | * I/O Interceptor 41 | * Phase 1: 42 | * 43 | * we intercept functions (e.g., from recorder-posix.c) and then 44 | * call this interception funciton. 45 | * 46 | * Here, we first run the original function so we can get the ouput 47 | * parameters correctly. 48 | * 49 | * We also construct a [struct Record] for each function. But later we 50 | * can change the fields, e.g., fopen will convert the FILE* to an integer res. 51 | * 52 | */ 53 | #define RECORDER_INTERCEPTOR_PROLOGUE_CORE(ret, func, real_args) \ 54 | Record *record = recorder_malloc(sizeof(Record)); \ 55 | record->func_id = get_function_id_by_name(#func); \ 56 | record->tid = recorder_gettid(); \ 57 | logger_record_enter(record); \ 58 | record->tstart = recorder_wtime(); \ 59 | GOTCHA_SET_REAL_CALL_NOCHECK(func); \ 60 | ret res = GOTCHA_REAL_CALL(func) real_args ; \ 61 | record->tend = recorder_wtime(); \ 62 | record->res = NULL; \ 63 | if (sizeof(ret)) { \ 64 | record->res = malloc(sizeof(ret)); \ 65 | memcpy(record->res, &res, sizeof(ret)); \ 66 | } 67 | 68 | // Fortran wrappers call this 69 | // ierr is of type MPI_Fint*, set only for fortran calls 70 | #define RECORDER_INTERCEPTOR_PROLOGUE_F(ret, func, real_args, ierr) \ 71 | if(!logger_initialized()) { \ 72 | GOTCHA_SET_REAL_CALL_NOCHECK(func); \ 73 | ret res = GOTCHA_REAL_CALL(func) real_args ; \ 74 | if ((ierr) != NULL) { *(ierr) = res; } \ 75 | return res; \ 76 | } \ 77 | RECORDER_INTERCEPTOR_PROLOGUE_CORE(ret, func, real_args) \ 78 | if ((ierr) != NULL) { *(ierr) = res; } 79 | 80 | // C wrappers call this 81 | #define RECORDER_INTERCEPTOR_PROLOGUE(ret, func, real_args) \ 82 | /*RECORDER_LOGINFO("[Recorder] intercept %s\n", #func);*/ \ 83 | if(!logger_initialized()) { \ 84 | GOTCHA_SET_REAL_CALL_NOCHECK(func); \ 85 | ret res = GOTCHA_REAL_CALL(func) real_args ; \ 86 | return res; \ 87 | } \ 88 | RECORDER_INTERCEPTOR_PROLOGUE_CORE(ret, func, real_args) 89 | 90 | /** 91 | * I/O Interceptor 92 | * Phase 2: 93 | * 94 | * Set other fields of the record, i.e, arg_count and args. 95 | * Finally write out the record 96 | * 97 | */ 98 | #define RECORDER_INTERCEPTOR_EPILOGUE(record_arg_count, record_args) \ 99 | record->arg_count = record_arg_count; \ 100 | record->args = record_args; \ 101 | logger_record_exit(record); \ 102 | return res; 103 | 104 | #endif /* __RECORDER_H */ 105 | -------------------------------------------------------------------------------- /lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | # Include source and build directories 3 | #------------------------------------------------------------------------------ 4 | include_directories(${CMAKE_SOURCE_DIR}/include) 5 | include_directories(${CMAKE_SOURCE_DIR}/deps/GOTCHA/include) 6 | 7 | # set(CMAKE_POSITION_INDEPENDENT_CODE ON) 8 | 9 | #------------------------------------------------------------------------------ 10 | # Set sources 11 | #------------------------------------------------------------------------------ 12 | set(RECORDER_SRCS 13 | ${CMAKE_SOURCE_DIR}/include/recorder.h 14 | ${CMAKE_SOURCE_DIR}/include/recorder-sequitur.h 15 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-hdf5.c 16 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-cst-cfg.c 17 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-mpi.c 18 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-init-finalize.c 19 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-posix.c 20 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-utils.c 21 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-logger.c 22 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-gotcha.c 23 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-function-profiler.c 24 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-pattern-recognition.c 25 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-timestamps.c 26 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-sequitur.c 27 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-sequitur-symbol.c 28 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-sequitur-digram.c 29 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-sequitur-logger.c 30 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-sequitur-utils.c) 31 | 32 | 33 | 34 | #------------------------------------------------------------------------------ 35 | # External dependencies 36 | #------------------------------------------------------------------------------ 37 | find_package(HDF5 REQUIRED) 38 | if(HDF5_FOUND) 39 | include_directories(${HDF5_INCLUDE_DIRS}) 40 | set(RECORDER_EXT_INCLUDE_DEPENDENCIES ${HDF5_INCLUDE_DIRS} 41 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES}) 42 | set(RECORDER_EXT_LIB_DEPENDENCIES 43 | ${HDF5_LIBRARIES} ${RECORDER_EXT_LIB_DEPENDENCIES}) 44 | else() 45 | message(STATUS, "HDF5 not found") 46 | endif() 47 | 48 | # PnetCDF is not a CMAKE project, we use find_path() to locate it 49 | # this requires users to set -DRECORDER_WITH_PNETCDF=/xx/xx/ 50 | find_path(PNETCDF_INCLUDE_DIRS pnetcdf.h ${RECORDER_WITH_PNETCDF}/include) 51 | find_path(PNETCDF_LIBRARY_FOUND libpnetcdf.so ${RECORDER_WITH_PNETCDF}/lib) 52 | if(PNETCDF_INCLUDE_DIRS AND PNETCDF_LIBRARY_FOUND) 53 | message("-- " "Found PnetCDF:" ${RECORDER_WITH_PNETCDF}) 54 | set(RECORDER_SRCS 55 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-pnetcdf.c 56 | ${RECORDER_SRCS} 57 | ) 58 | include_directories(${PNETCDF_INCLUDE_DIRS}) 59 | set(PNETCDF_LIBRARIES ${RECORDER_WITH_PNETCDF}/lib/libpnetcdf.so) 60 | set(RECORDER_EXT_INCLUDE_DEPENDENCIES ${PNETCDF_INCLUDE_DIRS} 61 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES}) 62 | set(RECORDER_EXT_LIB_DEPENDENCIES 63 | ${PNETCDF_LIBRARIES} ${RECORDER_EXT_LIB_DEPENDENCIES}) 64 | else() 65 | message("-- " "PnetCDF not found. Will build Recorder without it.") 66 | endif() 67 | 68 | # Look for NetCDF 69 | # -DRECORDER_WITH_NETCDF=/xx/xx/ 70 | find_path(NETCDF_INCLUDE_DIRS netcdf.h ${RECORDER_WITH_NETCDF}/include) 71 | find_path(NETCDF_LIBRARY_FOUND libnetcdf.so ${RECORDER_WITH_NETCDF}/lib) 72 | if(NETCDF_INCLUDE_DIRS AND NETCDF_LIBRARY_FOUND) 73 | message("-- " "Found NetCDF:" ${RECORDER_WITH_NETCDF}) 74 | set(RECORDER_SRCS 75 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-netcdf.c 76 | ${RECORDER_SRCS} 77 | ) 78 | include_directories(${NETCDF_INCLUDE_DIRS}) 79 | set(NETCDF_LIBRARIES ${RECORDER_WITH_NETCDF}/lib/libnetcdf.so) 80 | set(RECORDER_EXT_INCLUDE_DEPENDENCIES ${NETCDF_INCLUDE_DIRS} 81 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES}) 82 | set(RECORDER_EXT_LIB_DEPENDENCIES 83 | ${NETCDF_LIBRARIES} ${RECORDER_EXT_LIB_DEPENDENCIES}) 84 | else() 85 | message("-- " "NetCDF not found. Will build Recorder without it.") 86 | endif() 87 | 88 | 89 | find_package(MPI REQUIRED) 90 | if(MPI_FOUND) 91 | include_directories(${MPI_CXX_INCLUDE_DIRS}) 92 | set(RECORDER_EXT_INCLUDE_DEPENDENCIES ${MPI_C_INCLUDE_DIRS} 93 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES}) 94 | set(RECORDER_EXT_LIB_DEPENDENCIES 95 | ${MPI_C_LIBRARIES} ${RECORDER_EXT_LIB_DEPENDENCIES}) 96 | else() 97 | message(STATUS, "MPI not found") 98 | endif() 99 | 100 | find_package(ZLIB REQUIRED) 101 | if(ZLIB_FOUND) 102 | include_directories(${ZLIB_INCLUDE_DIRS}) 103 | set(RECORDER_EXT_INCLUDE_DEPENDENCIES ${ZLIB_INCLUDE_DIRS} 104 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES}) 105 | set(RECORDER_EXT_LIB_DEPENDENCIES 106 | ${ZLIB_LIBRARIES} ${RECORDER_EXT_LIB_DEPENDENCIES}) 107 | else() 108 | message(STATUS, "ZLIB not found") 109 | endif() 110 | 111 | 112 | if(RECORDER_ENABLE_CUDA_TRACE) 113 | find_package(CUDA REQUIRED) 114 | find_package(CUDAToolkit REQUIRED) 115 | if(CUDAToolkit_FOUND) 116 | set(RECORDER_SRCS 117 | ${CMAKE_CURRENT_SOURCE_DIR}/recorder-cuda-profiler.c 118 | ${RECORDER_SRCS} 119 | ) 120 | 121 | message("-- " "Found CUDA: TRUE") 122 | include_directories(${CUDA_INCLUDE_DIRS}) 123 | set(RECORDER_EXT_INCLUDE_DEPENDENCIES 124 | ${CUDA_INCLUDE_DIRS} ${RECORDER_EXT_INCLUDE_DEPENDENCIES}) 125 | set(RECORDER_EXT_LIB_DEPENDENCIES 126 | ${CUDA_LIBRARIES} ${RECORDER_EXT_LIB_DEPENDENCIES}) 127 | set(RECORDER_EXT_LIB_DEPENDENCIES 128 | ${CUDA_cupti_LIBRARY} ${RECORDER_EXT_LIB_DEPENDENCIES}) 129 | else() 130 | message("-- " "Found CUDA: FALSE") 131 | endif() 132 | endif() 133 | 134 | 135 | INCLUDE(CheckFunctionExists) 136 | CHECK_FUNCTION_EXISTS(__xstat HAVE___XSTAT) 137 | CHECK_FUNCTION_EXISTS(__xstat64 HAVE___XSTAT64) 138 | CHECK_FUNCTION_EXISTS(__lxstat HAVE___LXSTAT) 139 | CHECK_FUNCTION_EXISTS(__lxstat64 HAVE___LXSTAT64) 140 | CHECK_FUNCTION_EXISTS(__fxstat HAVE___FXSTAT) 141 | CHECK_FUNCTION_EXISTS(__fxstat64 HAVE___FXSTAT64) 142 | 143 | 144 | #------------------------------------------------------------------------------ 145 | # Libraries 146 | #------------------------------------------------------------------------------ 147 | set(RECORDER_BUILD_INCLUDE_DEPENDENCIES 148 | ${CMAKE_CURRENT_SOURCE_DIR} 149 | ${CMAKE_CURRENT_BINARY_DIR} 150 | ) 151 | add_library(recorder ${RECORDER_SRCS}) 152 | #set_property(TARGET recorder PROPERTY POSITION_INDEPENDENT_CODE ON) 153 | target_include_directories(recorder 154 | PUBLIC "$" 155 | $ 156 | ) 157 | 158 | target_link_libraries(recorder 159 | PUBLIC gotcha # this must match add_library(target) in GOTCHA project, 160 | PUBLIC ${RECORDER_EXT_LIB_DEPENDENCIES} 161 | PUBLIC pthread 162 | ) 163 | 164 | target_compile_definitions(recorder 165 | PUBLIC _LARGEFILE64_SOURCE 166 | PUBLIC RECORDER_PRELOAD 167 | PRIVATE $<$:HAVE___XSTAT> 168 | PRIVATE $<$:HAVE___XSTAT64> 169 | PRIVATE $<$:HAVE___LXSTAT> 170 | PRIVATE $<$:HAVE___LXSTAT64> 171 | PRIVATE $<$:HAVE___FXSTAT> 172 | PRIVATE $<$:HAVE___FXSTAT64> 173 | PRIVATE $<$,$>:RECORDER_WITH_PNETCDF> 174 | PRIVATE $<$,$>:RECORDER_WITH_NETCDF> 175 | PRIVATE $<$:RECORDER_ENABLE_FCNTL_TRACE> 176 | PRIVATE $<$:RECORDER_ENABLE_CUDA_TRACE> 177 | ) 178 | 179 | recorder_set_lib_options(recorder "recorder" ${RECORDER_LIBTYPE}) 180 | 181 | #----------------------------------------------------------------------------- 182 | # Specify project header files to be installed 183 | #----------------------------------------------------------------------------- 184 | set(RECORDER_HEADERS 185 | ${CMAKE_SOURCE_DIR}/include/recorder.h 186 | ${CMAKE_SOURCE_DIR}/include/recorder-logger.h 187 | ${CMAKE_SOURCE_DIR}/include/recorder-utils.h 188 | ${CMAKE_SOURCE_DIR}/include/uthash.h 189 | ) 190 | 191 | #----------------------------------------------------------------------------- 192 | # Add file(s) to CMake Install 193 | #----------------------------------------------------------------------------- 194 | install( 195 | FILES 196 | ${RECORDER_HEADERS} 197 | DESTINATION 198 | ${RECORDER_INSTALL_INCLUDE_DIR} 199 | COMPONENT 200 | headers 201 | ) 202 | 203 | #----------------------------------------------------------------------------- 204 | # Add Target(s) to CMake Install 205 | #----------------------------------------------------------------------------- 206 | install( 207 | TARGETS 208 | recorder 209 | EXPORT 210 | ${RECORDER_EXPORTED_TARGETS} 211 | LIBRARY DESTINATION ${RECORDER_INSTALL_LIB_DIR} 212 | ARCHIVE DESTINATION ${RECORDER_INSTALL_LIB_DIR} 213 | RUNTIME DESTINATION ${RECORDER_INSTALL_BIN_DIR} 214 | ) 215 | 216 | #----------------------------------------------------------------------------- 217 | # Add Target(s) to CMake Install for import into other projects 218 | #----------------------------------------------------------------------------- 219 | install( 220 | EXPORT 221 | ${RECORDER_EXPORTED_TARGETS} 222 | DESTINATION 223 | ${RECORDER_INSTALL_DATA_DIR}/cmake/recorder 224 | FILE 225 | ${RECORDER_EXPORTED_TARGETS}.cmake 226 | ) 227 | 228 | #----------------------------------------------------------------------------- 229 | # Export all exported targets to the build tree for use by parent project 230 | #----------------------------------------------------------------------------- 231 | if(NOT RECORDER_EXTERNALLY_CONFIGURED) 232 | EXPORT ( 233 | TARGETS 234 | ${RECORDER_EXPORTED_LIBS} 235 | FILE 236 | ${RECORDER_EXPORTED_TARGETS}.cmake 237 | ) 238 | endif() 239 | 240 | #------------------------------------------------------------------------------ 241 | # Set variables for parent scope 242 | #------------------------------------------------------------------------------ 243 | # Used by config.cmake.build.in and Testing 244 | set(RECORDER_INCLUDES_BUILD_TIME 245 | ${CMAKE_CURRENT_SOURCE_DIR} 246 | ${CMAKE_CURRENT_BINARY_DIR} 247 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES} 248 | PARENT_SCOPE 249 | ) 250 | 251 | # Used by config.cmake.install.in 252 | set(RECORDER_INCLUDES_INSTALL_TIME 253 | ${RECORDER_INSTALL_INCLUDE_DIR} 254 | ${RECORDER_EXT_INCLUDE_DEPENDENCIES} 255 | PARENT_SCOPE 256 | ) 257 | -------------------------------------------------------------------------------- /lib/recorder-function-profiler.c: -------------------------------------------------------------------------------- 1 | #define _XOPEN_SOURCE 500 2 | #define _GNU_SOURCE 3 | 4 | #include 5 | #include 6 | #include "recorder.h" 7 | #include "utlist.h" 8 | #include "uthash.h" 9 | 10 | 11 | typedef struct tstart_node { 12 | double tstart; 13 | struct tstart_node *next; 14 | } tstart_node_t; 15 | 16 | typedef struct func_hash { 17 | int key_len; 18 | void *key; // thread id + func addr as key 19 | tstart_node_t *tstart_head; 20 | UT_hash_handle hh; 21 | } func_hash_t; 22 | 23 | 24 | func_hash_t* func_table; 25 | 26 | 27 | void* compose_func_hash_key(void* func, int *key_len) { 28 | pthread_t tid = recorder_gettid(); 29 | *key_len = sizeof(pthread_t) + sizeof(void*); 30 | void* key = recorder_malloc(*key_len); 31 | memcpy(key, &tid, sizeof(pthread_t)); 32 | memcpy(key+sizeof(pthread_t), &func, sizeof(void*)); 33 | return key; 34 | } 35 | 36 | 37 | void __cyg_profile_func_enter(void *func, void *caller) 38 | __attribute__((no_instrument_function)); 39 | 40 | void __cyg_profile_func_enter (void *func, void *caller) 41 | { 42 | if(!logger_initialized()) return; 43 | 44 | Dl_info info; 45 | if(!dladdr(func, &info)) return; 46 | if(!info.dli_sname && !info.dli_fname) return; 47 | 48 | //printf("enter %s %s\n", info.dli_fname, info.dli_sname); 49 | 50 | func_hash_t *entry = NULL; 51 | int key_len; 52 | void* key = compose_func_hash_key(func, &key_len); 53 | HASH_FIND(hh, func_table, key, key_len, entry); 54 | if(entry) { 55 | recorder_free(key, key_len); 56 | } else { 57 | entry = recorder_malloc(sizeof(func_hash_t)); 58 | entry->key = key; 59 | entry->key_len = key_len; 60 | entry->tstart_head = NULL; 61 | HASH_ADD_KEYPTR(hh, func_table, entry->key, entry->key_len, entry); 62 | } 63 | 64 | tstart_node_t *node = recorder_malloc(sizeof(tstart_node_t)); 65 | node->tstart = recorder_wtime(); 66 | LL_PREPEND(entry->tstart_head, node); 67 | } 68 | 69 | 70 | void __cyg_profile_func_exit(void *func, void *caller) 71 | __attribute__((no_instrument_function)); 72 | 73 | 74 | void __cyg_profile_func_exit (void *func, void *caller) 75 | { 76 | if(!logger_initialized()) return; 77 | 78 | Dl_info info; 79 | if(!dladdr(func, &info)) return; 80 | if(!info.dli_sname && !info.dli_fname) return; 81 | 82 | func_hash_t *entry = NULL; 83 | int key_len; 84 | void* key = compose_func_hash_key(func, &key_len); 85 | HASH_FIND(hh, func_table, key, key_len, entry); 86 | recorder_free(key, key_len); 87 | 88 | if(entry) { 89 | Record *record = recorder_malloc(sizeof(Record)); 90 | record->func_id = RECORDER_USER_FUNCTION; 91 | record->call_depth = 0; 92 | record->tid = recorder_gettid(); 93 | record->tstart = entry->tstart_head->tstart; 94 | record->tend = recorder_wtime(); 95 | record->arg_count = 2; 96 | record->args = recorder_malloc(record->arg_count*sizeof(char*)); 97 | record->args[0] = strdup(info.dli_fname?info.dli_fname:"???"); 98 | record->args[1] = strdup(info.dli_sname?info.dli_sname:"???"); 99 | 100 | LL_DELETE(entry->tstart_head, entry->tstart_head); 101 | write_record(record); 102 | 103 | if(entry->tstart_head == NULL) { 104 | HASH_DEL(func_table, entry); 105 | recorder_free(entry->key, entry->key_len); 106 | recorder_free(entry, sizeof(func_hash_t)); 107 | } 108 | 109 | free_record(record); 110 | } else { 111 | // Shouldn't be possible 112 | printf("Not possible!\n"); 113 | } 114 | } 115 | 116 | -------------------------------------------------------------------------------- /lib/recorder-init-finalize.c: -------------------------------------------------------------------------------- 1 | #define _XOPEN_SOURCE 500 2 | #define _POSIX_C_SOURCE 200809L 3 | #define _GNU_SOURCE /* for RTLD_NEXT */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "mpi.h" 15 | #include "recorder.h" 16 | #include "recorder-gotcha.h" 17 | 18 | 19 | static double local_tstart, local_tend; 20 | static int rank, nprocs; 21 | 22 | 23 | void signal_handler(int sig); 24 | 25 | /** 26 | * First we will intercept the GNU constructor, 27 | * where we perform recorder_init(). 28 | * 29 | * If this is an MPI program, then later we will intercept 30 | * one of MPI_Init* call, where we update the mpi info 31 | * using update_mpi_info(). Only in that function, we actually 32 | * create the log directory. 33 | * 34 | * If this is not an MPI program, then we create the log 35 | * directory at the first flush time in recorder-logger.c 36 | */ 37 | void recorder_init() { 38 | 39 | // avoid double init; 40 | if (logger_initialized()) return; 41 | 42 | /* 43 | signal(SIGSEGV, signal_handler); 44 | signal(SIGINT, signal_handler); 45 | signal(SIGTERM, signal_handler); 46 | */ 47 | 48 | gotcha_init(); 49 | logger_init(); 50 | utils_init(); 51 | 52 | local_tstart = recorder_wtime(); 53 | RECORDER_LOGDBG("[Recorder] recorder initialized.\n"); 54 | } 55 | 56 | void update_mpi_info() { 57 | 58 | GOTCHA_SET_REAL_CALL(MPI_Comm_size, RECORDER_MPI); 59 | GOTCHA_SET_REAL_CALL(MPI_Comm_rank, RECORDER_MPI); 60 | 61 | int mpi_initialized = 0; 62 | PMPI_Initialized(&mpi_initialized); // we do not intercept MPI_Initialized() call. 63 | 64 | rank = 0; 65 | nprocs = 1; 66 | if(mpi_initialized) { 67 | GOTCHA_REAL_CALL(MPI_Comm_rank)(MPI_COMM_WORLD, &rank); 68 | GOTCHA_REAL_CALL(MPI_Comm_size)(MPI_COMM_WORLD, &nprocs); 69 | } 70 | 71 | logger_set_mpi_info(rank, nprocs); 72 | } 73 | 74 | void recorder_finalize() { 75 | 76 | // check if already finialized 77 | if (!logger_initialized()) return; 78 | 79 | logger_finalize(); 80 | utils_finalize(); 81 | 82 | local_tend = recorder_wtime(); 83 | 84 | if (rank == 0) { 85 | RECORDER_LOGINFO("[Recorder] elapsed time: %.2f\n", local_tend-local_tstart); 86 | } 87 | } 88 | 89 | int MPI_Init(int *argc, char ***argv) { 90 | RECORDER_LOGDBG("[Recorder] MPI_Init\n"); 91 | int ret = PMPI_Init(argc, argv); 92 | recorder_init(); 93 | update_mpi_info(); 94 | return ret; 95 | } 96 | 97 | void mpi_init_(MPI_Fint* ierr) { 98 | RECORDER_LOGDBG("[Recorder] mpi_init_\n"); 99 | int argc = 0; 100 | char** argv = NULL; 101 | int ret = PMPI_Init(&argc, &argv); 102 | recorder_init(); 103 | update_mpi_info(); 104 | *ierr = (MPI_Fint)ret; 105 | } 106 | 107 | int MPI_Init_thread(int *argc, char ***argv, int required, int *provided) { 108 | RECORDER_LOGDBG("[Recorder] MPI_Init_thread\n"); 109 | int ret = PMPI_Init_thread(argc, argv, required, provided); 110 | recorder_init(); 111 | update_mpi_info(); 112 | return ret; 113 | } 114 | 115 | void mpi_init_thread_(MPI_Fint* required, MPI_Fint* provided, MPI_Fint* ierr) { 116 | RECORDER_LOGDBG("[Recorder] mpi_init_thread_\n"); 117 | int argc = 0; 118 | char** argv = NULL; 119 | int ret = PMPI_Init_thread(&argc, &argv, *((int*)required), provided); 120 | recorder_init(); 121 | update_mpi_info(); 122 | *ierr = (MPI_Fint)ret; 123 | } 124 | 125 | int MPI_Finalize(void) { 126 | RECORDER_LOGDBG("[Recorder] MPI_Finalize\n"); 127 | recorder_finalize(); 128 | return PMPI_Finalize(); 129 | } 130 | 131 | void mpi_finalize_(MPI_Fint* ierr) { 132 | RECORDER_LOGDBG("[Recorder] mpi_finalize_\n"); 133 | recorder_finalize(); 134 | *ierr = 0; 135 | } 136 | 137 | 138 | 139 | #ifdef __GNUC__ 140 | 141 | /** 142 | * Handle non mpi programs 143 | */ 144 | void __attribute__((constructor)) no_mpi_init() { 145 | char* with_non_mpi = getenv(RECORDER_WITH_NON_MPI); 146 | if(with_non_mpi && atoi(with_non_mpi) == 1) { 147 | recorder_init(); 148 | } 149 | } 150 | 151 | void __attribute__((destructor)) no_mpi_finalize() { 152 | char* with_non_mpi = getenv(RECORDER_WITH_NON_MPI); 153 | if(with_non_mpi && atoi(with_non_mpi) == 1) { 154 | recorder_finalize(); 155 | } 156 | } 157 | 158 | #endif 159 | 160 | 161 | void signal_handler(int sig) { 162 | /* 163 | * print backtrace for debug 164 | void *array[20]; 165 | size_t size; 166 | size = backtrace(array, 20); 167 | RECORDER_LOGERR("Error: signal %d:\n", sig); 168 | backtrace_symbols_fd(array, size, STDOUT_FILENO); 169 | exit(1); 170 | */ 171 | 172 | if(rank == 0) 173 | printf("[Recorder] signal [%s] captured, finalize now.\n", strsignal(sig)); 174 | recorder_finalize(); 175 | } 176 | -------------------------------------------------------------------------------- /lib/recorder-pattern-recognition.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "uthash.h" 8 | #include "recorder.h" 9 | #include "recorder-pattern-recognition.h" 10 | 11 | struct offset_cs_entry { 12 | int offset_key_start; 13 | int offset_key_end; 14 | CallSignature* cs; 15 | }; 16 | 17 | static offset_map_t* func2offset_map; 18 | 19 | 20 | off64_t iopr_intraprocess(const char *func, off64_t offset) { 21 | offset_map_t* entry = NULL; 22 | HASH_FIND_STR(func2offset_map, func, entry); 23 | if (!entry) { 24 | entry = (offset_map_t*) malloc(sizeof(offset_map_t)); 25 | entry->func = strdup(func); 26 | entry->offset = offset; 27 | HASH_ADD_STR(func2offset_map, func, entry); 28 | return offset; 29 | } else { 30 | off64_t delta = offset - entry->offset; 31 | entry->offset = offset; 32 | return delta; 33 | } 34 | } 35 | 36 | int count_function(RecorderLogger *logger, int filter_func_id) { 37 | int func_count = 0; 38 | 39 | CallSignature *entry, *tmp; 40 | HASH_ITER(hh, logger->cst, entry, tmp) { 41 | void* ptr = entry->key+sizeof(pthread_t); 42 | int func_id; 43 | memcpy(&func_id, ptr, sizeof(func_id)); 44 | if(func_id == filter_func_id) 45 | func_count++; 46 | } 47 | 48 | return func_count; 49 | } 50 | 51 | void iopr_interprocess_by_func(RecorderLogger *logger, char* func_name, int offset_arg_idx) { 52 | 53 | int filter_func_id = get_function_id_by_name(func_name); 54 | int func_count = count_function(logger, filter_func_id); 55 | 56 | struct offset_cs_entry *offset_cs_entries = malloc(sizeof(struct offset_cs_entry) * func_count); 57 | long int *offsets = malloc(sizeof(long int) * func_count); 58 | 59 | int args_start = cs_key_args_start(); 60 | 61 | int idx = 0; 62 | CallSignature* entry; 63 | CallSignature* tmp; 64 | HASH_ITER(hh, logger->cst, entry, tmp) { 65 | void* ptr = entry->key+sizeof(pthread_t); 66 | 67 | int func_id; 68 | memcpy(&func_id, ptr, sizeof(func_id)); 69 | 70 | if(func_id == filter_func_id) { 71 | 72 | char* key = (char*) entry->key; 73 | 74 | int arg_idx = 0; 75 | int start = args_start, end = args_start; 76 | 77 | for(int i = args_start; i < entry->key_len; i++) { 78 | if(key[i] == ' ') { 79 | if(arg_idx == offset_arg_idx) { 80 | end = i; 81 | break; 82 | } else { 83 | arg_idx++; 84 | start = i; 85 | } 86 | } 87 | } 88 | 89 | assert(end > start); 90 | char offset_str[64] = {0}; 91 | memcpy(offset_str, key+start+1, end-start-1); 92 | long int offset = atol(offset_str); 93 | 94 | offsets[idx] = offset; 95 | offset_cs_entries[idx].offset_key_start = start; 96 | offset_cs_entries[idx].offset_key_end = end; 97 | offset_cs_entries[idx].cs = entry; 98 | 99 | idx++; 100 | } 101 | } 102 | 103 | GOTCHA_SET_REAL_CALL(MPI_Comm_split, RECORDER_MPI); 104 | GOTCHA_SET_REAL_CALL(MPI_Comm_size, RECORDER_MPI); 105 | GOTCHA_SET_REAL_CALL(MPI_Comm_rank, RECORDER_MPI); 106 | GOTCHA_SET_REAL_CALL(MPI_Comm_free, RECORDER_MPI); 107 | GOTCHA_SET_REAL_CALL(MPI_Allgather, RECORDER_MPI); 108 | 109 | MPI_Comm comm; 110 | int comm_size, comm_rank; 111 | GOTCHA_REAL_CALL(MPI_Comm_split)(MPI_COMM_WORLD, func_count, logger->rank, &comm); 112 | GOTCHA_REAL_CALL(MPI_Comm_size)(comm, &comm_size); 113 | GOTCHA_REAL_CALL(MPI_Comm_rank)(comm, &comm_rank); 114 | 115 | if(comm_rank == 0) 116 | RECORDER_LOGDBG("%s count: %d, comm size: %d\n", func_name, func_count, comm_size); 117 | 118 | if(comm_size > 2) { 119 | long int *all_offsets = calloc(comm_size*(func_count), sizeof(long int)); 120 | GOTCHA_REAL_CALL(MPI_Allgather)(offsets, func_count, MPI_LONG, 121 | all_offsets, func_count, MPI_LONG, comm); 122 | 123 | // Fory every offset of the same I/O call 124 | // check if it is the form of offset = a * rank + b; 125 | for(int i = 0; i < func_count; i++) { 126 | 127 | long int o1 = all_offsets[i]; 128 | long int o2 = all_offsets[i+func_count]; 129 | long int a = o2 - o1; 130 | long int b = o1; 131 | int same_pattern = 1; 132 | for(int r = 0; r < comm_size; r++) { 133 | long int o = all_offsets[i+func_count*r]; 134 | if(o != a*r+b) { 135 | same_pattern = 0; 136 | break; 137 | } 138 | } 139 | 140 | // Everyone has the same pattern of offset 141 | // Then modify the call signature to store 142 | // the pattern instead of the actuall offset 143 | // TODO we should store a and b, but now we 144 | // store a only 145 | if(same_pattern) { 146 | HASH_DEL(logger->cst, offset_cs_entries[i].cs); 147 | 148 | int start = offset_cs_entries[i].offset_key_start; 149 | int end = offset_cs_entries[i].offset_key_end; 150 | 151 | char* tmp = calloc(64, 1); 152 | sprintf(tmp, "%ld*r+%ld", a, b); 153 | 154 | if(comm_rank == 0) 155 | RECORDER_LOGDBG("pattern recognized %d: offset = %ld*rank+%ld\n", offset_cs_entries[i].cs->terminal_id, a, b); 156 | 157 | int old_keylen = offset_cs_entries[i].cs->key_len; 158 | int new_keylen = old_keylen - (end-start-1) + strlen(tmp); 159 | int new_arg_strlen = new_keylen - args_start; 160 | 161 | void* newkey = malloc(new_keylen); 162 | void* oldkey = offset_cs_entries[i].cs->key; 163 | 164 | memcpy(newkey, oldkey, start+1); 165 | memcpy(newkey+args_start-sizeof(int), &new_arg_strlen, sizeof(int)); 166 | memcpy(newkey+start+1, tmp, strlen(tmp)); 167 | memcpy(newkey+start+1+strlen(tmp), oldkey+end, old_keylen-end); 168 | 169 | offset_cs_entries[i].cs->key = newkey; 170 | offset_cs_entries[i].cs->key_len = new_keylen; 171 | HASH_ADD_KEYPTR(hh, logger->cst, offset_cs_entries[i].cs->key, offset_cs_entries[i].cs->key_len, offset_cs_entries[i].cs); 172 | 173 | 174 | free(oldkey); 175 | free(tmp); 176 | } 177 | } 178 | free(all_offsets); 179 | } 180 | 181 | GOTCHA_REAL_CALL(MPI_Comm_free)(&comm); 182 | free(offsets); 183 | free(offset_cs_entries); 184 | } 185 | 186 | void iopr_interprocess(RecorderLogger *logger) { 187 | 188 | // Non-MPI programs 189 | // no need for interprocess pattern recognition 190 | int mpi_initialized; 191 | PMPI_Initialized(&mpi_initialized); 192 | if (!mpi_initialized) 193 | return; 194 | 195 | iopr_interprocess_by_func(logger, "lseek", 1); 196 | iopr_interprocess_by_func(logger, "lseek64", 1); 197 | iopr_interprocess_by_func(logger, "pread", 3); 198 | iopr_interprocess_by_func(logger, "pread64", 3); 199 | iopr_interprocess_by_func(logger, "pwrite", 3); 200 | iopr_interprocess_by_func(logger, "pwrite64", 3); 201 | iopr_interprocess_by_func(logger, "MPI_File_read_at", 1); 202 | iopr_interprocess_by_func(logger, "MPI_File_read_at_all", 1); 203 | iopr_interprocess_by_func(logger, "MPI_File_write_at", 1); 204 | iopr_interprocess_by_func(logger, "MPI_File_write_at_all", 1); 205 | } 206 | -------------------------------------------------------------------------------- /lib/recorder-sequitur-digram.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) by Argonne National Laboratory 3 | * See COPYRIGHT in top-level directory 4 | */ 5 | 6 | #include 7 | #include "recorder-sequitur.h" 8 | #include "recorder-utils.h" 9 | 10 | 11 | void* build_digram_key(int v1, int exp1, int v2, int exp2) { 12 | void *key = recorder_malloc(DIGRAM_KEY_LEN); 13 | memcpy(key, &v1, sizeof(int)); 14 | memcpy(key+sizeof(int), &exp1, sizeof(int)); 15 | memcpy(key+sizeof(int)*2, &v2, sizeof(int)); 16 | memcpy(key+sizeof(int)*3, &exp2, sizeof(int)); 17 | return key; 18 | } 19 | 20 | 21 | /** 22 | * Look up a digram in the hash table 23 | * 24 | * @param v1 The symbol value of the first symbol of the digram 25 | * @param v2 The symbol value of the second symbol of the digram 26 | */ 27 | Symbol* digram_get(Digram *digram_table, Symbol* sym1, Symbol* sym2) { 28 | 29 | void* key = build_digram_key(sym1->val, sym1->exp, sym2->val, sym2->exp); 30 | 31 | Digram *found; 32 | HASH_FIND(hh, digram_table, key, DIGRAM_KEY_LEN, found); 33 | recorder_free(key, DIGRAM_KEY_LEN); 34 | 35 | if(found) { 36 | return found->symbol; 37 | } 38 | return NULL; 39 | } 40 | 41 | /** 42 | * Insert a digram into the hash table 43 | * 44 | * @param symbol The first symbol of the digram 45 | * 46 | */ 47 | int digram_put(Digram **digram_table, Symbol *symbol) { 48 | if (symbol == NULL || symbol->next == NULL) 49 | return -1; 50 | 51 | void* key = build_digram_key(symbol->val, symbol->exp, symbol->next->val, symbol->next->exp); 52 | 53 | Digram *found; 54 | HASH_FIND(hh, *digram_table, key, DIGRAM_KEY_LEN, found); 55 | 56 | // Found the same digram in the table already 57 | if(found) { 58 | recorder_free(key, DIGRAM_KEY_LEN); 59 | return 1; 60 | } else { 61 | Digram *digram = recorder_malloc(sizeof(Digram)); 62 | digram->key = key; 63 | digram->symbol = symbol; 64 | HASH_ADD_KEYPTR(hh, *digram_table, digram->key, DIGRAM_KEY_LEN, digram); 65 | return 0; 66 | } 67 | } 68 | 69 | 70 | int digram_delete(Digram **digram_table, Symbol *symbol) { 71 | if(symbol == NULL || symbol->next == NULL) 72 | return 0; 73 | 74 | void* key = build_digram_key(symbol->val, symbol->exp, symbol->next->val, symbol->next->exp); 75 | 76 | Digram *found; 77 | HASH_FIND(hh, *digram_table, key, DIGRAM_KEY_LEN, found); 78 | recorder_free(key, DIGRAM_KEY_LEN); 79 | 80 | // 1 1 1, this sequence only has one digram (1, 1) points to the first 1. 81 | // if somehow digram_delete is called on the 2nd 1, we should not delete the 82 | // digram. This can happen for this sequence 1 1 1 2 1 2 83 | if(found && found->symbol == symbol) { 84 | HASH_DELETE(hh, *digram_table, found); 85 | recorder_free(found->key, DIGRAM_KEY_LEN); 86 | recorder_free(found, sizeof(Digram)); 87 | return 0; 88 | } 89 | 90 | return -1; 91 | } 92 | 93 | -------------------------------------------------------------------------------- /lib/recorder-sequitur-logger.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) by Argonne National Laboratory 3 | * See COPYRIGHT in top-level directory 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "recorder-sequitur.h" 10 | #include "recorder-utils.h" 11 | #include "mpi.h" 12 | #include "uthash.h" 13 | 14 | typedef struct UniqueGrammar_t { 15 | int ugi; // unique grammar id 16 | void *key; // serialized grammar stream as key 17 | int count; 18 | UT_hash_handle hh; 19 | } UniqueGrammar; 20 | 21 | static UniqueGrammar *unique_grammars; 22 | static int current_ugi = 0; 23 | 24 | /** 25 | * Store the Grammer in an integer array 26 | * 27 | * | #rules | 28 | * | rule 1 head | #symbols of rule 1 | symbol 1, ..., symbol N | 29 | * | rule 2 head | #symbols of rule 2 | symbol 1, ..., symbol N | 30 | * ... 31 | * 32 | * @len: [out] the length of the array: 1 + 2 * number of rules + number of symbols 33 | * @return: return the array, need to be freed by the caller 34 | * 35 | */ 36 | int* serialize_grammar(Grammar *grammar, int* serialized_integers) { 37 | 38 | int total_integers = 1; // 0: number of rules 39 | int symbols_count = 0, rules_count = 0; 40 | 41 | Symbol *rule, *sym; 42 | DL_COUNT(grammar->rules, rule, rules_count); 43 | 44 | total_integers += 2 * rules_count; 45 | 46 | DL_FOREACH(grammar->rules, rule) { 47 | DL_COUNT(rule->rule_body, sym, symbols_count); 48 | total_integers += symbols_count*2; // val and exp 49 | } 50 | 51 | int i = 0; 52 | int *data = recorder_malloc(sizeof(int) * total_integers); 53 | data[i++] = rules_count; 54 | DL_FOREACH(grammar->rules, rule) { 55 | DL_COUNT(rule->rule_body, sym, symbols_count); 56 | data[i++] = rule->val; 57 | data[i++] = symbols_count; 58 | 59 | DL_FOREACH(rule->rule_body, sym) { 60 | data[i++] = sym->val; // rule id does not change 61 | data[i++] = sym->exp; 62 | } 63 | } 64 | 65 | *serialized_integers = total_integers; 66 | return data; 67 | } 68 | 69 | void sequitur_save_unique_grammars(const char* path, Grammar* lg, int mpi_rank, int mpi_size) { 70 | int grammar_ids[mpi_size]; 71 | int integers; 72 | int *local_grammar = serialize_grammar(lg, &integers); 73 | 74 | int recvcounts[mpi_size], displs[mpi_size]; 75 | PMPI_Gather(&integers, 1, MPI_INT, recvcounts, 1, MPI_INT, 0, MPI_COMM_WORLD); 76 | 77 | displs[0] = 0; 78 | size_t gathered_integers = recvcounts[0]; 79 | for(int i = 1; i < mpi_size;i++) { 80 | gathered_integers += recvcounts[i]; 81 | displs[i] = displs[i-1] + recvcounts[i-1]; 82 | } 83 | 84 | int *gathered_grammars = NULL; 85 | if(mpi_rank == 0) 86 | gathered_grammars = recorder_malloc(sizeof(int) * gathered_integers); 87 | 88 | PMPI_Gatherv(local_grammar, integers, MPI_INT, gathered_grammars, recvcounts, displs, MPI_INT, 0, MPI_COMM_WORLD); 89 | recorder_free(local_grammar, sizeof(int)*integers); 90 | 91 | if(mpi_rank !=0) return; 92 | 93 | char ug_filename[1096] = {0}; 94 | sprintf(ug_filename, "%s/ug.cfg", path); 95 | FILE* ug_file = fopen(ug_filename, "wb"); 96 | 97 | // Go through each rank's grammar 98 | for(int rank = 0; rank < mpi_size; rank++) { 99 | 100 | // Serialized grammar 101 | int* g = gathered_grammars + displs[rank]; 102 | int g_len = recvcounts[rank] * sizeof(int); 103 | //printf("rank: %d, grammar lengh: %d\n", rank, g_len); 104 | 105 | UniqueGrammar *ug_entry = NULL; 106 | HASH_FIND(hh, unique_grammars, g, g_len, ug_entry); 107 | 108 | if(ug_entry) { 109 | // A duplicated grammar, only need to store its id 110 | ug_entry->count++; 111 | grammar_ids[rank] = ug_entry->ugi; 112 | } else { 113 | ug_entry = recorder_malloc(sizeof(UniqueGrammar)); 114 | ug_entry->ugi = current_ugi++; 115 | ug_entry->key = g; // use the existing memory, do not copy it 116 | HASH_ADD_KEYPTR(hh, unique_grammars, ug_entry->key, g_len, ug_entry); 117 | grammar_ids[rank] = ug_entry->ugi; 118 | recorder_write_zlib((unsigned char*)g, g_len, ug_file); 119 | } 120 | } 121 | fclose(ug_file); 122 | 123 | // Clean up the hash table, and gathered grammars 124 | int num_unique_grammars = HASH_COUNT(unique_grammars); 125 | 126 | UniqueGrammar *ug, *tmp; 127 | HASH_ITER(hh, unique_grammars, ug, tmp) { 128 | HASH_DEL(unique_grammars, ug); 129 | recorder_free(ug, sizeof(UniqueGrammar)); 130 | } 131 | 132 | char ug_metadata_fname[1096] = {0}; 133 | sprintf(ug_metadata_fname, "%s/ug.mt", path); 134 | FILE* f = fopen(ug_metadata_fname, "wb"); 135 | fwrite(grammar_ids, sizeof(int), mpi_size, f); 136 | fwrite(&num_unique_grammars, sizeof(int), 1, f); 137 | fflush(f); 138 | fclose(f); 139 | 140 | RECORDER_LOGINFO("[Recorder] unique grammars: %d\n", num_unique_grammars); 141 | } 142 | -------------------------------------------------------------------------------- /lib/recorder-sequitur-symbol.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) by Argonne National Laboratory 3 | * See COPYRIGHT in top-level directory 4 | */ 5 | 6 | #include 7 | #include "recorder-sequitur.h" 8 | #include "recorder-utils.h" 9 | 10 | 11 | Symbol* new_symbol(int val, int exp, bool terminal, Symbol *rule_head) { 12 | Symbol* symbol = recorder_malloc(sizeof(Symbol)); 13 | symbol->val = val; 14 | symbol->exp = exp; 15 | symbol->terminal = terminal; 16 | 17 | symbol->ref = 0; 18 | symbol->rule = NULL; 19 | symbol->rule_head = rule_head; 20 | symbol->rule_body = NULL; 21 | 22 | symbol->prev = NULL; 23 | symbol->next = NULL; 24 | return symbol; 25 | } 26 | 27 | 28 | /** 29 | * Insert a symbol after the pos in rule 30 | * 31 | * `rule` of terminals and non-terminals point to the rule it belongs to 32 | * 33 | * rule_head of an non-terminal points to its corresponding rule 34 | * and the rule_head filed in this case will be set before 35 | * calling this function 36 | */ 37 | void symbol_put(Symbol *rule, Symbol *pos, Symbol *sym) { 38 | if(!IS_RULE_HEAD(sym)) 39 | sym->rule = rule; 40 | 41 | if(pos == NULL) // insert as the head 42 | DL_PREPEND_ELEM(rule->rule_body, rule->rule_body, sym); 43 | else 44 | DL_APPEND_ELEM(rule->rule_body, pos, sym); 45 | 46 | if(IS_NONTERMINAL(sym)) 47 | rule_ref(sym->rule_head); 48 | } 49 | void symbol_delete(Symbol *rule, Symbol *sym, bool deref) { 50 | if(IS_NONTERMINAL(sym) && deref) 51 | rule_deref(sym->rule_head); 52 | 53 | DL_DELETE(rule->rule_body, sym); 54 | recorder_free(sym, sizeof(Symbol)); 55 | sym = NULL; 56 | } 57 | 58 | 59 | /** 60 | * New rule head symbol 61 | */ 62 | Symbol* new_rule(Grammar *grammar) { 63 | Symbol* rule = new_symbol(grammar->rule_id, 1, false, NULL); 64 | grammar->rule_id = grammar->rule_id - 1; 65 | return rule; 66 | } 67 | 68 | /** 69 | * Insert a rule into the rule list 70 | * 71 | */ 72 | void rule_put(Symbol **rules_head, Symbol *rule) { 73 | DL_APPEND(*rules_head, rule); 74 | } 75 | 76 | /** 77 | * Delete a rule from the list 78 | * 79 | */ 80 | void rule_delete(Symbol **rules_head, Symbol *rule) { 81 | DL_DELETE(*rules_head, rule); 82 | recorder_free(rule, sizeof(Symbol)); 83 | rule = NULL; 84 | } 85 | 86 | void rule_ref(Symbol *rule) { 87 | rule->ref++; 88 | } 89 | 90 | void rule_deref(Symbol *rule) { 91 | rule->ref--; 92 | } 93 | -------------------------------------------------------------------------------- /lib/recorder-sequitur-utils.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) by Argonne National Laboratory 3 | * See COPYRIGHT in top-level directory 4 | */ 5 | 6 | #include 7 | #include 8 | #include "recorder-sequitur.h" 9 | 10 | void sequitur_print_digrams(Grammar *grammar) { 11 | Digram *digram, *tmp; 12 | 13 | printf("digrams count: %d\n", HASH_COUNT(grammar->digram_table)); 14 | HASH_ITER(hh, grammar->digram_table, digram, tmp) { 15 | int v1, v2; 16 | memcpy(&v1, digram->key, sizeof(int)); 17 | memcpy(&v2, digram->key+sizeof(int)*2, sizeof(int)); 18 | 19 | if(digram->symbol->rule) 20 | printf("digram(%d, %d, rule:%d): %d %d\n", v1, v2, digram->symbol->rule->val, digram->symbol->val, digram->symbol->next->val); 21 | else 22 | printf("digram(%d, %d, rule:): %d %d\n", v1, v2, digram->symbol->val, digram->symbol->next->val); 23 | } 24 | } 25 | 26 | void sequitur_print_rules(Grammar *grammar) { 27 | Symbol *rule, *sym; 28 | int rules_count = 0, symbols_count = 0; 29 | DL_COUNT(grammar->rules, rule, rules_count); 30 | 31 | DL_FOREACH(grammar->rules, rule) { 32 | int count; 33 | DL_COUNT(rule->rule_body, sym, count); 34 | symbols_count += count; 35 | 36 | printf("Rule %d :-> ", rule->val); 37 | 38 | DL_FOREACH(rule->rule_body, sym) { 39 | if(sym->exp > 1) 40 | printf("%d^%d ", sym->val, sym->exp); 41 | else 42 | printf("%d ", sym->val); 43 | } 44 | printf("\n"); 45 | //#endif 46 | } 47 | 48 | /* 49 | printf("\n=======================\nNumber of rule: %d\n", rules_count); 50 | printf("Number of symbols: %d\n", symbols_count); 51 | printf("Number of Digrams: %d\n=======================\n", HASH_COUNT(grammar.digram_table)); 52 | */ 53 | printf("[recorder] Rules: %d, Symbols: %d\n", rules_count, symbols_count); 54 | } 55 | -------------------------------------------------------------------------------- /lib/recorder-sequitur.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) by Argonne National Laboratory 3 | * See COPYRIGHT in top-level directory 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include "mpi.h" 10 | #include "recorder-sequitur.h" 11 | #include "recorder-utils.h" 12 | 13 | // Uncomment to print debugging messages 14 | // define SEQUITUR_DEBUG 15 | 16 | void delete_symbol(Symbol *sym) { 17 | symbol_delete(sym->rule, sym, true); 18 | } 19 | 20 | 21 | int check_digram(Grammar *grammar, Symbol *sym); 22 | 23 | /** 24 | * Replace a digram by a rule (non-terminal) 25 | * 26 | * @delete_digram: we only put digram after we perform check_digram(); 27 | * during the process of check_digram, we found a match and try to this==match 28 | * In this case, the matched digram is already in the digram table, however the 29 | * new digram has not been inserted into the table yet, so we don't delete becuase 30 | * other rules body may have the same key. 31 | * 32 | */ 33 | void replace_digram(Grammar *grammar, Symbol *origin, Symbol *rule, bool delete_digram) { 34 | if(!IS_RULE_HEAD(rule)) 35 | ERROR_ABORT("replace_digram: not a rule head?\n"); 36 | 37 | // Create an non-terminal 38 | Symbol *replaced = new_symbol(rule->val, 1, false, rule); 39 | 40 | // carefule here, if orgin is the first symbol, then 41 | // NULL will be used as the tail node. 42 | Symbol *prev = NULL; 43 | if(origin->rule->rule_body != origin) 44 | prev = origin->prev; 45 | if(prev != NULL) 46 | digram_delete(&(grammar->digram_table), prev); 47 | 48 | // delete digram before deleting symbols, otherwise we won't have correct digrams 49 | if(delete_digram) { 50 | digram_delete(&(grammar->digram_table), origin); 51 | digram_delete(&(grammar->digram_table), origin->next); 52 | } 53 | 54 | // delete symbol will set origin to NULL 55 | // so we need to store its rule and also delete origin->next first. 56 | Symbol *origin_rule = origin->rule; 57 | delete_symbol(origin->next); 58 | delete_symbol(origin); 59 | 60 | symbol_put(origin_rule, prev, replaced); 61 | 62 | 63 | // Add a new symbol (replaced) after prev 64 | // may introduce another repeated digram that we need to check 65 | if( check_digram(grammar, prev) == 0) { 66 | if(prev == NULL) { 67 | check_digram(grammar, replaced); 68 | } else { 69 | // it is possible that the 'replaced' symbol was deleted 70 | // by the check digram function due to twins-removal rule 71 | // if that's the case, we can not check the 'replaced'. 72 | if(prev->next==replaced) 73 | check_digram(grammar, replaced); 74 | } 75 | } 76 | } 77 | 78 | /** 79 | * Rule Utility 80 | * Replace a rule with its body if the rule is used only once 81 | * 82 | * @sym: is an non-terminal which should be replaced by sym->rule_head->rule_body 83 | */ 84 | void expand_instance(Grammar *grammar, Symbol *sym) { 85 | Symbol *rule = sym->rule_head; 86 | // just double check to make sure 87 | if(rule->ref != 1) 88 | ERROR_ABORT("Attempt to delete a rule that has multiple references!\n"); 89 | 90 | digram_delete(&(grammar->digram_table), sym); 91 | 92 | int n = 0; 93 | Symbol *this, *tmp; 94 | Symbol *tail = sym; 95 | DL_FOREACH_SAFE(rule->rule_body, this, tmp) { 96 | // delete the digram of the old rule (rule body) 97 | digram_delete(&(grammar->digram_table), this); 98 | 99 | Symbol *s = new_symbol(this->val, this->exp, this->terminal, this->rule_head); 100 | symbol_put(sym->rule, tail, s); 101 | tail = s; 102 | n++; 103 | 104 | // delete the symbol of the old rule (rule body) 105 | delete_symbol(this); 106 | } 107 | 108 | this = sym->next; 109 | for(int i = 0; i < n; i++) { 110 | digram_put(&(grammar->digram_table), this); 111 | this = this->next; 112 | } 113 | 114 | delete_symbol(sym); 115 | rule_delete(&(grammar->rules), rule); 116 | } 117 | 118 | /** 119 | * Handle the case in which a just-created digram matches 120 | * a previously existing one. 121 | * 122 | */ 123 | void process_match(Grammar *grammar, Symbol *this, Symbol *match) { 124 | Symbol *rule = NULL; 125 | 126 | // 1. The match consists of entire body of a rule 127 | // Then we replace the new digram with this rule 128 | if(match->prev == match->next) { 129 | rule = match->rule; 130 | replace_digram(grammar, this, match->rule, false); 131 | } else { 132 | // 2. Otherwise, we create a new rule and replace the repeated digrams with this rule 133 | rule = new_rule(grammar); 134 | symbol_put(rule, rule->rule_body, new_symbol(this->val, this->exp, this->terminal, this->rule_head)); 135 | symbol_put(rule, rule->rule_body->prev, new_symbol(this->next->val, this->next->exp, this->next->terminal, this->next->rule_head)); 136 | rule_put(&(grammar->rules), rule); 137 | 138 | replace_digram(grammar, match, rule, true); 139 | replace_digram(grammar, this, rule, false); 140 | 141 | // Insert the rule body into the digram table 142 | digram_put(&(grammar->digram_table), rule->rule_body); 143 | } 144 | 145 | 146 | // Check for "Rule Utility" 147 | // The first symbol of the just-created rule, 148 | // if is an non-terminal could be underutilized 149 | if(rule && rule->rule_body) { 150 | Symbol* tocheck = rule->rule_body->rule_head; 151 | if(tocheck && tocheck->ref < 2 && tocheck->exp < 2) { 152 | #ifdef SEQUITUR_DEBUG 153 | printf("rule utility:%d %d\n", tocheck->val, tocheck->ref); 154 | #endif 155 | expand_instance(grammar, rule->rule_body); 156 | } 157 | } 158 | 159 | } 160 | 161 | /** 162 | * Return 1 means the digram is replaced by a rule 163 | * (Either a new rule or an exisiting rule) 164 | */ 165 | int check_digram(Grammar *grammar, Symbol *sym) { 166 | 167 | if(sym == NULL || sym->next == NULL || sym->next == sym) 168 | return 0; 169 | 170 | // First of all, twins-removal rule. 171 | // Check if digram is of form a^i a^j 172 | // If so, represent it using a^(i+j) 173 | if(grammar->twins_removal && sym->val == sym->next->val) { 174 | digram_delete(&(grammar->digram_table), sym->prev); 175 | sym->exp = sym->exp + sym->next->exp; 176 | //delete_symbol(sym->next); 177 | symbol_delete(sym->next->rule, sym->next, false); 178 | return check_digram(grammar, sym->prev); 179 | } 180 | 181 | 182 | Symbol *match = digram_get(grammar->digram_table, sym, sym->next); 183 | 184 | if(match == NULL) { 185 | // Case 1. new digram, put it in the table 186 | #ifdef SEQUITUR_DEBUG 187 | printf("new digram %d %d\n", sym->val, sym->next->val); 188 | #endif 189 | digram_put(&(grammar->digram_table), sym); 190 | return 0; 191 | } 192 | 193 | if(match->next == sym) { 194 | // Case 2. match found but overlap: do nothing 195 | #ifdef SEQUITUR_DEBUG 196 | printf("found digram but overlap\n"); 197 | #endif 198 | return 0; 199 | } else { 200 | // Case 3. non-overlapping match found 201 | #ifdef SEQUITUR_DEBUG 202 | printf("found non-overlapping digram %d %d\n", sym->val, sym->next->val); 203 | #endif 204 | process_match(grammar, sym, match); 205 | return 1; 206 | } 207 | 208 | } 209 | 210 | Symbol* append_terminal(Grammar* grammar, int val, int exp) { 211 | 212 | Symbol *sym = new_symbol(val, exp, true, NULL); 213 | 214 | Symbol *main_rule = grammar->rules; 215 | Symbol *tail; 216 | 217 | if(main_rule->rule_body) 218 | tail = main_rule->rule_body->prev; // Get the last symbol 219 | else 220 | tail = main_rule->rule_body; // NULL, no symbol yet 221 | 222 | symbol_put(main_rule, tail, sym); 223 | check_digram(grammar, sym->prev); 224 | 225 | return sym; 226 | } 227 | 228 | void sequitur_cleanup(Grammar *grammar) { 229 | Digram *digram, *tmp; 230 | HASH_ITER(hh, grammar->digram_table, digram, tmp) { 231 | HASH_DEL(grammar->digram_table, digram); 232 | recorder_free(digram->key, DIGRAM_KEY_LEN); 233 | recorder_free(digram, sizeof(Digram)); 234 | } 235 | 236 | Symbol *rule, *sym, *tmp2, *tmp3; 237 | DL_FOREACH_SAFE(grammar->rules, rule, tmp2) { 238 | DL_FOREACH_SAFE(rule->rule_body, sym, tmp3) { 239 | DL_DELETE(rule->rule_body, sym); 240 | recorder_free(sym, sizeof(Symbol)); 241 | } 242 | DL_DELETE(grammar->rules, rule); 243 | recorder_free(rule, sizeof(Symbol)); 244 | } 245 | 246 | grammar->digram_table = NULL; 247 | grammar->rules = NULL; 248 | grammar->rule_id = -1; 249 | } 250 | 251 | void sequitur_init_rule_id(Grammar *grammar, int start_rule_id, bool twins_removal) { 252 | grammar->digram_table = NULL; 253 | grammar->rules = NULL; 254 | grammar->rule_id = start_rule_id; 255 | grammar->twins_removal = twins_removal; 256 | 257 | 258 | // Add the main rule: S, which will be the head of the rule list 259 | rule_put(&(grammar->rules), new_rule(grammar)); 260 | } 261 | 262 | void sequitur_init(Grammar *grammar) { 263 | sequitur_init_rule_id(grammar, -1, true); 264 | } 265 | 266 | void sequitur_update(Grammar *grammar, int *update_terminal_id) { 267 | Symbol* rule, *sym; 268 | DL_FOREACH(grammar->rules, rule) { 269 | DL_FOREACH(rule->rule_body, sym) { 270 | if(sym->val >= 0) 271 | sym->val = update_terminal_id[sym->val]; 272 | } 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /lib/recorder-timestamps.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "mpi.h" 6 | #include "recorder.h" 7 | 8 | void ts_get_filename(RecorderLogger *logger, char* ts_filename) { 9 | sprintf(ts_filename, "%s/%d.ts", logger->traces_dir, logger->rank); 10 | } 11 | 12 | void ts_write_out(RecorderLogger* logger) { 13 | if (logger->ts_compression) { 14 | size_t buf_size = logger->ts_index * sizeof(uint32_t); 15 | recorder_write_zlib((unsigned char*)logger->ts, buf_size, logger->ts_file); 16 | } else { 17 | GOTCHA_REAL_CALL(fwrite)(logger->ts, logger->ts_index, sizeof(uint32_t), logger->ts_file); 18 | } 19 | } 20 | 21 | void ts_merge_files(RecorderLogger* logger) { 22 | GOTCHA_SET_REAL_CALL(fread, RECORDER_POSIX); 23 | GOTCHA_SET_REAL_CALL(fwrite, RECORDER_POSIX); 24 | GOTCHA_SET_REAL_CALL(fseek, RECORDER_POSIX); 25 | GOTCHA_SET_REAL_CALL(ftell, RECORDER_POSIX); 26 | GOTCHA_SET_REAL_CALL(fopen, RECORDER_POSIX); 27 | GOTCHA_SET_REAL_CALL(fclose, RECORDER_POSIX); 28 | GOTCHA_SET_REAL_CALL(MPI_File_open, RECORDER_MPIIO); 29 | GOTCHA_SET_REAL_CALL(MPI_File_write_at_all, RECORDER_MPIIO); 30 | GOTCHA_SET_REAL_CALL(MPI_File_close, RECORDER_MPIIO); 31 | GOTCHA_SET_REAL_CALL(MPI_File_sync, RECORDER_MPIIO); 32 | 33 | MPI_Offset file_size = 0, offset = 0; 34 | size_t file_size_t; 35 | void* in; 36 | GOTCHA_REAL_CALL(fseek)(logger->ts_file, 0, SEEK_END); 37 | file_size = (MPI_Offset) GOTCHA_REAL_CALL(ftell)(logger->ts_file); 38 | file_size_t = (size_t) file_size; 39 | in = malloc(file_size_t); 40 | GOTCHA_REAL_CALL(fseek)(logger->ts_file, 0, SEEK_SET); 41 | GOTCHA_REAL_CALL(fread)(in, 1, file_size_t, logger->ts_file); 42 | 43 | char merged_ts_filename[1024]; 44 | sprintf(merged_ts_filename, "%s/recorder.ts", logger->traces_dir); 45 | 46 | int mpi_initialized; 47 | PMPI_Initialized(&mpi_initialized); 48 | if (!mpi_initialized) { 49 | // Non-MPI programs, we use fwrite to write to "recorder.ts" file 50 | // TODO: we can simply rename 0.ts to recorder.ts instead of read 51 | // and write 52 | FILE* merged_file = GOTCHA_REAL_CALL(fopen)(merged_ts_filename, "wb"); 53 | GOTCHA_REAL_CALL(fwrite)(&file_size_t, sizeof(size_t), 1, merged_file); 54 | GOTCHA_REAL_CALL(fwrite)(in, 1, file_size_t, merged_file); 55 | GOTCHA_REAL_CALL(fclose)(merged_file); 56 | } else { 57 | // For MPI programs, we use MPI-IO to collectively write to 58 | // the recorder.ts file 59 | MPI_File fh; 60 | GOTCHA_REAL_CALL(MPI_File_open)(MPI_COMM_WORLD, merged_ts_filename, MPI_MODE_CREATE|MPI_MODE_WRONLY, MPI_INFO_NULL, &fh); 61 | // first write out the compressed file size of each rank 62 | GOTCHA_REAL_CALL(MPI_File_write_at_all)(fh, logger->rank*sizeof(size_t), &file_size_t, sizeof(size_t), MPI_BYTE, MPI_STATUS_IGNORE); 63 | // then write the acutal content of each rank 64 | // we don't intercept MPI_Exscan 65 | MPI_Exscan(&file_size, &offset, 1, MPI_OFFSET, MPI_SUM, MPI_COMM_WORLD); 66 | offset += (logger->nprocs * sizeof(size_t)); 67 | GOTCHA_REAL_CALL(MPI_File_write_at_all)(fh, offset, in, file_size, MPI_BYTE, MPI_STATUS_IGNORE); 68 | GOTCHA_REAL_CALL(MPI_File_sync)(fh); 69 | GOTCHA_REAL_CALL(MPI_File_close)(&fh); 70 | } 71 | free(in); 72 | } 73 | -------------------------------------------------------------------------------- /m4/README: -------------------------------------------------------------------------------- 1 | So the m4 direcotry (even empty) will not be ignored by git. 2 | -------------------------------------------------------------------------------- /test/test_hdf5.c: -------------------------------------------------------------------------------- 1 | #include "hdf5.h" 2 | #include "mpi.h" 3 | 4 | #define FILE "workfile.out" 5 | 6 | int main() { 7 | 8 | MPI_Init(NULL, NULL); 9 | 10 | hid_t file_id, dataset_id, dataspace_id; 11 | hsize_t dims[2]; 12 | herr_t status; 13 | 14 | hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); 15 | 16 | file_id = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); 17 | 18 | // Create data space for the dataset 19 | dims[0] = 4; 20 | dims[1] = 6; 21 | dataspace_id = H5Screate_simple(2, dims, NULL); 22 | 23 | // Create the dataset 24 | dataset_id = H5Dcreate2(file_id, "/dest", H5T_STD_I32BE, dataspace_id, 25 | H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 26 | 27 | status = H5Dclose(dataset_id); 28 | status = H5Sclose(dataspace_id); 29 | status = H5Fclose(file_id); 30 | 31 | // Do it again 32 | file_id = H5Fopen(FILE, H5F_ACC_RDWR, H5P_DEFAULT); 33 | dataspace_id = H5Screate_simple(2, dims, NULL); 34 | dataset_id = H5Dcreate2(file_id, "/dest2", H5T_STD_I32BE, dataspace_id, 35 | H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 36 | status = H5Dclose(dataset_id); 37 | status = H5Sclose(dataspace_id); 38 | status = H5Fclose(file_id); 39 | 40 | MPI_Finalize(); 41 | return 0; 42 | } 43 | -------------------------------------------------------------------------------- /test/test_hybrid.c: -------------------------------------------------------------------------------- 1 | // C program to illustrate use of fork() & 2 | // exec() system call for process creation 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | int main(int argc, char* argv[]) { 14 | 15 | MPI_Init(&argc, &argv); 16 | 17 | pid_t pid; 18 | int ret = 1; 19 | int status; 20 | pid = fork(); 21 | 22 | if (pid == -1){ 23 | printf("can't fork, error occured\n"); 24 | exit(EXIT_FAILURE); 25 | } 26 | else if (pid == 0){ 27 | 28 | printf("child process, pid = %u\n",getpid()); 29 | // Here It will return Parent of child Process means Parent process it self 30 | printf("parent of child process, pid = %u\n",getppid()); 31 | 32 | // the argv list first argument should point to 33 | // filename associated with file being executed 34 | // the array pointer must be terminated by NULL 35 | // pointer 36 | /* 37 | char * argv_list[] = {"test_posix", NULL}; 38 | execv("./test_posix",argv_list); 39 | exit(0); 40 | */ 41 | 42 | char * argv_list[] = {"mpirun", "-np", "2", "./test_mpi", NULL}; 43 | ret = execv("/usr/bin/mpirun", argv_list); 44 | printf("return from execv: %d, %s\n", ret, strerror(errno)); 45 | exit(0); 46 | } 47 | else{ 48 | // parent 49 | for(int i = 0; i < 10; i++) { 50 | MPI_Barrier(MPI_COMM_WORLD); 51 | } 52 | 53 | waitpid(pid, &status, 0); 54 | exit(0); 55 | } 56 | 57 | 58 | MPI_Finalize(); 59 | 60 | return 0; 61 | } 62 | 63 | -------------------------------------------------------------------------------- /test/test_iopr.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | int main(int argc, char** argv) { 11 | MPI_Init(&argc, &argv); 12 | 13 | int fd = open("./workfile.out", O_CREAT|O_WRONLY); 14 | printf("fd: %d\n", fd); 15 | 16 | off_t offset = 0; 17 | char data[] = "hello world"; 18 | 19 | for(int i = 0; i < 10; i++) { 20 | lseek(fd, i*5, SEEK_SET); 21 | pwrite(fd, data, 5, offset); 22 | offset += 5; 23 | } 24 | 25 | close(fd); 26 | 27 | MPI_Finalize(); 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /test/test_mpi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "mpi.h" 3 | 4 | typedef enum { false, true } bool; 5 | 6 | static int rank; 7 | 8 | void testfunc1() { 9 | } 10 | void testfunc2() { 11 | } 12 | void testfunc3() { 13 | } 14 | 15 | 16 | int main(int argc, char *argv[]) { 17 | 18 | MPI_Init(&argc, &argv); 19 | 20 | int world_size; 21 | 22 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 23 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 24 | 25 | 26 | MPI_Comm comm = MPI_COMM_WORLD; 27 | //MPI_Comm_dup(MPI_COMM_WORLD, &comm); 28 | 29 | MPI_Datatype type = MPI_LONG_INT; 30 | MPI_Type_commit(&type); 31 | 32 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 33 | int name_len; 34 | MPI_Get_processor_name(processor_name, &name_len); 35 | 36 | MPI_Comm_set_errhandler(comm, MPI_ERRORS_RETURN); 37 | MPI_Comm_free(&comm); 38 | 39 | void *sbuf = processor_name; 40 | int scount = name_len; 41 | char rbuf[MPI_MAX_PROCESSOR_NAME*world_size]; 42 | MPI_Alltoall(sbuf, scount, MPI_BYTE, rbuf, scount, MPI_BYTE, MPI_COMM_WORLD); 43 | 44 | // MPI Info related 45 | MPI_Info info; 46 | MPI_Info_create(&info); 47 | MPI_Info_set(info, "cb_nodes", "2"); 48 | 49 | // IO-Realted MPI Calls 50 | MPI_File fh; 51 | MPI_Status status; 52 | MPI_Offset offset; 53 | int i, a[10]; 54 | for (i=0;i<10;i++) a[i] = 5; 55 | 56 | int err = MPI_File_open(MPI_COMM_WORLD, "workfile.out", MPI_MODE_RDWR | MPI_MODE_CREATE, info, &fh); 57 | 58 | MPI_File_set_view(fh, 0, MPI_INT, MPI_INT, "native", info); 59 | 60 | MPI_File_set_atomicity(fh, 1); 61 | MPI_File_write_at(fh, 0, a, 10, MPI_INT, &status); 62 | MPI_File_seek(fh, 100, MPI_SEEK_SET); 63 | MPI_File_get_size(fh, &offset); 64 | 65 | MPI_File_close(&fh); 66 | 67 | MPI_Barrier(MPI_COMM_WORLD); 68 | 69 | int flag, idx; 70 | if(world_size > 1) { 71 | if (rank == 0) { 72 | MPI_Send(sbuf, scount, MPI_BYTE, 1, 0, MPI_COMM_WORLD); 73 | MPI_Send(sbuf, scount, MPI_BYTE, 1, 0, MPI_COMM_WORLD); 74 | } else if(rank == 1){ 75 | MPI_Request req[2]; 76 | int finished[2]; 77 | MPI_Irecv(rbuf, scount, MPI_BYTE, 0, 0, MPI_COMM_WORLD, &req[0]); 78 | MPI_Irecv(rbuf, scount, MPI_BYTE, 0, 0, MPI_COMM_WORLD, &req[1]); 79 | MPI_Testsome(2, req, &idx, finished, MPI_STATUSES_IGNORE); 80 | MPI_Testany(2, req, &idx, &flag, MPI_STATUS_IGNORE); 81 | MPI_Testall(2, req, &flag, MPI_STATUSES_IGNORE); 82 | } 83 | } 84 | 85 | MPI_Request req; 86 | MPI_Bcast(a, 1, MPI_INT, 0, MPI_COMM_WORLD); 87 | 88 | //MPI_Ibcast(a, 1, MPI_INT, 0, MPI_COMM_WORLD, &req); 89 | //MPI_Wait(&req, MPI_STATUS_IGNORE); 90 | //MPI_Test(&req, &flag, MPI_STATUS_IGNORE); 91 | 92 | if(world_size > 2) { 93 | int recv, send; 94 | MPI_Reduce(&send, &recv, 1, MPI_INT, MPI_MAX, 2, MPI_COMM_WORLD); 95 | } 96 | 97 | for(int i = 0; i < 10; i++) { 98 | MPI_Barrier(MPI_COMM_WORLD); 99 | } 100 | 101 | testfunc1(); 102 | testfunc2(); 103 | testfunc3(); 104 | 105 | MPI_Finalize(); 106 | return 0; 107 | } 108 | -------------------------------------------------------------------------------- /test/test_phdf5.c: -------------------------------------------------------------------------------- 1 | #include "hdf5.h" 2 | #include "mpi.h" 3 | 4 | #define FILE_NAME_1 "workfile.1.out" 5 | #define FILE_NAME_2 "workfile.2.out" 6 | 7 | void parallel_hdf5_test(const char* filename) { 8 | hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); 9 | H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); 10 | //H5Pset_fapl_split(plist_id, "-metadata.h5", H5P_DEFAULT, "-data.h5", H5P_DEFAULT); 11 | 12 | hid_t file_id, dataset_id, dataspace_id, attr_id; 13 | hsize_t dims[2] = {2, 3}; 14 | herr_t status; 15 | int data[] = {100, 100, 100, 100, 100, 100}; 16 | 17 | file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); 18 | dataspace_id = H5Screate_simple(2, dims, NULL); 19 | 20 | dataset_id = H5Dcreate2(file_id, "/dest", H5T_STD_I32BE, dataspace_id, 21 | H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); 22 | H5Fflush(dataset_id, H5F_SCOPE_GLOBAL); 23 | H5Dwrite(dataset_id, H5T_STD_I32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); 24 | H5Fflush(dataset_id, H5F_SCOPE_GLOBAL); 25 | 26 | 27 | attr_id = H5Acreate2(dataset_id, "min", H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT); 28 | H5Awrite(attr_id, H5T_STD_I32BE, &data[0]); 29 | H5Fflush(attr_id, H5F_SCOPE_GLOBAL); 30 | 31 | H5Awrite(attr_id, H5T_STD_I32BE, &data[1]); 32 | H5Fflush(attr_id, H5F_SCOPE_GLOBAL); 33 | 34 | H5Awrite(attr_id, H5T_STD_I32BE, &data[1]); 35 | H5Fflush(attr_id, H5F_SCOPE_GLOBAL); 36 | 37 | 38 | H5Aclose(attr_id); 39 | H5Dclose(dataset_id); 40 | H5Sclose(dataspace_id); 41 | H5Pclose(plist_id); 42 | H5Fclose(file_id); 43 | } 44 | 45 | int main(int argc, char *argv[]) { 46 | 47 | MPI_Init(&argc, &argv); 48 | 49 | parallel_hdf5_test(FILE_NAME_1); 50 | parallel_hdf5_test(FILE_NAME_2); 51 | 52 | MPI_Finalize(); 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /test/test_posix.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | int main() { 10 | 11 | struct stat buf; 12 | int res; 13 | 14 | res = stat("./workfile.out", &buf); 15 | 16 | res = lstat("./workfile.out", &buf); 17 | 18 | int fd = open("./workfile.out", O_RDONLY); 19 | res = fstat(fd, &buf); 20 | 21 | struct flock lk; 22 | fcntl(fd, F_GETLK, &lk); 23 | 24 | char data[] = "hello world"; 25 | write(fd, &data, sizeof(char)*5); 26 | res = fstat(fd, &buf); 27 | write(fd, &data, sizeof(char)*5); 28 | res = fstat(fd, &buf); 29 | write(fd, &data, sizeof(char)*7); 30 | close(fd); 31 | 32 | FILE* fp = fopen("./workfile.out", "w"); 33 | fseeko(fp, 10, SEEK_CUR); 34 | ftello(fp); 35 | fclose(fp); 36 | 37 | int a, b; 38 | a = 0; 39 | b = 10; 40 | fprintf(stderr, "hello world: %d %d\n", a, b); 41 | 42 | 43 | DIR* dir = opendir("/sys"); 44 | struct dirent *dent = readdir(dir); 45 | do { 46 | printf("dent %s\n", dent->d_name); 47 | dent = readdir(dir); 48 | } while(dent); 49 | closedir(dir); 50 | 51 | 52 | return 0; 53 | } 54 | -------------------------------------------------------------------------------- /test/test_signal.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "mpi.h" 4 | 5 | typedef enum { false, true } bool; 6 | 7 | static int rank; 8 | 9 | void testfunc1() { 10 | } 11 | void testfunc2() { 12 | } 13 | void testfunc3() { 14 | } 15 | 16 | 17 | int main(int argc, char *argv[]) { 18 | 19 | MPI_Init(&argc, &argv); 20 | 21 | int world_size; 22 | 23 | MPI_Comm_size(MPI_COMM_WORLD, &world_size); 24 | MPI_Comm_rank(MPI_COMM_WORLD, &rank); 25 | 26 | MPI_Comm comm = MPI_COMM_WORLD; 27 | //MPI_Comm_dup(MPI_COMM_WORLD, &comm); 28 | 29 | MPI_Datatype type = MPI_LONG_INT; 30 | MPI_Type_commit(&type); 31 | 32 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 33 | int name_len; 34 | MPI_Get_processor_name(processor_name, &name_len); 35 | 36 | MPI_Comm_set_errhandler(comm, MPI_ERRORS_RETURN); 37 | MPI_Comm_free(&comm); 38 | 39 | void *sbuf = processor_name; 40 | int scount = name_len; 41 | char rbuf[MPI_MAX_PROCESSOR_NAME*world_size]; 42 | MPI_Alltoall(sbuf, scount, MPI_BYTE, rbuf, scount, MPI_BYTE, MPI_COMM_WORLD); 43 | 44 | // MPI Info related 45 | MPI_Info info; 46 | MPI_Info_create(&info); 47 | MPI_Info_set(info, "cb_nodes", "2"); 48 | 49 | int recv, send; 50 | MPI_Reduce(&send, &recv, 1, MPI_INT, MPI_MAX, 2, MPI_COMM_WORLD); 51 | 52 | for(int i = 0; i < 10; i++) { 53 | MPI_Barrier(MPI_COMM_WORLD); 54 | } 55 | 56 | testfunc1(); 57 | testfunc2(); 58 | testfunc3(); 59 | 60 | // Sleep and wait for a kill() 61 | sleep(20); 62 | 63 | MPI_Finalize(); 64 | return 0; 65 | } 66 | -------------------------------------------------------------------------------- /test/vec.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011-2020 NVIDIA Corporation. All rights reserved 3 | * 4 | * Sample CUPTI app to print a trace of CUDA API and GPU activity 5 | */ 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #define DRIVER_API_CALL(apiFuncCall) \ 13 | do { \ 14 | CUresult _status = apiFuncCall; \ 15 | if (_status != CUDA_SUCCESS) { \ 16 | const char* errstr; \ 17 | cuGetErrorString(_status, &errstr); \ 18 | fprintf(stderr, "%s:%d: error: function %s failed with error %s.\n", \ 19 | __FILE__, __LINE__, #apiFuncCall, errstr); \ 20 | exit(-1); \ 21 | } \ 22 | } while (0) 23 | 24 | #define RUNTIME_API_CALL(apiFuncCall) \ 25 | do { \ 26 | cudaError_t _status = apiFuncCall; \ 27 | if (_status != cudaSuccess) { \ 28 | fprintf(stderr, "%s:%d: error: function %s failed with error %s.\n", \ 29 | __FILE__, __LINE__, #apiFuncCall, cudaGetErrorString(_status));\ 30 | exit(-1); \ 31 | } \ 32 | } while (0) 33 | 34 | #define CUPTI_CALL(call) \ 35 | do { \ 36 | CUptiResult _status = call; \ 37 | if (_status != CUPTI_SUCCESS) { \ 38 | const char *errstr; \ 39 | cuptiGetResultString(_status, &errstr); \ 40 | fprintf(stderr, "%s:%d: error: function %s failed with error %s.\n", \ 41 | __FILE__, __LINE__, #call, errstr); \ 42 | exit(-1); \ 43 | } \ 44 | } while (0) 45 | 46 | #define COMPUTE_N 50000 47 | 48 | extern void initTrace(void); 49 | extern void finiTrace(void); 50 | 51 | // Kernels 52 | __global__ void 53 | VecAdd(const int* A, const int* B, int* C, int N) 54 | { 55 | int i = blockDim.x * blockIdx.x + threadIdx.x; 56 | if (i < N) 57 | C[i] = A[i] + B[i]; 58 | } 59 | 60 | __global__ void 61 | VecSub(const int* A, const int* B, int* C, int N) 62 | { 63 | int i = blockDim.x * blockIdx.x + threadIdx.x; 64 | if (i < N) 65 | C[i] = A[i] - B[i]; 66 | } 67 | 68 | static void 69 | do_pass(cudaStream_t stream) 70 | { 71 | int *h_A, *h_B, *h_C; 72 | int *d_A, *d_B, *d_C; 73 | size_t size = COMPUTE_N * sizeof(int); 74 | int threadsPerBlock = 256; 75 | int blocksPerGrid = 0; 76 | 77 | // Allocate input vectors h_A and h_B in host memory 78 | // don't bother to initialize 79 | h_A = (int*)malloc(size); 80 | h_B = (int*)malloc(size); 81 | h_C = (int*)malloc(size); 82 | if (!h_A || !h_B || !h_C) { 83 | printf("Error: out of memory\n"); 84 | exit(-1); 85 | } 86 | 87 | // Allocate vectors in device memory 88 | RUNTIME_API_CALL(cudaMalloc((void**)&d_A, size)); 89 | RUNTIME_API_CALL(cudaMalloc((void**)&d_B, size)); 90 | RUNTIME_API_CALL(cudaMalloc((void**)&d_C, size)); 91 | 92 | RUNTIME_API_CALL(cudaMemcpyAsync(d_A, h_A, size, cudaMemcpyHostToDevice, stream)); 93 | RUNTIME_API_CALL(cudaMemcpyAsync(d_B, h_B, size, cudaMemcpyHostToDevice, stream)); 94 | 95 | blocksPerGrid = (COMPUTE_N + threadsPerBlock - 1) / threadsPerBlock; 96 | VecAdd<<>>(d_A, d_B, d_C, COMPUTE_N); 97 | VecSub<<>>(d_A, d_B, d_C, COMPUTE_N); 98 | 99 | RUNTIME_API_CALL(cudaMemcpyAsync(h_C, d_C, size, cudaMemcpyDeviceToHost, stream)); 100 | 101 | if (stream == 0) 102 | RUNTIME_API_CALL(cudaDeviceSynchronize()); 103 | else 104 | RUNTIME_API_CALL(cudaStreamSynchronize(stream)); 105 | 106 | free(h_A); 107 | free(h_B); 108 | free(h_C); 109 | RUNTIME_API_CALL(cudaFree(d_A)); 110 | RUNTIME_API_CALL(cudaFree(d_B)); 111 | RUNTIME_API_CALL(cudaFree(d_C)); 112 | } 113 | 114 | int 115 | main(int argc, char *argv[]) 116 | { 117 | CUdevice device; 118 | char deviceName[32]; 119 | int deviceNum = 0, devCount = 0; 120 | 121 | DRIVER_API_CALL(cuInit(0)); 122 | 123 | RUNTIME_API_CALL(cudaGetDeviceCount(&devCount)); 124 | //devCount = 1; 125 | 126 | for (deviceNum=0; deviceNum 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | extern "C" { // Needed to mix linking C and C++ sources 9 | #include "reader.h" 10 | #include "reader-private.h" 11 | } 12 | 13 | using namespace std; 14 | 15 | typedef struct RRecord_t { 16 | int rank; 17 | int seq_id; 18 | Record* record; 19 | } RRecord; 20 | 21 | 22 | RecorderReader *reader; 23 | vector records; 24 | 25 | static inline size_t str2sizet(char* arg) { 26 | size_t res; 27 | sscanf(arg, "%zu", &res); 28 | return res; 29 | } 30 | 31 | size_t get_eof(string filename, unordered_map local_eof, unordered_map global_eof) { 32 | size_t e1 = 0, e2 = 0; 33 | if(local_eof.find(filename) != local_eof.end()) 34 | e1 = local_eof[filename]; 35 | if(global_eof.find(filename) != global_eof.end()) 36 | e2 = global_eof[filename]; 37 | return max(e1, e2); 38 | } 39 | 40 | void handle_data_operation(RRecord &rr, 41 | unordered_map &offset_book, // 42 | unordered_map &local_eof, // (locally) 43 | unordered_map &global_eof, // (globally) 44 | unordered_map> &intervals, 45 | string current_mpifh, int current_mpi_call_depth) 46 | { 47 | Record *R = rr.record; 48 | const char* func = recorder_get_func_name(reader, R); 49 | 50 | if(!strstr(func, "read") && !strstr(func, "write")) 51 | return; 52 | 53 | Interval I; 54 | I.rank = rr.rank; 55 | I.seqId = rr.seq_id; 56 | I.tstart = R->tstart; 57 | I.isRead = strstr(func, "read") ? true: false; 58 | memset(I.mpifh, 0, sizeof(I.mpifh)); 59 | strcpy(I.mpifh, "-"); 60 | 61 | if(R->call_depth == current_mpi_call_depth+1) 62 | strcpy(I.mpifh, current_mpifh.c_str()); 63 | 64 | string filename = ""; 65 | if(strstr(func, "writev") || strstr(func, "readv")) { 66 | filename = R->args[0]; 67 | I.offset = offset_book[filename]; 68 | I.count = str2sizet(R->args[1]); 69 | offset_book[filename] += I.count; 70 | } else if(strstr(func, "fwrite") || strstr(func, "fread")) { 71 | filename = R->args[3]; 72 | I.offset = offset_book[filename]; 73 | I.count = str2sizet(R->args[1]) * str2sizet(R->args[2]); 74 | offset_book[filename] += I.count; 75 | } else if(strstr(func, "pwrite") || strstr(func, "pread")) { 76 | filename = R->args[0]; 77 | I.count = str2sizet(R->args[2]); 78 | I.offset = str2sizet(R->args[3]); 79 | } else if(strstr(func, "write") || strstr(func, "read")) { 80 | filename = R->args[0]; 81 | I.count = str2sizet(R->args[2]); 82 | I.offset = offset_book[filename]; 83 | offset_book[filename] += I.count; 84 | } else if(strstr(func, "fprintf")) { 85 | filename = R->args[0]; 86 | I.count = str2sizet(R->args[1]); 87 | offset_book[filename] += I.count; 88 | } 89 | 90 | if(filename != "") { 91 | 92 | if(local_eof.find(filename) == local_eof.end()) 93 | local_eof[filename] = I.offset + I.count; 94 | else 95 | local_eof[filename] = max(local_eof[filename], I.offset+I.count); 96 | 97 | /* TODO: 98 | * On POSIX systems, update global eof now 99 | * On other systems (e.,g with commit semantics 100 | * and session semantics), update global eof at 101 | * close/sync ? */ 102 | global_eof[filename] = get_eof(filename, local_eof, global_eof); 103 | 104 | intervals[filename].push_back(I); 105 | } 106 | } 107 | 108 | /* 109 | * Inspect metadata operations to make 110 | * sure we can correctly keep track of 111 | * the position of file pointers. 112 | * 113 | * Params: 114 | * offset_book: 115 | * local_eof: 116 | * global_eof: 117 | */ 118 | void handle_metadata_operation(RRecord &rr, 119 | unordered_map &offset_book, 120 | unordered_map &local_eof, 121 | unordered_map &global_eof 122 | ) 123 | { 124 | Record *R = rr.record; 125 | const char* func = recorder_get_func_name(reader, R); 126 | 127 | string filename = ""; 128 | 129 | if(strstr(func, "fopen") || strstr(func, "fdopen")) { 130 | filename = R->args[0]; 131 | offset_book[filename] = 0; 132 | 133 | // append 134 | if(strstr(R->args[1], "a")) 135 | offset_book[filename] = get_eof(filename, local_eof, global_eof); 136 | 137 | } else if(strstr(func, "open")) { 138 | filename = R->args[0]; 139 | offset_book[filename] = 0; 140 | 141 | /* TODO: Do O_APPEND, SEEK_SET, ... have 142 | * the same value on this machine and the machine where 143 | * traces were collected? */ 144 | int flag = atoi(R->args[1]); 145 | if(flag & O_APPEND) 146 | offset_book[filename] = get_eof(filename, local_eof, global_eof); 147 | 148 | } else if(strstr(func, "seek") || strstr(func, "seeko")) { 149 | filename = R->args[0]; 150 | size_t offset = str2sizet(R->args[1]); 151 | int whence = atoi(R->args[2]); 152 | 153 | if(whence == SEEK_SET) 154 | offset_book[filename] = offset; 155 | else if(whence == SEEK_CUR) 156 | offset_book[filename] += offset; 157 | else if(whence == SEEK_END) 158 | offset_book[filename] = get_eof(filename, local_eof, global_eof); 159 | 160 | } else if(strstr(func, "close") || strstr(func, "sync")) { 161 | filename = R->args[0]; 162 | 163 | // Update the global eof at close time 164 | global_eof[filename] = get_eof(filename, local_eof, global_eof); 165 | 166 | // Remove from the table 167 | if(strstr(func, "close")) 168 | offset_book.erase(filename); 169 | } else { 170 | return; 171 | } 172 | } 173 | 174 | bool compare_by_tstart(const RRecord &lhs, const RRecord &rhs) { 175 | return lhs.record->tstart < rhs.record->tstart; 176 | } 177 | 178 | static int current_seq_id = 0; 179 | 180 | void insert_one_record(Record* r, void* arg) { 181 | 182 | /* increase seq id before the checks 183 | * so we always have the correct seq id 184 | * when we insert a record */ 185 | current_seq_id++; 186 | 187 | int func_type = recorder_get_func_type(reader, r); 188 | const char* func = recorder_get_func_name(reader, r); 189 | 190 | if((func_type != RECORDER_POSIX) && (func_type != RECORDER_MPIIO)) 191 | return; 192 | 193 | // For MPI-IO calls keep only MPI_File_write* and MPI_File_read* 194 | if((func_type == RECORDER_MPIIO) && (!strstr(func, "MPI_File_write")) 195 | && (!strstr(func, "MPI_File_read")) && (!strstr(func, "MPI_File_iread")) 196 | && (!strstr(func, "MPI_File_iwrite"))) 197 | return; 198 | 199 | if(strstr(func, "dir") || strstr(func, "link")) 200 | return; 201 | 202 | RRecord rr; 203 | rr.record = r; 204 | rr.rank = *((int*) arg); 205 | rr.seq_id = current_seq_id - 1; 206 | 207 | records.push_back(rr); 208 | } 209 | 210 | void flatten_and_sort_records(RecorderReader *reader) { 211 | 212 | int nprocs = reader->metadata.total_ranks; 213 | 214 | for(int rank = 0; rank < nprocs; rank++) { 215 | current_seq_id = 0; 216 | recorder_decode_records2(reader, rank, insert_one_record, &rank); 217 | } 218 | 219 | sort(records.begin(), records.end(), compare_by_tstart); 220 | } 221 | 222 | /* 223 | * Return an array of 224 | * mapping. The length of this array will be 225 | * saved in 'num_files'. 226 | * caller is responsible for freeing space 227 | * after use. 228 | */ 229 | IntervalsMap* build_offset_intervals(RecorderReader *_reader, int *num_files) { 230 | 231 | reader = _reader; 232 | flatten_and_sort_records(reader); 233 | 234 | // 235 | unordered_map> intervals; 236 | 237 | unordered_map offset_books[reader->metadata.total_ranks]; 238 | unordered_map local_eofs[reader->metadata.total_ranks]; 239 | unordered_map global_eof; 240 | 241 | string current_mpifh = ""; 242 | int current_mpi_call_depth; 243 | 244 | int i; 245 | for(i = 0; i < records.size(); i++) { 246 | RRecord rr = records[i]; 247 | const char* func = recorder_get_func_name(reader, rr.record); 248 | // Only MPI_File_write* and MPI_File_read* calls here 249 | // thanks to insert_one_record() 250 | if(strstr(func, "MPI")) { 251 | current_mpifh = rr.record->args[0]; 252 | current_mpi_call_depth = (int) rr.record->call_depth; 253 | // POSIX calls 254 | } else { 255 | handle_metadata_operation(rr, offset_books[rr.rank], local_eofs[rr.rank], global_eof); 256 | handle_data_operation(rr, offset_books[rr.rank], local_eofs[rr.rank], global_eof, intervals, current_mpifh, current_mpi_call_depth); 257 | } 258 | } 259 | 260 | /* Now we have the list of intervals for all files, 261 | * we copy it from the C++ vector to a C style pointer. 262 | * The vector and the memory it allocated will be 263 | * freed once out of scope (leave this function) 264 | * Also, using a C style struct is easier for Python binding. 265 | */ 266 | *num_files = intervals.size(); 267 | 268 | IntervalsMap *IM = (IntervalsMap*) malloc(sizeof(IntervalsMap) * (*num_files)); 269 | 270 | i = 0; 271 | for(auto it = intervals.cbegin(); it != intervals.cend(); it++) { 272 | /* it->first: filename 273 | * it->second: vector */ 274 | IM[i].filename = strdup(it->first.c_str()); 275 | IM[i].num_intervals = it->second.size(); 276 | IM[i].intervals = (Interval*) malloc(sizeof(Interval) * it->second.size()); 277 | memcpy(IM[i].intervals, it->second.data(), sizeof(Interval) * it->second.size()); 278 | i++; 279 | } 280 | 281 | for(i = 0; i < records.size(); i++) { 282 | recorder_free_record(records[i].record); 283 | } 284 | return IM; 285 | } 286 | -------------------------------------------------------------------------------- /tools/conflict-detector.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | extern "C" { 8 | #include "reader.h" 9 | #include "reader-private.h" 10 | } 11 | using namespace std; 12 | 13 | // Allow setting the limit of number of 14 | // conflicts we detect and save to file 15 | // mainly used to accerlate the detection 16 | // and verification process. 17 | static int conflicts_cap = INT_MAX; 18 | 19 | int compare_by_offset(const void *lhs, const void *rhs) { 20 | Interval *first = (Interval*)lhs; 21 | Interval *second = (Interval*)rhs; 22 | return first->offset - second->offset; 23 | } 24 | 25 | int compare_by_index(const void *lhs, const void *rhs) { 26 | Interval *first = (Interval*)lhs; 27 | Interval *second = (Interval*)rhs; 28 | 29 | if (first->rank < second->rank) 30 | return 1; 31 | else if (first->rank == second->rank) 32 | return first->seqId < second->seqId; 33 | else 34 | return 0; 35 | } 36 | 37 | int sum_array(int *arr, int count) { 38 | int i, sum = 0; 39 | for(i = 0; i < count; i++) { 40 | sum += arr[i]; 41 | } 42 | return sum; 43 | } 44 | 45 | int is_conflict(Interval* i1, Interval* i2) { 46 | // TODO: same rank but multi-threaded? 47 | if(i1->rank == i2->rank) 48 | return false; 49 | if(i1->isRead && i2->isRead) 50 | return false; 51 | if(i1->offset+i1->count <= i2->offset) 52 | return false; 53 | if(i2->offset+i2->count <= i1->offset) 54 | return false; 55 | return true; 56 | } 57 | 58 | void detect_conflicts(IntervalsMap *IM, int num_files, const char* base_dir) { 59 | FILE* conflict_file; 60 | char path[512]; 61 | size_t total_conflicts = 0; 62 | 63 | sprintf(path, "%s/conflicts.dat", base_dir); 64 | conflict_file = fopen(path, "w"); 65 | 66 | int idx, i, j; 67 | for(idx = 0; idx < num_files; idx++) { 68 | 69 | char *filename = IM[idx].filename; 70 | Interval *intervals = IM[idx].intervals; 71 | //fprintf(conflict_file, "#%d:%s\n", idx, filename); 72 | 73 | // sort by offset 74 | qsort(intervals, IM[idx].num_intervals, sizeof(Interval), compare_by_offset); 75 | 76 | int i = 0, j = 0; 77 | Interval *i1, *i2; 78 | 79 | while(i < IM[idx].num_intervals-1) { 80 | i1 = &intervals[i]; 81 | j = i + 1; 82 | 83 | vector conflicts; 84 | while(j < IM[idx].num_intervals) { 85 | i2 = &intervals[j]; 86 | 87 | if(is_conflict(i1,i2)) { 88 | conflicts.push_back(i2); 89 | } 90 | 91 | if(i1->offset+i1->count > i2->offset) { 92 | j++; 93 | } else { 94 | // skip the rest, as they will all have 95 | // a bigger starting offset 96 | break; 97 | } 98 | } 99 | 100 | size_t num_conflict_pairs = conflicts.size(); 101 | if (num_conflict_pairs > 0) { 102 | total_conflicts += num_conflict_pairs; 103 | 104 | fwrite(&i1->rank, sizeof(int), 1, conflict_file); 105 | fwrite(&i1->seqId, sizeof(int), 1, conflict_file); 106 | fwrite(&num_conflict_pairs, sizeof(size_t), 1, conflict_file); 107 | 108 | // previously intervals were sorted by starting offset 109 | // when saving it out, we sort it by sequence id 110 | sort(conflicts.begin(), conflicts.end(), compare_by_index); 111 | 112 | // write out all conflict pairs 113 | for (vector::iterator it = conflicts.begin(); it!=conflicts.end(); ++it) { 114 | i2 = *it; 115 | fwrite(&i2->rank, sizeof(int), 1, conflict_file); 116 | fwrite(&i2->seqId, sizeof(int), 1, conflict_file); 117 | } 118 | conflicts.clear(); 119 | } 120 | 121 | // end of one starting operation 122 | // move on to the next one 123 | i++; 124 | if (total_conflicts > conflicts_cap) 125 | goto done; 126 | } // end of one file 127 | } 128 | 129 | done: 130 | fclose(conflict_file); 131 | } 132 | 133 | int main(int argc, char* argv[]) { 134 | 135 | RecorderReader reader; 136 | recorder_init_reader(argv[1], &reader); 137 | 138 | if (argc == 3) 139 | conflicts_cap = atoi(argv[2]); 140 | 141 | int i, num_files; 142 | IntervalsMap *IM = build_offset_intervals(&reader, &num_files); 143 | 144 | detect_conflicts(IM, num_files, argv[1]); 145 | 146 | // Free IM 147 | for(i = 0; i < num_files; i++) { 148 | free(IM[i].filename); 149 | free(IM[i].intervals); 150 | } 151 | free(IM); 152 | 153 | recorder_free_reader(&reader); 154 | return 0; 155 | } 156 | -------------------------------------------------------------------------------- /tools/generator/clean.py: -------------------------------------------------------------------------------- 1 | import sys 2 | with open(sys.argv[1]) as f: 3 | lines = f.readlines() 4 | for line in lines: 5 | if "nc_" in line: 6 | print(line.replace("\n","")) 7 | -------------------------------------------------------------------------------- /tools/generator/generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | 4 | arg_type_set = set() 5 | 6 | class Arg: 7 | def __init__(self, arg_str): 8 | # e.g., const char* str 9 | # e.g., hid_t fild_id 10 | arg_str = arg_str.strip() 11 | space_idx = arg_str.rfind(" ") 12 | self.arg_type = arg_str[0:space_idx] 13 | self.arg_name = arg_str[space_idx+1:] 14 | 15 | clean_arg_type = self.arg_type.replace("const", "") 16 | clean_arg_type = clean_arg_type.strip() 17 | self.clean_arg_type = clean_arg_type 18 | 19 | clean_arg_name = self.arg_name 20 | if self.arg_name.find("[") != -1: 21 | clean_arg_name = self.arg_name[0:self.arg_name.find("[")] 22 | clean_arg_name = clean_arg_name.replace("*", "") 23 | self.clean_arg_name = clean_arg_name 24 | 25 | arg_type_set.add(self.clean_arg_type) 26 | 27 | def __str__(self): 28 | return "%s %s" %(self.arg_type, self.arg_name) 29 | 30 | # ordinary type, i.e., not pointer, not array 31 | def is_type_ord(self): 32 | if "*" not in self.arg_type and "*" not in self.arg_name: 33 | if "[" not in self.arg_type and "[" not in self.arg_name: 34 | return True 35 | return False 36 | 37 | def is_type_ptr(self): 38 | star_count = 0 39 | star_count += self.arg_type.count("*") 40 | star_count += self.arg_name.count("*") 41 | if star_count == 1: 42 | if "[]" not in self.arg_name and "[]" not in self.arg_type: 43 | return True 44 | return False 45 | 46 | def is_type_ptrptr(self): 47 | star_count = 0 48 | star_count += self.arg_type.count("*") 49 | star_count += self.arg_name.count("*") 50 | if star_count == 2: 51 | if "[]" not in self.arg_name and "[]" not in self.arg_type: 52 | return True 53 | return False 54 | 55 | # this is to make sure var in int *var 56 | # is not NULL 57 | def get_safe_star_ptr(self): 58 | return "(%s==NULL)?-1:*%s" %(self.clean_arg_name, self.clean_arg_name) 59 | 60 | 61 | # possible outcome: 62 | # 1. itoa() for all integers, e.g., size_t, hid_t, H5_index_t 63 | # 2. ftoa() for all floating point values 64 | # 3. strtoa() for char* 65 | # 4. realrealpath() for const char* filename 66 | # 5. arrtostr() 67 | # 6. ptoa() for pointers or arguments we don't care 68 | def get_assemble_arg(self): 69 | 70 | # PnetCDF possible arg types: ['MPI_Offset', 'MPI_Comm', 'unsigned char', 71 | # 'char', 'short', 'long', 'unsigned long long', 'long long', 'MPI_Info', 72 | # 'unsigned short', 'float', 'signed char', 'MPI_Offset*', 'MPI_Datatype', 73 | # 'double', 'int', 'unsigned int', 'void'] 74 | 75 | # HDF5 possible arg types: ['H5O_native_info_t', 'H5FD_mem_t', 'unsigned long', 76 | # 'H5P_cls_create_func_t', 'H5P_prp_get_func_t', 'H5D_mpio_actual_chunk_opt_mode_t', 77 | # 'H5O_info1_t', 'hbool_t', 'H5L_info1_t', 'H5P_cls_copy_func_t', 'H5LT_lang_t', 78 | # 'H5L_elink_traverse_t', 'H5ES_err_info_t', 'H5I_future_discard_func_t', 79 | # 'H5FD_mpio_xfer_t', 'H5E_major_t', 'H5F_file_space_type_t', 'H5T_cset_t', 80 | # 'haddr_t', 'H5O_mcdt_search_cb_t', 'H5VL_class_value_t', 'H5A_operator2_t', 81 | # 'H5S_sel_type', 'H5F_flush_cb_t', 'H5F_info2_t', 'H5Z_filter_func_t', 82 | # 'H5A_operator1_t', 'float', 'H5T_conv_except_func_t', 'H5D_mpio_actual_io_mode_t', 83 | # 'H5ES_event_complete_func_t', 'H5I_future_realize_func_t', '..', 'H5O_iterate2_t', 84 | # 'H5F_close_degree_t', 'H5O_type_t', 'H5MM_free_t', 'H5FD_mirror_fapl_t', 85 | # 'H5_index_t', 'H5T_class_t', 'H5F_sect_info_t', 'H5P_prp_create_func_t', 'int', 86 | # 'H5FD_mpio_collective_opt_t', 'char', 'unsigned', 'H5FD_onion_fapl_info_t', 87 | # 'H5E_type_t', 'H5D_selection_io_mode_t', 'H5P_prp_compare_func_t', 'H5D_layout_t', 88 | # 'H5D_alloc_time_t', 'H5VL_subclass_t', 'unsigned short', 'H5D_operator_t', 'long', 89 | # 'H5F_scope_t', 'H5FD_class_value_t', 'H5D_gather_func_t', 'H5O_info2_t', 90 | # 'H5P_cls_close_func_t', 'H5L_info2_t', 'H5P_iterate_t', 'void', 'FILE', 91 | # 'H5L_iterate2_t', 'H5AC_cache_config_t', 'off_t', 'H5P_prp_close_func_t', 92 | # 'H5F_libver_t', 'unsigned char', 'H5E_auto2_t', 'H5D_space_status_t', 93 | # 'H5S_seloper_t', 'H5E_direction_t', 'unsigned int', 'H5Z_filter_t', 94 | # 'H5D_fill_value_t', 'hsize_t', 'MPI_Info', 'H5_atclose_func_t', 'hid_t', 'long long', 95 | # 'H5FD_subfiling_config_t', 'H5L_type_t', 'H5I_type_t', 'H5F_fspace_strategy_t', 96 | # 'H5T_direction_t', 'H5FD_ioc_config_t', 'H5F_info1_t', 'H5R_ref_t', 'H5S_class_t', 97 | # 'hdset_reg_ref_t', 'H5ES_event_insert_func_t', 'H5FD_ros3_fapl_t', 'H5Z_SO_scale_type_t', 98 | # 'H5FD_mpio_chunk_opt_t', 'H5G_iterate_t', 'H5D_chunk_iter_op_t', 'short', 'uint32_t', 99 | # 'H5E_walk1_t', 'H5P_prp_delete_func_t', 'unsigned long long', 'size_t', 100 | # 'H5MM_allocate_t', 'H5E_auto1_t', 'H5P_prp_copy_func_t', 'H5FD_splitter_vfd_config_t', 101 | # 'H5D_fill_time_t', 'H5G_info_t', 'H5R_type_t', 'H5Z_EDC_t', 'H5A_info_t', 'uint64_t', 102 | # 'MPI_Comm', 'H5O_iterate1_t', 'H5E_walk2_t', 'H5O_token_t', 'hssize_t', 'H5E_minor_t', 103 | # 'double', 'H5AC_cache_image_config_t', 'H5P_prp_set_func_t', 'H5G_stat_t', 'H5D_scatter_func_t', 104 | # 'char *', 'H5_iter_order_t', 'H5FD_hdfs_fapl_t', 'H5F_mem_t', 'H5DS_iterate_t', 105 | # 'H5FD_file_image_callbacks_t' 106 | 107 | # NetCDF 4.9.2 Arg types: 108 | # 'NC_memio', 'nc_type', 'void', 'NC_Dispatch', 'unsigned', 'size_t', 'long long', 'char', 109 | # 'double', 'signed char', 'nc_vlen_t', 'MPI_Info', 'unsigned short', 'int', 'MPI_Comm', 110 | # 'float', 'unsigned int', 'ptrdiff_t', 'long', 'unsigned long long', 'unsigned char', 'short' 111 | 112 | if self.is_type_ord() or self.is_type_ptr(): 113 | # itoa 114 | if self.clean_arg_type in ["MPI_Offset", "short", "long", "long long", "unsigned long long", \ 115 | "unsigned short", "int", "unsigned int", "nc_type", \ 116 | "unsigned long", "uint32_t", "uint64_t", "hssize_t", \ 117 | "H5_index_t", "H5_iter_order_t", "H5F_scope_t", "hsize_t", "hid_t", "unsigned", "hbool_t", "size_t"]: 118 | # TODO how about ptrptr type? 119 | return "itoa(%s)" %(self.get_safe_star_ptr() if self.is_type_ptr() else self.clean_arg_name) 120 | # ftoa 121 | elif self.clean_arg_type in ["float", "double"]: 122 | # TODO how about ptrptr type? 123 | return "ftoa(%s)" %(self.get_safe_star_ptr() if self.is_type_ptr() else self.clean_arg_name) 124 | # strtoa 125 | elif self.clean_arg_type in ["char", "unsigned char", "char *", "signed char"]: 126 | if self.is_type_ptr(): 127 | return "strtoa(%s)" %(self.clean_arg_name) 128 | elif self.is_type_ptrptr(): 129 | return "strtoa(%s)" %(self.get_safe_star_ptr()) 130 | if self.clean_arg_type in []: 131 | # TODO: handle HDF5 args like bypasses[2] 132 | return "arrtostr(%s)" %(self.clean_arg_name) 133 | 134 | # ptoa 135 | # handle all the rest 136 | return "ptoa(%s%s)" %("&" if self.is_type_ord() else "", self.clean_arg_name) 137 | 138 | # TODO handle different MPI datatypes; comm2name, type2name, etc. 139 | # TODO int varids[] is handled wrong: itoa(*varids) 140 | 141 | 142 | class Func: 143 | def __init__(self, func_str): 144 | ret_func_str = func_str.split("(")[0].strip() 145 | space_idx = ret_func_str.rfind(" ") 146 | self.ret_type = ret_func_str[0:space_idx] 147 | self.func_name = ret_func_str[space_idx+1:] 148 | self.args = [] 149 | 150 | # now parse arguments 151 | args_str = func_str.split("(")[1].split(")")[0] 152 | self.args_str = args_str 153 | args_str = args_str.replace(", ", ",") 154 | for arg_str in args_str.split(","): 155 | if arg_str != "void": 156 | self.args.append(Arg(arg_str)) 157 | 158 | def get_wrapper_signature(self): 159 | s = "%s WRAPPER_NAME(%s)(%s)" %(self.ret_type, self.func_name, self.args_str) 160 | return s 161 | 162 | def get_arg_names(self): 163 | arg_names = [arg.clean_arg_name for arg in self.args] 164 | s = ", ".join(arg_names) 165 | return s 166 | 167 | def get_assemble_arg_list(self): 168 | s = [] 169 | for arg in self.args: 170 | s.append( arg.get_assemble_arg() ) 171 | return ",".join(s) 172 | 173 | def __str__(self): 174 | return "%s %s args: %d" %(self.ret_type, self.func_name, len(self.args)) 175 | 176 | 177 | def read_funcs(path): 178 | lines = [] 179 | with open(path) as file: 180 | lines = [line.rstrip() for line in file] 181 | 182 | funcs = [] 183 | for line in lines: 184 | func_str = " ".join(line.split()) # Substitute multiple whitespace with single whitespace 185 | f = Func(func_str) 186 | funcs.append(f) 187 | 188 | return funcs 189 | 190 | 191 | """ 192 | add in recorder-logger.h 193 | """ 194 | def print_func_names(funcs): 195 | func_name_set = set() 196 | s = "" 197 | i = 0 198 | for f in funcs: 199 | if f.func_name not in func_name_set: 200 | func_name_set.add(f.func_name) 201 | else: 202 | print("huh???%s" %f.func_name) 203 | s += ("\"" + f.func_name + "\"") 204 | i += 1 205 | if i % 3 == 0: 206 | s += ",\n" 207 | else: 208 | s += ", " 209 | print(s) 210 | 211 | 212 | """ 213 | add in lib/recorder-xxx.c (e.g., recorder-hdf5.c) 214 | """ 215 | def print_gotcha_wrappers(funcs): 216 | for f in funcs: 217 | s = f.get_wrapper_signature() + " {\n" 218 | s += "\tRECORDER_INTERCEPTOR_PROLOGUE(%s, %s, (%s));\n" %(f.ret_type, f.func_name, f.get_arg_names()) 219 | if len(f.args) == 0: 220 | s += "\tchar **args = NULL;\n" 221 | else: 222 | s += "\tchar **args = assemble_args_list(%d, %s);\n" %(len(f.args), f.get_assemble_arg_list()) 223 | s += "\tRECORDER_INTERCEPTOR_EPILOGUE(%d, args);\n" %(len(f.args)) 224 | s += "}\n" 225 | print(s) 226 | 227 | """ 228 | add in lib/recorder-gotcha.c 229 | """ 230 | def print_gotcha_bindings(funcs): 231 | for f in funcs: 232 | s = "GOTCHA_WRAP_ACTION(%s)," %(f.func_name) 233 | print(s) 234 | 235 | """ 236 | add in include/recorder-gotcha.h 237 | """ 238 | def print_gotcha_wrappees(funcs): 239 | for f in funcs: 240 | s = "GOTCHA_WRAP(%s, %s, (%s));" %(f.func_name, f.ret_type, f.args_str) 241 | print(s) 242 | 243 | if __name__ == "__main__": 244 | 245 | funcs = read_funcs(sys.argv[1]) 246 | #print("Arg type set:") 247 | #print(arg_type_set) 248 | 249 | # 1. add in include/recorder-logger.h 250 | #print_func_names(funcs) 251 | 252 | # 2. add in include/recorder-gotcha.h 253 | #print_gotcha_wrappees(funcs) 254 | 255 | # 3. add in lib/recorder-gotcha.c 256 | #print_gotcha_bindings(funcs) 257 | 258 | # 4. add in lib/recorder-xxx.c (e.g., recorder-hdf5.c) 259 | print_gotcha_wrappers(funcs) 260 | -------------------------------------------------------------------------------- /tools/generator/hdf5-new-funcs.txt: -------------------------------------------------------------------------------- 1 | herr_t H5Aclose_async (hid_t attr_id, hid_t es_id) 2 | hid_t H5Acreate_async (hid_t loc_id, const char *attr_name, hid_t type_id, hid_t space_id, hid_t acpl_id, hid_t aapl_id, hid_t es_id) 3 | hid_t H5Acreate_by_name_async (hid_t loc_id, const char *obj_name, const char *attr_name, hid_t type_id, hid_t space_id, hid_t acpl_id, hid_t aapl_id, hid_t lapl_id, hid_t es_id) 4 | herr_t H5Aexists_async (hid_t obj_id, const char *attr_name, hbool_t *exists, hid_t es_id) 5 | herr_t H5Aexists_by_name_async (hid_t loc_id, const char *obj_name, const char *attr_name, hbool_t *exists, hid_t lapl_id, hid_t es_id) 6 | hid_t H5Aopen_async (hid_t obj_id, const char *attr_name, hid_t aapl_id, hid_t es_id) 7 | hid_t H5Aopen_by_idx_async (hid_t loc_id, const char *obj_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t aapl_id, hid_t lapl_id, hid_t es_id) 8 | hid_t H5Aopen_by_name_async (hid_t loc_id, const char *obj_name, const char *attr_name, hid_t aapl_id, hid_t lapl_id, hid_t es_id) 9 | herr_t H5Aread_async (hid_t attr_id, hid_t dtype_id, void *buf, hid_t es_id) 10 | herr_t H5Arename_async (hid_t loc_id, const char *old_name, const char *new_name, hid_t es_id) 11 | herr_t H5Arename_by_name_async (hid_t loc_id, const char *obj_name, const char *old_attr_name, const char *new_attr_name, hid_t lapl_id, hid_t es_id) 12 | herr_t H5Awrite_async (hid_t attr_id, hid_t type_id, const void *buf, hid_t es_id) 13 | hid_t H5Dcreate_async (hid_t loc_id, const char *name, hid_t type_id, hid_t space_id, hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id, hid_t es_id) 14 | hid_t H5Dopen_async (hid_t loc_id, const char *name, hid_t dapl_id, hid_t es_id) 15 | hid_t H5Dget_space_async (hid_t dset_id, hid_t es_id) 16 | herr_t H5Dread_async (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, void *buf, hid_t es_id) 17 | herr_t H5Dread_multi_async (size_t count, hid_t dset_id[], hid_t mem_type_id[], hid_t mem_space_id[], hid_t file_space_id[], hid_t dxpl_id, void *buf[], hid_t es_id) 18 | herr_t H5Dwrite_async (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, const void *buf, hid_t es_id) 19 | herr_t H5Dwrite_multi_async (size_t count, hid_t dset_id[], hid_t mem_type_id[], hid_t mem_space_id[], hid_t file_space_id[], hid_t dxpl_id, const void *buf[], hid_t es_id) 20 | herr_t H5Dset_extent_async (hid_t dset_id, const hsize_t size[], hid_t es_id) 21 | herr_t H5Dclose_async (hid_t dset_id, hid_t es_id) 22 | hid_t H5Fcreate_async (const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t es_id) 23 | hid_t H5Fopen_async (const char *filename, unsigned flags, hid_t access_plist, hid_t es_id) 24 | hid_t H5Freopen_async (hid_t file_id, hid_t es_id) 25 | herr_t H5Fflush_async (hid_t object_id, H5F_scope_t scope, hid_t es_id) 26 | herr_t H5Fclose_async (hid_t file_id, hid_t es_id) 27 | hid_t H5Gcreate_async (hid_t loc_id, const char *name, hid_t lcpl_id, hid_t gcpl_id, hid_t gapl_id, hid_t es_id) 28 | hid_t H5Gopen_async (hid_t loc_id, const char *name, hid_t gapl_id, hid_t es_id) 29 | herr_t H5Gget_info_async (hid_t loc_id, H5G_info_t *ginfo, hid_t es_id) 30 | herr_t H5Gget_info_by_name_async (hid_t loc_id, const char *name, H5G_info_t *ginfo, hid_t lapl_id, hid_t es_id) 31 | herr_t H5Gget_info_by_idx_async (hid_t loc_id, const char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, H5G_info_t *ginfo, hid_t lapl_id, hid_t es_id) 32 | herr_t H5Gclose_async (hid_t group_id, hid_t es_id) 33 | herr_t H5Lcreate_hard_async (hid_t cur_loc_id, const char *cur_name, hid_t new_loc_id, const char *new_name, hid_t lcpl_id, hid_t lapl_id, hid_t es_id) 34 | herr_t H5Lcreate_soft_async (const char *link_target, hid_t link_loc_id, const char *link_name, hid_t lcpl_id, hid_t lapl_id, hid_t es_id) 35 | herr_t H5Ldelete_async (hid_t loc_id, const char *name, hid_t lapl_id, hid_t es_id) 36 | herr_t H5Ldelete_by_idx_async (hid_t loc_id, const char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t lapl_id, hid_t es_id) 37 | herr_t H5Lexists_async (hid_t loc_id, const char *name, hbool_t *exists, hid_t lapl_id, hid_t es_id) 38 | herr_t H5Literate_async (hid_t group_id, H5_index_t idx_type, H5_iter_order_t order, hsize_t *idx_p, H5L_iterate2_t op, void *op_data, hid_t es_id) 39 | hid_t H5Mcreate_async (hid_t loc_id, const char *name, hid_t key_type_id, hid_t val_type_id, hid_t lcpl_id, hid_t mcpl_id, hid_t mapl_id, hid_t es_id) 40 | hid_t H5Mopen_async (hid_t loc_id, const char *name, hid_t mapl_id, hid_t es_id) 41 | herr_t H5Mclose_async (hid_t map_id, hid_t es_id) 42 | herr_t H5Mput_async (hid_t map_id, hid_t key_mem_type_id, const void *key, hid_t val_mem_type_id, const void *value, hid_t dxpl_id, hid_t es_id) 43 | herr_t H5Mget_async (hid_t map_id, hid_t key_mem_type_id, const void *key, hid_t val_mem_type_id, void *value, hid_t dxpl_id, hid_t es_id) 44 | hid_t H5Oopen_async (hid_t loc_id, const char *name, hid_t lapl_id, hid_t es_id) 45 | hid_t H5Oopen_by_idx_async (hid_t loc_id, const char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t lapl_id, hid_t es_id) 46 | herr_t H5Oget_info_by_name_async (hid_t loc_id, const char *name, H5O_info2_t *oinfo, unsigned fields, hid_t lapl_id, hid_t es_id) 47 | herr_t H5Ocopy_async (hid_t src_loc_id, const char *src_name, hid_t dst_loc_id, const char *dst_name, hid_t ocpypl_id, hid_t lcpl_id, hid_t es_id) 48 | herr_t H5Oclose_async (hid_t object_id, hid_t es_id) 49 | herr_t H5Oflush_async (hid_t obj_id, hid_t es_id) 50 | herr_t H5Orefresh_async (hid_t oid, hid_t es_id) 51 | hid_t H5Ropen_object_async (unsigned app_line, H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_id, hid_t es_id) 52 | hid_t H5Ropen_region_async (H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_id, hid_t es_id) 53 | hid_t H5Ropen_attr_async (H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t aapl_id, hid_t es_id) 54 | -------------------------------------------------------------------------------- /tools/meta-ops-checker.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "reader.h" 7 | 8 | #define max(a,b) (((a)>(b))?(a):(b)) 9 | 10 | Record callers[4]; 11 | 12 | // [func_id] = 3 by app 13 | // [func_id] = 2 by hdf5 14 | // [func_id] = 1 by mpi 15 | // [func_id] = 0 not used 16 | int meta_op_caller[256] = {0}; 17 | 18 | 19 | bool ignore_function(int func_id) { 20 | const char *func = func_list[func_id]; 21 | if(strstr(func, "MPI") || strstr(func, "H5")) 22 | return true; 23 | if(strstr(func, "dir") || strstr(func, "link")) 24 | return false; 25 | 26 | const char* ignored[] = {"read", "write", "seek", "tell", "open", "close", "sync"}; 27 | int i; 28 | for(i = 0; i < sizeof(ignored)/sizeof(char*); i++) { 29 | if(strstr(func, ignored[i])) 30 | return true; 31 | } 32 | 33 | return false; 34 | } 35 | 36 | void test(Record *records, int len, RecorderReader *reader) { 37 | int i; 38 | 39 | int depth = 0; 40 | callers[0].tend = 0; 41 | 42 | for(i = 0; i < len; i++) { 43 | Record record = records[i]; 44 | 45 | while(depth > 0) { 46 | if(record.tstart >= callers[depth].tend) 47 | depth--; 48 | else 49 | break; 50 | } 51 | 52 | Record caller = callers[depth]; 53 | callers[++depth] = record; 54 | 55 | if(!ignore_function(record.func_id)) { 56 | if(depth > 1) { 57 | const char* caller_func = reader->func_list[caller.func_id]; 58 | if(strstr(caller_func, "MPI")) 59 | meta_op_caller[record.func_id] = max(meta_op_caller[record.func_id], 1); 60 | if(strstr(caller_func, "H5")) 61 | meta_op_caller[record.func_id] = max(meta_op_caller[record.func_id], 2); 62 | } else { 63 | meta_op_caller[record.func_id] = 3; 64 | } 65 | 66 | } 67 | 68 | } 69 | } 70 | 71 | void print_metadata_ops(RecorderReader *reader) { 72 | printf("\n\n"); 73 | const char* included_funcs[] = { 74 | "__xstat", "__xstat64", "__lxstat", "__lxstat64", "__fxstat", "__fxstat64", 75 | "mmap", "getcwd", "mkdir", "chdir", "opendir", "readdir", "closedir", 76 | "umask", "unlink", "readlink", "access", 77 | "ftruncate", "fileno", "faccessat" 78 | }; 79 | int i; 80 | int func_id; 81 | 82 | for(i = 0; i < sizeof(included_funcs)/sizeof(char*); i++) { 83 | for(func_id = 0; func_id < 100; func_id++) { 84 | const char *f = reader->func_list[func_id]; 85 | if(0 == strcmp(f, included_funcs[i])) { 86 | printf("%d, ", meta_op_caller[func_id]); 87 | break; 88 | } 89 | } 90 | } 91 | printf("\n"); 92 | } 93 | 94 | 95 | int main(int argc, char **argv) { 96 | 97 | RecorderReader reader; 98 | recorder_read_traces(argv[1], &reader); 99 | 100 | int rank; 101 | for(rank = 0; rank < reader.RGD.total_ranks; rank++) { 102 | Record* records = reader.records[rank]; 103 | test(records, reader.RLDs[rank].total_records, &reader); 104 | } 105 | 106 | int func_id; 107 | for(func_id = 0; func_id < 100; func_id++) { 108 | if(!ignore_function(func_id) && meta_op_caller[func_id]) 109 | printf("func: %s, caller: %d\n", reader.func_list[func_id], meta_op_caller[func_id]); 110 | } 111 | print_metadata_ops(&reader); 112 | 113 | release_resources(&reader); 114 | return 0; 115 | } 116 | -------------------------------------------------------------------------------- /tools/reader-cst-cfg.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "./reader-private.h" 8 | 9 | void reader_free_cst(CST* cst) { 10 | for(int i = 0; i < cst->entries; i++) 11 | free(cst->cs_list[i].key); 12 | free(cst->cs_list); 13 | } 14 | 15 | void reader_free_cfg(CFG* cfg) { 16 | RuleHash *r, *tmp; 17 | HASH_ITER(hh, cfg->cfg_head, r, tmp) { 18 | HASH_DEL(cfg->cfg_head, r); 19 | free(r->rule_body); 20 | free(r); 21 | } 22 | } 23 | 24 | 25 | void reader_decode_cst_2_3(RecorderReader *reader, int rank, CST *cst) { 26 | cst->rank = rank; 27 | char cst_filename[1096] = {0}; 28 | sprintf(cst_filename, "%s/%d.cst", reader->logs_dir, rank); 29 | 30 | FILE* f = fopen(cst_filename, "rb"); 31 | 32 | int key_len; 33 | fread(&cst->entries, sizeof(int), 1, f); 34 | 35 | cst->cs_list = malloc(cst->entries * sizeof(CallSignature)); 36 | 37 | for(int i = 0; i < cst->entries; i++) { 38 | fread(&cst->cs_list[i].terminal_id, sizeof(int), 1, f); 39 | fread(&cst->cs_list[i].key_len, sizeof(int), 1, f); 40 | 41 | cst->cs_list[i].key = malloc(cst->cs_list[i].key_len); 42 | fread(cst->cs_list[i].key, 1, cst->cs_list[i].key_len, f); 43 | 44 | assert(cst->cs_list[i].terminal_id < cst->entries); 45 | } 46 | fclose(f); 47 | } 48 | 49 | void reader_decode_cfg_2_3(RecorderReader *reader, int rank, CFG* cfg) { 50 | cfg->rank = rank; 51 | char cfg_filename[1096] = {0}; 52 | sprintf(cfg_filename, "%s/%d.cfg", reader->logs_dir, rank); 53 | 54 | FILE* f = fopen(cfg_filename, "rb"); 55 | 56 | fread(&cfg->rules, sizeof(int), 1, f); 57 | 58 | cfg->cfg_head = NULL; 59 | for(int i = 0; i < cfg->rules; i++) { 60 | RuleHash *rule = malloc(sizeof(RuleHash)); 61 | 62 | fread(&(rule->rule_id), sizeof(int), 1, f); 63 | fread(&(rule->symbols), sizeof(int), 1, f); 64 | 65 | rule->rule_body = (int*) malloc(sizeof(int)*rule->symbols*2); 66 | fread(rule->rule_body, sizeof(int), rule->symbols*2, f); 67 | HASH_ADD_INT(cfg->cfg_head, rule_id, rule); 68 | } 69 | fclose(f); 70 | } 71 | 72 | void reader_decode_cst(int rank, void* buf, CST* cst) { 73 | cst->rank = rank; 74 | 75 | memcpy(&cst->entries, buf, sizeof(int)); 76 | buf += sizeof(int); 77 | 78 | // cst->cs_list will be stored in the terminal_id order. 79 | cst->cs_list = malloc(cst->entries * sizeof(CallSignature)); 80 | 81 | for(int i = 0; i < cst->entries; i++) { 82 | 83 | int terminal_id; 84 | memcpy(&terminal_id, buf, sizeof(int)); 85 | buf += sizeof(int); 86 | assert(terminal_id < cst->entries); 87 | 88 | CallSignature* cs = &(cst->cs_list[terminal_id]); 89 | cs->terminal_id = terminal_id; 90 | 91 | memcpy(&cs->rank, buf, sizeof(int)); 92 | buf += sizeof(int); 93 | memcpy(&cs->key_len, buf, sizeof(int)); 94 | buf += sizeof(int); 95 | memcpy(&cs->count, buf, sizeof(int)); 96 | buf += sizeof(int); 97 | 98 | cs->key = malloc(cs->key_len); 99 | memcpy(cs->key, buf, cs->key_len); 100 | buf += cs->key_len; 101 | } 102 | } 103 | 104 | void reader_decode_cfg(int rank, void* buf, CFG* cfg) { 105 | 106 | cfg->rank = rank; 107 | 108 | memcpy(&cfg->rules, buf, sizeof(int)); 109 | buf += sizeof(int); 110 | 111 | cfg->cfg_head = NULL; 112 | for(int i = 0; i < cfg->rules; i++) { 113 | RuleHash *rule = malloc(sizeof(RuleHash)); 114 | 115 | memcpy(&rule->rule_id, buf, sizeof(int)); 116 | buf += sizeof(int); 117 | memcpy(&rule->symbols, buf, sizeof(int)); 118 | buf += sizeof(int); 119 | 120 | rule->rule_body = (int*) malloc(sizeof(int)*rule->symbols*2); 121 | memcpy(rule->rule_body, buf, sizeof(int)*rule->symbols*2); 122 | buf += sizeof(int)*rule->symbols*2; 123 | HASH_ADD_INT(cfg->cfg_head, rule_id, rule); 124 | } 125 | } 126 | 127 | CST* reader_get_cst(RecorderReader* reader, int rank) { 128 | CST* cst = reader->csts[rank]; 129 | return cst; 130 | } 131 | 132 | CFG* reader_get_cfg(RecorderReader* reader, int rank) { 133 | CFG* cfg; 134 | if (reader->metadata.interprocess_compression) 135 | cfg = reader->cfgs[reader->ug_ids[rank]]; 136 | else 137 | cfg = reader->cfgs[rank]; 138 | return cfg; 139 | } 140 | 141 | // Caller needs to free the record after use 142 | // by using recorder_free_record() call. 143 | Record* reader_cs_to_record(CallSignature *cs) { 144 | 145 | Record *record = malloc(sizeof(Record)); 146 | 147 | char* key = cs->key; 148 | 149 | int pos = 0; 150 | memcpy(&record->tid, key+pos, sizeof(pthread_t)); 151 | pos += sizeof(pthread_t); 152 | memcpy(&record->func_id, key+pos, sizeof(record->func_id)); 153 | pos += sizeof(record->func_id); 154 | memcpy(&record->call_depth, key+pos, sizeof(record->call_depth)); 155 | pos += sizeof(record->call_depth); 156 | memcpy(&record->arg_count, key+pos, sizeof(record->arg_count)); 157 | pos += sizeof(record->arg_count); 158 | 159 | record->args = malloc(sizeof(char*) * record->arg_count); 160 | 161 | int arg_strlen; 162 | memcpy(&arg_strlen, key+pos, sizeof(int)); 163 | pos += sizeof(int); 164 | 165 | char* arg_str = key+pos; 166 | int ai = 0; 167 | int start = 0; 168 | for(int i = 0; i < arg_strlen; i++) { 169 | if(arg_str[i] == ' ') { 170 | record->args[ai++] = strndup(arg_str+start, (i-start)); 171 | start = i + 1; 172 | } 173 | } 174 | 175 | assert(ai == record->arg_count); 176 | return record; 177 | } 178 | 179 | -------------------------------------------------------------------------------- /tools/reader-private.h: -------------------------------------------------------------------------------- 1 | #ifndef _RECORDER_READER_PRIVATE_H_ 2 | #define _RECORDER_READER_PRIVATE_H_ 3 | #include 4 | #include "reader.h" 5 | 6 | #define POSIX_SEMANTICS 0 7 | #define COMMIT_SEMANTICS 1 8 | #define SESSION_SEMANTICS 2 9 | 10 | #ifdef __cplusplus 11 | extern "C" 12 | { 13 | #endif 14 | 15 | 16 | typedef struct Interval_t { 17 | int rank; 18 | int seqId; // The sequence id of the I/O call 19 | double tstart; 20 | size_t offset; 21 | size_t count; 22 | bool isRead; 23 | char mpifh[10]; 24 | } Interval; 25 | 26 | /* Per-file intervals 27 | * 28 | */ 29 | typedef struct IntervalsMap_t { 30 | char* filename; 31 | size_t num_intervals; 32 | Interval *intervals; // Pointer to Interval, copied from vector 33 | } IntervalsMap; 34 | 35 | 36 | 37 | /** 38 | * Read CST and CFG from files to RecorderReader 39 | * 40 | * With interprocess compression, we have 41 | * one merged CST and multiple CFG files. 42 | * Each CFG file stores a unique grammar. 43 | * 44 | * Without interprocess compression, we have 45 | * one CST and one CFG file per process. 46 | * 47 | * ! These two functions should only be used internally 48 | * recorder_get_cst_cfg() can be used to perform 49 | * custom tasks with CST and CFG 50 | */ 51 | void reader_decode_cst_2_3(RecorderReader *reader, int rank, CST *cst); 52 | void reader_decode_cfg_2_3(RecorderReader *reader, int rank, CFG *cfg); 53 | void reader_decode_cst(int rank, void* buf, CST* cst); 54 | void reader_decode_cfg(int rank, void* buf, CFG* cfg); 55 | void reader_free_cst(CST *cst); 56 | void reader_free_cfg(CFG *cfg); 57 | CST* reader_get_cst(RecorderReader* reader, int rank); 58 | CFG* reader_get_cfg(RecorderReader* reader, int rank); 59 | 60 | Record* reader_cs_to_record(CallSignature *cs); 61 | 62 | IntervalsMap* build_offset_intervals(RecorderReader *reader, int *num_files); 63 | 64 | #ifdef __cplusplus 65 | } 66 | #endif 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /tools/reader.h: -------------------------------------------------------------------------------- 1 | #ifndef _RECORDER_READER_H_ 2 | #define _RECORDER_READER_H_ 3 | #include 4 | #include "recorder-logger.h" 5 | 6 | #define POSIX_SEMANTICS 0 7 | #define COMMIT_SEMANTICS 1 8 | #define SESSION_SEMANTICS 2 9 | 10 | #ifdef __cplusplus 11 | extern "C" 12 | { 13 | #endif 14 | 15 | typedef struct CST_t { 16 | int rank; 17 | int entries; 18 | CallSignature *cs_list; // CallSignature is defined in recorder-logger.h 19 | } CST; 20 | 21 | typedef struct RuleHash_t { 22 | int rule_id; 23 | int *rule_body; // 2i+0: val of symbol i, 2i+1: exp of symbol i 24 | int symbols; // There are a total of 2*symbols integers in the rule body 25 | UT_hash_handle hh; 26 | } RuleHash; 27 | 28 | typedef struct CFG_t { 29 | int rank; 30 | int rules; 31 | RuleHash* cfg_head; 32 | } CFG; 33 | 34 | typedef struct RecorderReader_t { 35 | 36 | RecorderMetadata metadata; 37 | 38 | int supported_funcs; // total number of supported functions, it is the length of func_list 39 | char** func_list; 40 | char logs_dir[1024]; 41 | 42 | int mpi_start_idx; 43 | int hdf5_start_idx; 44 | int pnetcdf_start_idx; 45 | int netcdf_start_idx; 46 | 47 | double prev_tstart; 48 | 49 | // in the case of metadata.interprocess_compression = true 50 | // store the unique grammars in ugs. 51 | // cfgs[rank] = ugs[ug_ids[rank]]; 52 | int num_ugs; // number of unique grammars 53 | int* ug_ids; // index of unique grammar in cfgs 54 | CFG** ugs; // store actual grammars 55 | 56 | // in the case of metadata.interprocess_compression = false 57 | // we have one file for each rank's cst and one file 58 | // for each rank's cfg. We directly store them in csts[rnak] 59 | // and cfgs[rank]. 60 | CST** csts; 61 | CFG** cfgs; 62 | 63 | int trace_version_major; 64 | int trace_version_minor; 65 | } RecorderReader; 66 | 67 | 68 | /** 69 | * Similar but simplified Record structure 70 | * for use by recorder-viz 71 | */ 72 | typedef struct PyRecord_t { 73 | double tstart, tend; 74 | unsigned char call_depth; 75 | int func_id; 76 | int tid; 77 | unsigned char arg_count; 78 | char **args; 79 | } PyRecord; 80 | 81 | 82 | /** 83 | * Simplified Record structure 84 | * for use by the VerifyIO python 85 | * code. 86 | * 87 | * Note in this structure, char** args 88 | * no longer store every argument, but 89 | * only those needed by VerifyIO 90 | */ 91 | typedef struct VerifyIORecord_t { 92 | int func_id; 93 | unsigned char call_depth; 94 | unsigned char arg_count; 95 | char** args; 96 | } VerifyIORecord; 97 | 98 | 99 | void recorder_init_reader(const char* logs_dir, RecorderReader *reader); 100 | void recorder_free_reader(RecorderReader *reader); 101 | 102 | void recorder_free_record(Record* r); 103 | 104 | /** 105 | * This function reads all records of a rank 106 | * 107 | * For each record decoded, the user_op() function 108 | * will be called with the decoded record 109 | * 110 | * void user_op(Record *r, void* user_arg); 111 | * void* user_arg can be used to pass in user argument. 112 | * 113 | */ 114 | void recorder_decode_records(RecorderReader* reader, int rank, 115 | void (*user_op)(Record* r, void* user_arg), void* user_arg); 116 | // used by to implement read_all_records for recorder_viz 117 | void recorder_decode_records2(RecorderReader* reader, int rank, 118 | void (*user_op)(Record* r, void* user_arg), void* user_arg); 119 | 120 | const char* recorder_get_func_name(RecorderReader* reader, Record* record); 121 | 122 | /* 123 | * Return one of the follows (mutual exclusive) : 124 | * - RECORDER_POSIX 125 | * - RECORDER_MPIIO 126 | * - RECORDER_MPI 127 | * - RECORDER_HDF5 128 | * - RECORDER_FTRACE 129 | */ 130 | int recorder_get_func_type(RecorderReader* reader, Record* record); 131 | 132 | #ifdef __cplusplus 133 | } 134 | #endif 135 | 136 | #endif 137 | -------------------------------------------------------------------------------- /tools/recorder-summary.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "reader.h" 11 | #include "reader-private.h" 12 | #include "recorder-logger.h" 13 | 14 | 15 | 16 | void print_cst(RecorderReader* reader, CST* cst) { 17 | printf("\nBelow are the unique call signatures: \n"); 18 | 19 | for(int i = 0; i < cst->entries; i++) { 20 | Record* record = reader_cs_to_record(&cst->cs_list[i]); 21 | 22 | const char* func_name = recorder_get_func_name(reader, record); 23 | printf("%s(", func_name); 24 | 25 | bool user_func = (recorder_get_func_type(reader, record) == RECORDER_USER_FUNCTION); 26 | for(int arg_id = 0; !user_func && arg_id < record->arg_count; arg_id++) { 27 | char *arg = record->args[arg_id]; 28 | printf(" %s", arg); 29 | } 30 | 31 | printf(" ), count: %d\n", cst->cs_list[i].count); 32 | recorder_free_record(record); 33 | } 34 | } 35 | 36 | void print_statistics(RecorderReader* reader, CST* cst) { 37 | 38 | int* unique_signature = (int*) malloc(sizeof(int)*reader->supported_funcs); 39 | int* call_count = (int*) malloc(sizeof(int)*reader->supported_funcs); 40 | memset(unique_signature, 0, sizeof(int)*reader->supported_funcs); 41 | memset(call_count, 0, sizeof(int)*reader->supported_funcs); 42 | 43 | int mpi_count = 0, mpiio_count = 0, netcdf_count = 0; 44 | int pnetcdf_count = 0, hdf5_count = 0, posix_count = 0; 45 | 46 | for(int i = 0; i < cst->entries; i++) { 47 | Record* record = reader_cs_to_record(&cst->cs_list[i]); 48 | const char* func_name = recorder_get_func_name(reader, record); 49 | 50 | int type = recorder_get_func_type(reader, record); 51 | if(type == RECORDER_MPI) 52 | mpi_count += cst->cs_list[i].count; 53 | if(type == RECORDER_MPIIO) 54 | mpiio_count += cst->cs_list[i].count; 55 | if(type == RECORDER_HDF5) 56 | hdf5_count += cst->cs_list[i].count; 57 | if(type == RECORDER_PNETCDF) 58 | pnetcdf_count += cst->cs_list[i].count; 59 | if(type == RECORDER_NETCDF) 60 | netcdf_count += cst->cs_list[i].count; 61 | if(type == RECORDER_POSIX) 62 | posix_count += cst->cs_list[i].count; 63 | 64 | unique_signature[record->func_id]++; 65 | call_count[record->func_id] += cst->cs_list[i].count; 66 | 67 | recorder_free_record(record); 68 | } 69 | long int total = posix_count + mpi_count + mpiio_count + hdf5_count + pnetcdf_count; 70 | printf("Total: %ld\nPOSIX: %d\nMPI: %d\nMPI-IO: %d\nHDF5: %d\nPnetCDF: %d\nNetCDF: %d\n", 71 | total, posix_count, mpi_count, mpiio_count, hdf5_count, pnetcdf_count, netcdf_count); 72 | 73 | printf("\n%-25s %18s %18s\n", "Func", "Unique Signature", "Total Call Count"); 74 | for(int i = 0; i < reader->supported_funcs; i++) { 75 | if(unique_signature[i] > 0) { 76 | printf("%-25s %18d %18d\n", reader->func_list[i], unique_signature[i], call_count[i]); 77 | } 78 | } 79 | 80 | free(unique_signature); 81 | free(call_count); 82 | } 83 | 84 | void print_metadata(RecorderReader* reader) { 85 | RecorderMetadata* meta = &(reader->metadata); 86 | 87 | time_t t = meta->start_ts; 88 | struct tm *lt = localtime(&t); 89 | char tmbuf[64]; 90 | strftime(tmbuf, sizeof(tmbuf), "%Y-%m-%d %H:%M:%S", lt); 91 | 92 | printf("========Recorder Tracing Parameters========\n"); 93 | printf("Tracing start time: %s\n", tmbuf); 94 | printf("Total processes: %d\n", meta->total_ranks); 95 | printf("POSIX tracing: %s\n", meta->posix_tracing?"Enabled":"Disabled"); 96 | printf("MPI tracing: %s\n", meta->mpi_tracing?"Enabled":"Disabled"); 97 | printf("MPI-IO tracing: %s\n", meta->mpiio_tracing?"Enabled":"Disabled"); 98 | printf("HDF5 tracing: %s\n", meta->hdf5_tracing?"Enabled":"Disabled"); 99 | printf("PnetCDF tracing: %s\n", meta->pnetcdf_tracing?"Enabled":"Disabled"); 100 | printf("NetCDF tracing: %s\n", meta->netcdf_tracing?"Enabled":"Disabled"); 101 | printf("Store thread id: %s\n", meta->store_tid?"True":"False"); 102 | printf("Store call depth: %s\n", meta->store_call_depth?"True":"False"); 103 | printf("Timestamp compression: %s\n", meta->ts_compression?"True":"False"); 104 | printf("Interprocess compression: %s\n", meta->interprocess_compression?"True":"False"); 105 | printf("Intraprocess pattern recognition: %s\n", meta->intraprocess_pattern_recognition?"True":"False"); 106 | printf("Interprocess pattern recognition: %s\n", meta->interprocess_pattern_recognition?"True":"False"); 107 | printf("===========================================\n\n"); 108 | } 109 | 110 | 111 | int main(int argc, char **argv) { 112 | bool show_cst = false; 113 | int opt; 114 | while ((opt = getopt(argc, argv, "a")) != -1) { 115 | switch(opt) { 116 | case 'a': 117 | show_cst = true; 118 | break; 119 | defaut: 120 | fprintf(stderr, "Usage: %s [-a] [path to traces]\n", argv[0]); 121 | } 122 | } 123 | 124 | RecorderReader reader; 125 | recorder_init_reader(argv[optind], &reader); 126 | 127 | CST* cst = reader_get_cst(&reader, 0); 128 | print_metadata(&reader); 129 | print_statistics(&reader, cst); 130 | 131 | if (show_cst) { 132 | print_cst(&reader, cst); 133 | } 134 | 135 | recorder_free_reader(&reader); 136 | 137 | return 0; 138 | } 139 | -------------------------------------------------------------------------------- /tools/recorder2parquet.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "reader.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | struct ParquetWriter { 14 | int rank; 15 | char* base_file; 16 | int64_t index; 17 | arrow::Int32Builder indexBuilder, categoryBuilder, rankBuilder, threadBuilder, arg_countBuilder, levelBuilder; 18 | arrow::FloatBuilder tstartBuilder, tendBuilder; 19 | arrow::StringBuilder func_idBuilder, args_Builder[10]; 20 | 21 | std::shared_ptr indexArray, categoryArray, rankArray, threadArray, arg_countArray, levelArray, tstartArray, tendArray, 22 | func_idArray, argsArray[10]; 23 | 24 | std::shared_ptr schema; 25 | const int64_t chunk_size = 1024; 26 | const int64_t NUM_ROWS = 1024*1024*64; // 1B 27 | int64_t row_group = 0; 28 | ParquetWriter(int, char*); 29 | 30 | void finish(); 31 | }; 32 | 33 | ParquetWriter::ParquetWriter(int _rank, char* _path) { 34 | rank = _rank; 35 | row_group = 0; 36 | 37 | base_file = _path; 38 | schema = arrow::schema( 39 | {arrow::field("index", arrow::int32()), arrow::field("rank", arrow::int32()), 40 | arrow::field("thread_id", arrow::int32()), arrow::field("cat", arrow::int32()), 41 | arrow::field("tstart", arrow::float32()), arrow::field("tend", arrow::float32()), 42 | arrow::field("func_id", arrow::utf8()), arrow::field("level", arrow::int32()), 43 | arrow::field("arg_count", arrow::int32()), arrow::field("args_1", arrow::utf8()), 44 | arrow::field("args_2", arrow::utf8()), arrow::field("args_3", arrow::utf8()), 45 | arrow::field("args_4", arrow::utf8()), arrow::field("args_5", arrow::utf8()), 46 | arrow::field("args_6", arrow::utf8()), arrow::field("args_7", arrow::utf8()), 47 | arrow::field("args_8", arrow::utf8()), arrow::field("args_9", arrow::utf8()), 48 | arrow::field("args_10", arrow::utf8())}); 49 | index = 0; 50 | 51 | indexBuilder = arrow::Int32Builder(); 52 | indexArray.reset(); 53 | rankBuilder = arrow::Int32Builder(); 54 | rankArray.reset(); 55 | threadBuilder = arrow::Int32Builder(); 56 | threadArray.reset(); 57 | categoryBuilder = arrow::Int32Builder(); 58 | categoryArray.reset(); 59 | arg_countBuilder = arrow::Int32Builder(); 60 | arg_countArray.reset(); 61 | levelBuilder = arrow::Int32Builder(); 62 | levelArray.reset(); 63 | 64 | tstartBuilder = arrow::FloatBuilder(); 65 | tstartArray.reset(); 66 | tendBuilder = arrow::FloatBuilder(); 67 | tendArray.reset(); 68 | 69 | 70 | func_idBuilder = arrow::StringBuilder(); 71 | func_idArray.reset(); 72 | for (int i =0; i< 10; i++){ 73 | args_Builder[i] = arrow::StringBuilder(); 74 | argsArray[i].reset(); 75 | } 76 | } 77 | 78 | void ParquetWriter::finish(void) { 79 | indexBuilder.Finish(&indexArray); 80 | rankBuilder.Finish(&rankArray); 81 | threadBuilder.Finish(&threadArray); 82 | tstartBuilder.Finish(&tstartArray); 83 | tendBuilder.Finish(&tendArray); 84 | func_idBuilder.Finish(&func_idArray); 85 | levelBuilder.Finish(&levelArray); 86 | categoryBuilder.Finish(&categoryArray); 87 | arg_countBuilder.Finish(&arg_countArray); 88 | for (int arg_id = 0; arg_id < 10; arg_id++) { 89 | args_Builder[arg_id].Finish(&argsArray[arg_id]); 90 | } 91 | 92 | auto table = arrow::Table::Make(schema, {indexArray, rankArray,threadArray,categoryArray, 93 | tstartArray, tendArray, 94 | func_idArray, levelArray, arg_countArray , 95 | argsArray[0], argsArray[1], argsArray[2], 96 | argsArray[3], argsArray[4], argsArray[5], 97 | argsArray[6], argsArray[7], argsArray[8], 98 | argsArray[9] }); 99 | char path[256]; 100 | sprintf(path, "%s_%d.parquet" , base_file, row_group); 101 | PARQUET_ASSIGN_OR_THROW(auto outfile, arrow::io::FileOutputStream::Open(path)); 102 | PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), outfile, 1024)); 103 | } 104 | 105 | RecorderReader reader; 106 | 107 | void handle_one_record(Record* record, void* arg) { 108 | ParquetWriter *writer = (ParquetWriter*) arg; 109 | 110 | writer->rankBuilder.Append(writer->rank); 111 | writer->threadBuilder.Append(record->tid); 112 | writer->tstartBuilder.Append(record->tstart); 113 | writer->tendBuilder.Append(record->tend); 114 | int cat = recorder_get_func_type(&reader, record); 115 | if (cat == RECORDER_FTRACE){ 116 | writer->func_idBuilder.Append(record->args[0]); 117 | record->arg_count = 0; 118 | }else { 119 | writer->func_idBuilder.Append(recorder_get_func_name(&reader, record)); 120 | } 121 | writer->categoryBuilder.Append(cat); 122 | writer->levelBuilder.Append(record->level); 123 | writer->arg_countBuilder.Append(record->arg_count); 124 | 125 | for (int arg_id = 0; arg_id < 10; arg_id++) { 126 | if(arg_id < record->arg_count) 127 | writer->args_Builder[arg_id].Append(record->args[arg_id]); 128 | else 129 | writer->args_Builder[arg_id].Append(""); 130 | } 131 | writer->index ++; 132 | writer->indexBuilder.Append(writer->index); 133 | if(writer->index % writer->NUM_ROWS == 0) { 134 | writer->indexBuilder.Finish(&writer->indexArray); 135 | writer->rankBuilder.Finish(&writer->rankArray); 136 | writer->threadBuilder.Finish(&writer->threadArray); 137 | writer->categoryBuilder.Finish(&writer->categoryArray); 138 | writer->tstartBuilder.Finish(&writer->tstartArray); 139 | writer->tendBuilder.Finish(&writer->tendArray); 140 | writer->func_idBuilder.Finish(&writer->func_idArray); 141 | writer->levelBuilder.Finish(&writer->levelArray); 142 | writer->arg_countBuilder.Finish(&writer->arg_countArray); 143 | for (int arg_id = 0; arg_id < 10; arg_id++) { 144 | writer->args_Builder[arg_id].Finish(&writer->argsArray[arg_id]); 145 | } 146 | 147 | auto table = arrow::Table::Make(writer->schema, {writer->indexArray, writer->rankArray,writer->threadArray, writer->categoryArray, 148 | writer->tstartArray, writer->tendArray, 149 | writer->func_idArray, writer->levelArray, writer->arg_countArray , 150 | writer->argsArray[0], writer->argsArray[1], writer->argsArray[2], 151 | writer->argsArray[3], writer->argsArray[4], writer->argsArray[5], 152 | writer->argsArray[6], writer->argsArray[7], writer->argsArray[8], 153 | writer->argsArray[9] }); 154 | char path[256]; 155 | sprintf(path, "%s_%d.parquet" , writer->base_file, writer->row_group); 156 | PARQUET_ASSIGN_OR_THROW(auto outfile, arrow::io::FileOutputStream::Open(path)); 157 | PARQUET_THROW_NOT_OK(parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), outfile, 1024*1024*128)); 158 | 159 | writer->row_group++; 160 | writer->indexBuilder = arrow::Int32Builder(); 161 | writer->indexArray.reset(); 162 | writer->rankBuilder = arrow::Int32Builder(); 163 | writer->rankArray.reset(); 164 | writer->threadBuilder = arrow::Int32Builder(); 165 | writer->threadArray.reset(); 166 | writer->categoryBuilder = arrow::Int32Builder(); 167 | writer->categoryArray.reset(); 168 | writer->arg_countBuilder = arrow::Int32Builder(); 169 | writer->arg_countArray.reset(); 170 | writer->levelBuilder = arrow::Int32Builder(); 171 | writer->levelArray.reset(); 172 | 173 | writer->tstartBuilder = arrow::FloatBuilder(); 174 | writer->tstartArray.reset(); 175 | writer->tendBuilder = arrow::FloatBuilder(); 176 | writer->tendArray.reset(); 177 | 178 | 179 | writer->func_idBuilder = arrow::StringBuilder(); 180 | writer->func_idArray.reset(); 181 | for (int i =0; i< 10; i++){ 182 | writer->args_Builder[i] = arrow::StringBuilder(); 183 | writer->argsArray[i].reset(); 184 | } 185 | } 186 | } 187 | 188 | int min(int a, int b) { return a < b ? a : b; } 189 | int max(int a, int b) { return a > b ? a : b; } 190 | 191 | 192 | int main(int argc, char **argv) { 193 | 194 | char parquet_file_dir[256], parquet_file_path[256]; 195 | sprintf(parquet_file_dir, "%s/_parquet", argv[1]); 196 | mkdir(parquet_file_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); 197 | int mpi_size, mpi_rank; 198 | MPI_Init(&argc, &argv); 199 | MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); 200 | MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); 201 | 202 | if(mpi_rank == 0) 203 | mkdir(parquet_file_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); 204 | MPI_Barrier(MPI_COMM_WORLD); 205 | recorder_init_reader(argv[1], &reader); 206 | 207 | // Each rank will process n files (n ranks traces) 208 | int n = max(reader.metadata.total_ranks/mpi_size, 1); 209 | int start_rank = n * mpi_rank; 210 | int end_rank = min(reader.metadata.total_ranks, n*(mpi_rank+1)); 211 | sprintf(parquet_file_path, "%s/%d" , parquet_file_dir, mpi_rank); 212 | 213 | ParquetWriter writer(0, parquet_file_path); 214 | for(int rank = start_rank; rank < end_rank; rank++) { 215 | writer.rank = rank; 216 | recorder_decode_records(&reader, rank, handle_one_record, &writer); 217 | printf("\r[Recorder] rank %d finished, unique call signatures: %d\n", rank, cst.entries); 218 | } 219 | writer.finish(); 220 | recorder_free_reader(&reader); 221 | 222 | MPI_Barrier(MPI_COMM_WORLD); 223 | MPI_Finalize(); 224 | 225 | return 0; 226 | } 227 | -------------------------------------------------------------------------------- /tools/recorder2text.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "reader.h" 9 | 10 | #define DECIMAL 6 11 | 12 | RecorderReader reader; 13 | static char formatting_record[32]; 14 | static char formatting_fname[128]; 15 | 16 | int digits_count(int n) { 17 | int digits = 0; 18 | do { 19 | n /= 10; 20 | ++digits; 21 | } while (n != 0); 22 | return digits; 23 | } 24 | 25 | void write_to_textfile(Record *record, void* arg) { 26 | FILE* f = (FILE*) arg; 27 | 28 | bool user_func = (record->func_id == RECORDER_USER_FUNCTION); 29 | 30 | const char* func_name = recorder_get_func_name(&reader, record); 31 | 32 | fprintf(f, formatting_record, record->tstart, record->tend, // record->tid 33 | func_name, record->call_depth, recorder_get_func_type(&reader, record)); 34 | 35 | for(int arg_id = 0; !user_func && arg_id < record->arg_count; arg_id++) { 36 | char *arg = record->args[arg_id]; 37 | fprintf(f, " %s", arg); 38 | } 39 | 40 | fprintf(f, " )\n"); 41 | } 42 | 43 | 44 | int min(int a, int b) { return a < b ? a : b; } 45 | int max(int a, int b) { return a > b ? a : b; } 46 | 47 | int main(int argc, char **argv) { 48 | 49 | char textfile_dir[512]; 50 | char textfile_path[1024]; 51 | snprintf(textfile_dir, sizeof(textfile_dir), "%s/_text", argv[1]); 52 | 53 | int mpi_size, mpi_rank; 54 | MPI_Init(&argc, &argv); 55 | MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); 56 | MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); 57 | 58 | if(mpi_rank == 0) 59 | mkdir(textfile_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); 60 | MPI_Barrier(MPI_COMM_WORLD); 61 | 62 | recorder_init_reader(argv[1], &reader); 63 | 64 | int decimal = log10(1 / reader.metadata.time_resolution); 65 | sprintf(formatting_record, "%%.%df %%.%df %%s %%d %%d (", decimal, decimal); 66 | sprintf(formatting_fname, "%%s/%%0%dd.txt", digits_count(reader.metadata.total_ranks)); 67 | 68 | // Each rank will process n files (n ranks traces) 69 | int n = max(reader.metadata.total_ranks/mpi_size, 1); 70 | int start_rank = n * mpi_rank; 71 | int end_rank = min(reader.metadata.total_ranks, n*(mpi_rank+1)); 72 | 73 | for(int rank = start_rank; rank < end_rank; rank++) { 74 | sprintf(textfile_path, formatting_fname, textfile_dir, rank); 75 | FILE* fout = fopen(textfile_path, "w"); 76 | 77 | recorder_decode_records(&reader, rank, write_to_textfile, fout); 78 | 79 | fclose(fout); 80 | 81 | printf("\r[Recorder] rank %d finished\n", rank); 82 | } 83 | 84 | recorder_free_reader(&reader); 85 | 86 | MPI_Barrier(MPI_COMM_WORLD); 87 | MPI_Finalize(); 88 | 89 | return 0; 90 | } 91 | -------------------------------------------------------------------------------- /tools/recorder2timeline.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "reader.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | RecorderReader reader; 16 | 17 | struct Writer{ 18 | std::ofstream outFile; 19 | const char* sep; 20 | int rank, my_rank, total_ranks; 21 | }; 22 | 23 | static const char* type_name(int type) { 24 | switch (type) { 25 | case RECORDER_POSIX: 26 | return "POSIX"; 27 | case RECORDER_MPIIO: 28 | return "MPI I/O"; 29 | case RECORDER_MPI: 30 | return "MPI"; 31 | case RECORDER_HDF5: 32 | return "HDF5"; 33 | case RECORDER_FTRACE: 34 | return "USER"; 35 | default: 36 | return "UNKNOWN"; 37 | } 38 | } 39 | 40 | struct timeline_ts{ 41 | double value; 42 | }; 43 | std::ostream& 44 | operator<<( std::ostream& out, 45 | const timeline_ts& ts ) 46 | { 47 | std::ios oldState(nullptr); 48 | oldState.copyfmt(out); 49 | 50 | // Chrome traces are always in micro seconds and Recorder timestamps in 51 | // seconds. No need to use the timer resolution from the trace to convert 52 | // to microseconds. 53 | out << std::fixed << std::setw(5) << std::setprecision(3) << (ts.value * 1e6); 54 | out.copyfmt(oldState); 55 | 56 | return out; 57 | } 58 | 59 | void write_to_json(Record *record, void* arg) { 60 | 61 | int cat = recorder_get_func_type(&reader, record); 62 | if (record->call_depth == 0 || (cat == 0 || cat == 1 || cat == 3)) { 63 | Writer *writer = (Writer *) arg; 64 | bool user_func = (record->func_id == RECORDER_USER_FUNCTION); 65 | const char *func_name = recorder_get_func_name(&reader, record); 66 | 67 | if (user_func) 68 | func_name = record->args[0]; 69 | std::stringstream ss; 70 | ss << writer->sep 71 | << "{\"pid\":" << writer->rank 72 | << ",\"tid\":" << record->tid 73 | << ",\"ts\":" << timeline_ts{record->tstart} 74 | << ",\"name\":\"" << func_name 75 | << "\",\"cat\":\"" << type_name(cat) 76 | << "\",\"ph\":\"X\"" 77 | << ",\"dur\":" << timeline_ts{record->tend - record->tstart} 78 | << ",\"args\":{"; 79 | if (!user_func) { 80 | ss << "\"args\":["; 81 | const char* sep = ""; 82 | for (int arg_id = 0; arg_id < record->arg_count; arg_id++) { 83 | char *arg = record->args[arg_id]; 84 | ss << sep << "\"" << arg << "\""; 85 | sep = ","; 86 | } 87 | ss << "],"; 88 | } 89 | ss << "\"tend\": \"" << record->tend << "\"}}"; 90 | writer->outFile << ss.rdbuf(); 91 | writer->sep = ",\n"; 92 | } 93 | } 94 | 95 | int min(int a, int b) { return a < b ? a : b; } 96 | int max(int a, int b) { return a > b ? a : b; } 97 | 98 | int main(int argc, char **argv) { 99 | 100 | if(argc!=2) { 101 | std::cerr << "Usage: " << argv[0] << " \n"; 102 | std::exit(1); 103 | } 104 | 105 | int mpi_size, mpi_rank; 106 | MPI_Init(&argc, &argv); 107 | MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); 108 | MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); 109 | 110 | char textfile_dir[256]; 111 | sprintf(textfile_dir, "%s/_chrome", argv[1]); 112 | 113 | if(mpi_rank == 0) 114 | mkdir(textfile_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); 115 | MPI_Barrier(MPI_COMM_WORLD); 116 | 117 | recorder_init_reader(argv[1], &reader); 118 | 119 | // Each rank will process n files (n ranks traces) 120 | int n = max(reader.metadata.total_ranks/mpi_size, 1); 121 | int start_rank = n * mpi_rank; 122 | int end_rank = min(reader.metadata.total_ranks, n*(mpi_rank+1)); 123 | char textfile_path[256]; 124 | sprintf(textfile_path, "%s/timeline_%d.json", textfile_dir, mpi_rank); 125 | Writer local; 126 | local.outFile.open(textfile_path, std::ofstream::trunc|std::ofstream::out); 127 | local.outFile << "{\"traceEvents\": [\n"; 128 | local.sep = ""; 129 | for(int rank = start_rank; rank < end_rank; rank++) { 130 | local.rank = rank; 131 | recorder_decode_records(&reader, rank, write_to_json, &local); 132 | printf("\r[Recorder] rank %d finished, %s\n", rank, textfile_path); 133 | } 134 | local.outFile << "],\n\"displayTimeUnit\": \"ms\",\"systemTraceEvents\": \"SystemTraceData\",\"otherData\": {\"version\": \"Taxonomy v1.0\" }, \"stackFrames\": {}, \"samples\": []}\n"; 135 | local.outFile.close(); 136 | recorder_free_reader(&reader); 137 | 138 | MPI_Barrier(MPI_COMM_WORLD); 139 | MPI_Finalize(); 140 | 141 | return 0; 142 | } 143 | -------------------------------------------------------------------------------- /tools/reporter/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uiuc-hpc/Recorder/ed49c548a5f64624fc9e1e358511325af931ae88/tools/reporter/CMakeLists.txt -------------------------------------------------------------------------------- /tools/reporter/reporter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import sys 4 | import recorder_viz 5 | from recorder_viz import RecorderReader 6 | 7 | if __name__ == "__main__": 8 | reader = RecorderReader(sys.argv[1]) 9 | recorder_viz.generate_report(reader, "recorder-report.html") 10 | -------------------------------------------------------------------------------- /tools/verifyio/README.md: -------------------------------------------------------------------------------- 1 | 2 | ### Verify if I/O operations are properly synchronized uder specific semantics 3 | ------------------------------ 4 | 5 | 6 | 7 | Make sure `$RECORDER_INSTALL_PATH` is the install location of Recorder. 8 | 9 | Dependencies: `recorder-viz` and `networkx`. Both can be installed using pip. 10 | 11 | Steps: 12 | 1. Run program with Recorder to generate traces. 13 | 2. Run the conflict detector to report **potential** conflicting I/O accesses. 14 | Those acesses are only potentially conflicting as here we do not take happens-before order into consideration yet. 15 | 16 | `$RECORDER_INSTALL_PATH/bin/conflict_detector /path/to/traces` 17 | 18 | This command will write all potential conflicts found to the file `/path/to/traces/conflicts.txt` 19 | 20 | 3. Finally run the verification code, which checks if those potential conflicting operations are properly synchronzied. 21 | 22 | The `semantics` option needs to match the one provided by the underlying file system. For example, if the traces were collected on UnifyFS, set it to "commit". 23 | 24 | ```python 25 | python ./verifyio.py -h # print out usage 26 | 27 | #Example: 28 | python ./verifyio.py /path/to/traces --semantics=mpi 29 | ``` 30 | 31 | 32 | 33 | #### Note on the third step: 34 | 35 | The code first matches all MPI calls to build a graph representing the happens-before order. Each node in the graph represents a MPI call, if there is a path from node A to node B, then A must happens-before B. 36 | 37 | Given a conflicing I/O pair of accesses (op1, op2). Using the graph, we can figure out if op1 happens-before op2. If so, they are properly synchronzied. 38 | This works if we assume the POSIX semantics. E.g., op1(by rank1)->send(by rank1)->recv(by rank2)->op2(by rank2), this path tells us op1 and op2 are properly synchronized. 39 | 40 | However, things are a little different with default MPI user-imposed semantics (i.e., nonatomic mode). According to the MPI standard, many collective calls do not guarantee the synchronization beteen the involved processes. The standard explictly says the following collectives are guaranteed to be synchronized: 41 | - MPI_Barrier 42 | - MPI_Allgather 43 | - MPI_Alltoall and their V and W variants 44 | - MPI_Allreduce 45 | - MPI_Reduce_scatter 46 | - MPI_Reduce_scatter_block 47 | 48 | With user-imposed semantics, the **"sync-barrier-sync"** construct is required to guarnatee sequencial consistency. Barrier can be replaced by a send-recv or the collectives listed above. Sync is one of MPI_File_open, MPI_File_close or MPI_File_sync. 49 | 50 | Now, given two potential conflicting accesses op1 an op2, our job is to find out if there is a **sync-barrier-sync** in between. 51 | -------------------------------------------------------------------------------- /tools/verifyio/read_nodes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding, utf-8 3 | import struct 4 | from itertools import repeat 5 | from verifyio_graph import VerifyIONode 6 | 7 | accepted_mpi_funcs = [ 8 | 'MPI_Send', 'MPI_Ssend', 'MPI_Issend', 'MPI_Isend', 9 | 'MPI_Recv', 'MPI_Sendrecv', 'MPI_Irecv', 10 | 'MPI_Wait', 'MPI_Waitall', 'MPI_Waitany', 11 | 'MPI_Waitsome', 'MPI_Test', 'MPI_Testall', 12 | 'MPI_Testany', 'MPI_Testsome', 'MPI_Bcast', 13 | 'MPI_Ibcast', 'MPI_Reduce', 'MPI_Ireduce', 14 | 'MPI_Gather', 'MPI_Igather', 'MPI_Gatherv', 15 | 'MPI_Igatherv', 'MPI_Barrier', 'MPI_Alltoall', 16 | 'MPI_Allreduce', 'MPI_Allgatherv', 17 | 'MPI_Reduce_scatter', 'MPI_File_open', 18 | 'MPI_File_close', 'MPI_File_read_at_all', 19 | 'MPI_File_write_at_all', 'MPI_File_set_size', 20 | 'MPI_File_set_view', 'MPI_File_sync', 21 | 'MPI_File_read_all', 'MPI_File_read_ordered', 22 | 'MPI_File_write_all','MPI_File_write_ordered', 23 | 'MPI_Comm_dup', 'MPI_Comm_split', 24 | 'MPI_Comm_split_type', 'MPI_Cart_create', 25 | 'MPI_Cart_sub' 26 | ] 27 | 28 | accepted_meta_funcs = [ 29 | 'fsync', 'open', 'fopen', 'close', 'fclose' 30 | ] 31 | 32 | def create_verifyio_node(rank_seqid_tuple, reader): 33 | rank, seq_id = rank_seqid_tuple 34 | func = reader.funcs[reader.records[rank][seq_id].func_id] 35 | return VerifyIONode(rank, seq_id, func) 36 | 37 | 38 | def read_verifyio_nodes_and_conflicts(reader): 39 | 40 | vio_nodes = [[] for i in repeat(None, reader.nprocs)] 41 | 42 | func_list = reader.funcs 43 | for rank in range(reader.nprocs): 44 | records = reader.records[rank] 45 | for seq_id in range(reader.num_records[rank]): 46 | func = func_list[records[seq_id].func_id] 47 | mpifh = None 48 | # Retrive needed MPI calls 49 | if func in accepted_mpi_funcs: 50 | if func.startswith("MPI_File"): 51 | mpifh = records[seq_id].args[0].decode('utf-8') 52 | mpi_node = VerifyIONode(rank, seq_id, func, -1, mpifh) 53 | vio_nodes[rank].append(mpi_node) 54 | # Retrive needed metadata I/O calls 55 | elif func in accepted_meta_funcs: 56 | fh = records[seq_id].args[0].decode('utf-8') 57 | metadata_io_node = VerifyIONode(rank, seq_id, func, -1, fh) 58 | vio_nodes[rank].append(metadata_io_node) 59 | 60 | # Finally, retrive needed I/O calls according 61 | # to the conflict file 62 | conflict_rank_seqid_groups = read_all_conflicts(reader) 63 | unique_conflict_ops = {} 64 | 65 | conflict_vio_node_groups = [] 66 | 67 | for cg in conflict_rank_seqid_groups: 68 | # cg[0] is (c1_rank, c1_seq_id) 69 | # cg[1] is rank grouped list of c2_rank, c2_seq_id) 70 | 71 | if cg[0] not in unique_conflict_ops: 72 | c1 = create_verifyio_node(cg[0], reader) 73 | unique_conflict_ops[cg[0]] = c1 74 | vio_nodes[cg[0][0]].append(c1) 75 | else: 76 | c1 = unique_conflict_ops[cg[0]] 77 | 78 | c2s = [[] for _ in range(reader.nprocs)] 79 | for c2_rank, c2s_per_rank in enumerate(cg[1]): 80 | for c2_seq_id in c2s_per_rank: 81 | c2_rank_seqid = (c2_rank, c2_seq_id) 82 | if (c2_rank, c2_seq_id) not in unique_conflict_ops: 83 | c2 = create_verifyio_node(c2_rank_seqid, reader) 84 | unique_conflict_ops[c2_rank_seqid] = c2 85 | vio_nodes[c2_rank].append(c2) 86 | else: 87 | c2 = unique_conflict_ops[c2_rank_seqid] 88 | c2s[rank].append(c2) 89 | 90 | group = [c1, c2s] 91 | conflict_vio_node_groups.append(group) 92 | 93 | return vio_nodes, conflict_vio_node_groups 94 | 95 | 96 | def read_one_conflict_group(f, reader): 97 | data = f.read(16) 98 | if not data: 99 | return None 100 | 101 | conflict_ops = [[] for _ in range(reader.nprocs)] 102 | 103 | # int, int, size_t 104 | c1_rank, c1_seqid, num_pairs = struct.unpack("iiN", data) 105 | #print(c1_rank, c1_seqid, num_pairs) 106 | offset = 0 107 | data = f.read(4*num_pairs*2) 108 | for i in range(num_pairs): 109 | c2_rank, c2_seqid = struct.unpack("ii", data[offset:offset+8]) 110 | offset += 8 111 | conflict_ops[c2_rank].append(c2_seqid) 112 | 113 | conflict_group = ((c1_rank, c1_seqid), conflict_ops) 114 | return conflict_group 115 | 116 | 117 | ''' 118 | Read conflict pairs from the conflict file generated 119 | using the conflict-detector program. 120 | 121 | The conflict file is a binary file has the following format: 122 | 123 | for each conflict group: 124 | c1_rank:int, c1_seq_id:int, num_pairs:size_t 125 | c2_rank:int, c2_seq_id:int, c2_rank:int, c2_seq_id:int, ... 126 | 127 | This function returns a list of conflict groups, it will then 128 | be used later to create groups of VerifyIONode and return to 129 | the caller. 130 | Each conflict group has this format: 131 | [(c1_rank,c1_seq_id), [/*rank0:*/[seqid1, seqid2,...], /*rank1*/:[...] ]] 132 | 133 | ''' 134 | def read_all_conflicts(reader): 135 | conflict_groups = [] 136 | with open(reader.logs_dir+"/conflicts.dat", mode="rb") as f: 137 | while True: 138 | conflict_group = read_one_conflict_group(f, reader) 139 | if conflict_group: 140 | conflict_groups.append(conflict_group) 141 | else: 142 | # reached the end of file 143 | break 144 | return conflict_groups 145 | 146 | 147 | if __name__ == "__main__": 148 | read_all_conflict_pairs() 149 | -------------------------------------------------------------------------------- /tools/verifyio/recorder_reader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | from ctypes import * 4 | import sys, os, glob, struct 5 | 6 | 7 | class VerifyIORecord(Structure): 8 | # The fields must be identical as PyRecord in tools/reader.h 9 | _fields_ = [ 10 | ("func_id", c_int), 11 | ("call_depth", c_ubyte), 12 | ("arg_count", c_ubyte), 13 | ("args", POINTER(c_char_p)), # Note in python3, args[i] is 'bytes' type 14 | ] 15 | 16 | # In Python3, self.args[i] is 'bytes' type 17 | # For compatable reason, we convert it to str type 18 | # and will only use self.arg_strs[i] to access the filename 19 | """ 20 | def args_to_strs(self): 21 | arg_strs = [''] * self.arg_count 22 | for i in range(self.arg_count): 23 | if(type(self.args[i]) == str): 24 | arg_strs[i] = self.args[i] 25 | else: 26 | arg_strs[i] = self.args[i].decode('utf-8') 27 | return arg_strs 28 | """ 29 | 30 | 31 | """ 32 | self.funcs: a list of supported funcitons 33 | self.nprocs 34 | self.num_records[rank] 35 | self.records[Rank]: per-rank list of VerifyIORecord 36 | """ 37 | class RecorderReader: 38 | 39 | def str2char_p(self, s): 40 | return c_char_p( s.encode('utf-8') ) 41 | 42 | def __init__(self, logs_dir): 43 | if "RECORDER_INSTALL_PATH" not in os.environ: 44 | msg="Error:\n"\ 45 | " RECORDER_INSTALL_PATH environment variable is not set.\n" \ 46 | " Please set it to the path where you installed Recorder." 47 | print(msg) 48 | exit(1) 49 | 50 | recorder_install_path = os.path.abspath(os.environ["RECORDER_INSTALL_PATH"]) 51 | libreader_path = recorder_install_path + "/lib/libreader.so" 52 | 53 | if not os.path.isfile(libreader_path): 54 | msg="Error:\n"\ 55 | " Could not find Recorder reader library\n"\ 56 | " Please make sure Recorder is installed at %s",\ 57 | recorder_install_path 58 | print(msg) 59 | exit(1); 60 | 61 | # Load function list and the number of processes 62 | self.logs_dir = logs_dir 63 | self.__read_num_procs(self.logs_dir + "/recorder.mt") 64 | self.__load_func_list(self.logs_dir + "/recorder.mt") 65 | 66 | # Set up C reader library 67 | # Read all VerifyIORecord 68 | self.libreader = cdll.LoadLibrary(libreader_path) 69 | self.libreader.recorder_read_verifyio_records.restype = POINTER(POINTER(VerifyIORecord)) 70 | num_records = (c_size_t * self.nprocs)() 71 | self.records = self.libreader.recorder_read_verifyio_records(self.str2char_p(self.logs_dir), num_records) 72 | self.num_records = [0 for x in range(self.nprocs)] 73 | for rank in range(self.nprocs): 74 | self.num_records[rank] = num_records[rank] 75 | 76 | # We dont need the entire RecorderMetadata 77 | # we only need to read the first integer, which 78 | # is the number of processes 79 | def __read_num_procs(self, metadata_file): 80 | with open(metadata_file, 'rb') as f: 81 | self.nprocs = struct.unpack('i', f.read(4))[0] 82 | 83 | # read supported list of functions from the metadata file 84 | # invoked in __init__() only 85 | def __load_func_list(self, metadata_file): 86 | with open(metadata_file, 'rb') as f: 87 | f.seek(1024, 0) # skip the reserved metadata block (fixed 1024 bytes) 88 | self.funcs = f.read().splitlines() 89 | self.funcs = [func.decode('utf-8') for func in self.funcs] 90 | 91 | 92 | 93 | if __name__ == "__main__": 94 | 95 | import resource, psutil 96 | print(resource.getrusage(resource.RUSAGE_SELF)) 97 | print('RAM Used (GB):', psutil.virtual_memory()[3]/1000000000) 98 | 99 | reader = RecorderReader(sys.argv[1]) 100 | 101 | print(resource.getrusage(resource.RUSAGE_SELF)) 102 | print('RAM Used (GB):', psutil.virtual_memory()[3]/1000000000) 103 | -------------------------------------------------------------------------------- /tools/verifyio/vector_clock.py: -------------------------------------------------------------------------------- 1 | import networkx as nx 2 | import numpy as np 3 | 4 | G = nx.DiGraph() 5 | 6 | #P1 7 | G.add_node('A',local=[1,0,0]) 8 | G.add_node('B',local=[2,0,0]) 9 | G.add_node('C',local=[3,0,0]) 10 | G.add_node('D',local=[4,0,0]) 11 | G.add_node('E',local=[5,0,0]) 12 | G.add_node('F',local=[6,0,0]) 13 | G.add_node('G',local=[7,0,0]) 14 | #P2 15 | G.add_node('H',local=[0,1,0]) 16 | G.add_node('I',local=[0,2,0]) 17 | G.add_node('J',local=[0,3,0]) 18 | #P3 19 | G.add_node('K',local=[0,0,1]) 20 | G.add_node('L',local=[0,0,2]) 21 | G.add_node('M',local=[0,0,3]) 22 | 23 | G.add_edge('A', 'B') 24 | G.add_edge('B', 'C') 25 | G.add_edge('C', 'D') 26 | G.add_edge('D', 'E') 27 | G.add_edge('E', 'F') 28 | G.add_edge('F', 'G') 29 | G.add_edge('H', 'I') 30 | G.add_edge('I', 'J') 31 | G.add_edge('K', 'L') 32 | G.add_edge('L', 'M') 33 | G.add_edge('B', 'I') 34 | G.add_edge('D', 'M') 35 | G.add_edge('F', 'J') 36 | G.add_edge('H', 'C') 37 | G.add_edge('L', 'E') 38 | 39 | ''' 40 | #P1 41 | G.add_node('A',local=[1,0,0]) 42 | G.add_node('B',local=[2,0,0]) 43 | G.add_node('C',local=[3,0,0]) 44 | G.add_node('D',local=[4,0,0]) 45 | G.add_node('Z',local=[5,0,0]) 46 | #P2 47 | G.add_node('E',local=[0,1,0]) 48 | G.add_node('F',local=[0,2,0]) 49 | G.add_node('G',local=[0,3,0]) 50 | #P3 51 | G.add_node('H',local=[0,0,1]) 52 | G.add_node('I',local=[0,0,2]) 53 | G.add_node('J',local=[0,0,3]) 54 | 55 | G.add_edge('A', 'B') 56 | G.add_edge('B', 'C') 57 | G.add_edge('C', 'D') 58 | G.add_edge('D', 'Z') 59 | G.add_edge('E', 'F') 60 | G.add_edge('F', 'G') 61 | G.add_edge('H', 'I') 62 | G.add_edge('I', 'J') 63 | G.add_edge('B', 'F') 64 | G.add_edge('G', 'D') 65 | G.add_edge('H', 'E') 66 | G.add_edge('Z', 'J') 67 | ''' 68 | 69 | 70 | 71 | for e in nx.topological_sort(G): 72 | #print (e, '\t', G._node[e]['local']) 73 | l1 = G._node[e]['local'] 74 | predecessors = G.predecessors(e) 75 | for eachpred in predecessors: 76 | l2 = G._node[eachpred]['local'] 77 | l = list(map(max, zip(l1, l2))) 78 | l1 = l 79 | 80 | G._node[e]['local'] = l1 81 | 82 | for e in G.nodes(): 83 | print (e, G._node[e]['local']) 84 | 85 | 86 | def causal_order(x,y): 87 | a = np.array(G._node[x]['local']) 88 | b = np.array(G._node[y]['local']) 89 | 90 | c = a>=b 91 | d = b>=a 92 | 93 | print c 94 | print d 95 | if c.all(): 96 | print y,' happened before ', x 97 | 98 | elif d.all(): 99 | print x,' happened before ', y 100 | 101 | else: 102 | print 'Concurrent events' 103 | 104 | 105 | 106 | 107 | #causal_order('G','K') 108 | 109 | 110 | -------------------------------------------------------------------------------- /tools/verifyio/verifyio_graph.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | import networkx as nx 4 | 5 | class VerifyIONode: 6 | def __init__(self, rank, seq_id, func, fd = -1, mpifh = None): 7 | self.rank = rank 8 | # This is the index with respect to all 9 | # Recorder trace records 10 | self.seq_id = seq_id 11 | # This is the index with respect to all 12 | # VerifyIONode we read 13 | self.index = -1 14 | self.func = func 15 | # An integer maps to the filename 16 | self.fd = fd 17 | # Store the MPI-IO file handle so we can match 18 | # I/O calls with sync/commit calls during 19 | # the verification. 20 | self.mpifh = mpifh 21 | 22 | def graph_key(self): 23 | return str(self.rank) + "-" + str(self.seq_id) + "-" + str(self.func) 24 | 25 | def __str__(self): 26 | #if "write" in self.func or "read" in self.func or \ 27 | # self.func.startswith("MPI_File_"): 28 | # return "Rank %d: %dth %s(%s)" %(self.rank, self.seq_id, self.func, self.mpifh) 29 | #else: 30 | # return "Rank %d: %dth %s" %(self.rank, self.seq_id, self.func) 31 | return "" %(self.rank, self.seq_id, self.func) 32 | 33 | 34 | ''' 35 | Essentially a wrapper for networkx DiGraph 36 | ''' 37 | class VerifyIOGraph: 38 | def __init__(self, nodes, edges, include_vc=False): 39 | self.G = nx.DiGraph() 40 | self.nodes = nodes # A list of VerifyIONode sorted by seq_id 41 | self.include_vc = include_vc 42 | self.__build_graph(nodes, edges, include_vc) 43 | 44 | def num_nodes(self): 45 | return len(self.G.nodes) 46 | 47 | # next (program-order) node of funcs in the sam rank 48 | def next_po_node(self, current, funcs): 49 | nodes = self.nodes[current.rank] 50 | if not funcs: 51 | return nodes[current.index+1] 52 | target = None 53 | for i in range(current.index+1, len(nodes)): 54 | if nodes[i].func in funcs: 55 | target = nodes[i] 56 | break 57 | return target 58 | 59 | # previous (program-order) node of funcs in the same rank 60 | def prev_po_node(self, current, funcs): 61 | nodes = self.nodes[current.rank] 62 | if not funcs: 63 | return nodes[current.index-1] 64 | target = None 65 | for i in range(current.index-1, 0, -1): 66 | if nodes[i].func in funcs: 67 | target = nodes[i] 68 | break 69 | return target 70 | 71 | # next (happens-beofre) node of funcs in the target rank 72 | def next_hb_node(self, current, funcs, target_rank): 73 | target = None 74 | nodes = self.nodes[target_rank] 75 | for target in nodes: 76 | if (target.func in funcs) and (self.has_path(current, target)): 77 | target.append(target) 78 | break 79 | return target 80 | 81 | def add_edge(self, h, t): 82 | self.G.add_edge(h.graph_key(), t.graph_key()) 83 | 84 | def remove_edge(self, h, t): 85 | self.G.remove_edge(h.graph_key(), t.graph_key()) 86 | 87 | def has_path(self, src, dst): 88 | return nx.has_path(self.G, src.graph_key(), dst.graph_key()) 89 | 90 | def plot_graph(self, fname): 91 | import matplotlib.pyplot as plt 92 | nx.draw_networkx(self.G) 93 | #plt.savefig(fname) 94 | plt.show() 95 | 96 | def get_vector_clock(self, n): 97 | return self.G.nodes[n.graph_key()]['vc'] 98 | 99 | # caller need to assume there is no cycle 100 | # in the DAG. 101 | def run_vector_clock(self): 102 | for node_key in nx.topological_sort(self.G): 103 | vc = self.G.nodes[node_key]['vc'] 104 | for eachpred in self.G.predecessors(node_key): 105 | pred_vc = self.G.nodes[eachpred]['vc'].copy() 106 | pred_vc[self.key2rank(eachpred)] += 1 107 | vc = list(map(max, zip(vc, pred_vc))) 108 | 109 | self.G.nodes[node_key]['vc'] = vc 110 | #print(node_key, vc) 111 | 112 | def run_transitive_closure(self): 113 | tc = nx.transitive_closure(self.G) 114 | 115 | # Retrive rank from node key 116 | def key2rank(self, key): 117 | return (int)(key.split('-')[0]) 118 | 119 | def shortest_path(self, src, dst): 120 | 121 | if (not src) or (not dst): 122 | print("shortest_path Error: must specify src and dst (VerifyIONode)") 123 | return [] 124 | 125 | # nx.shortest_path will return a list of nodes in 126 | # keys. we then retrive the real VerifyIONode and return a 127 | # list of them 128 | path_in_keys = nx.shortest_path(self.G, src.graph_key(), dst.graph_key()) 129 | path = [] 130 | for key in path_in_keys: 131 | path.append(key) 132 | return path 133 | 134 | 135 | # private method to build the networkx DiGraph 136 | # called only by __init__ 137 | # nodes: per rank of nodes of type VerifyIONode 138 | # Add neighbouring nodes of the same rank 139 | # This step will add all nodes 140 | def __build_graph(self, all_nodes, mpi_edges, include_vc): 141 | # 1. Add program orders 142 | nprocs = len(all_nodes) 143 | for rank in range(nprocs): 144 | for i in range(len(all_nodes[rank]) - 1): 145 | h = all_nodes[rank][i] 146 | t = all_nodes[rank][i+1] 147 | self.add_edge(h, t) 148 | 149 | # Include vector clock for each node 150 | if not include_vc: continue 151 | vc1 = [0] * (nprocs + 1) # one more for ghost rank 152 | vc2 = [0] * (nprocs + 1) # one more for ghost rank 153 | vc1[rank] = i 154 | vc2[rank] = i+1 155 | self.G.nodes[h.graph_key()]['vc'] = vc1 156 | self.G.nodes[t.graph_key()]['vc'] = vc2 157 | 158 | # corner case: when this rank has only one node 159 | # the code will not run into the previous for loop 160 | if len(all_nodes[rank]) == 1: 161 | n = all_nodes[rank][0] 162 | self.G.add_node(n.graph_key(), vc=[0]*(nprocs+1)) 163 | 164 | # 2. Add synchornzation orders (using mpi edges) 165 | # Before calling this function, we should 166 | # have added all nodes. We use this function 167 | # to add edges of matching MPI calls 168 | from match_mpi import MPICallType 169 | ghost_node_count = 0 170 | for edge in mpi_edges: 171 | 172 | # case i: point to point calls 173 | if edge.call_type == MPICallType.POINT_TO_POINT: 174 | self.add_edge(edge.head, edge.tail) 175 | continue 176 | 177 | # case ii: collective calls 178 | # We handle all collect calls in a same way 179 | # just use them as a fence to establish order 180 | # between I/O calls 181 | # add a ghost node and connect all predecessors 182 | # and successors from all ranks. 183 | # This prvents circle 184 | mpi_calls = edge.get_all_involved_calls() 185 | if len(mpi_calls) <= 1: continue 186 | for mpi_call in mpi_calls: 187 | 188 | # Add a ghost node and connect all predecessors 189 | # and successors from all ranks. This prvents the circle 190 | ghost_node = VerifyIONode(rank=nprocs, seq_id=ghost_node_count, func="ghost") 191 | for h in mpi_calls: 192 | # use list() to make a copy to avoid the runtime 193 | # error of "dictionary size changed during iteration" 194 | try: 195 | successors = list(self.G.successors(h.graph_key())) 196 | for successor in successors: 197 | self.G.remove_edge(h.graph_key(), successor) 198 | if len(list(self.G.successors(h.graph_key()))) != 0 : 199 | print("Not possible!") 200 | for successor in successors: 201 | self.G.add_edge(ghost_node.graph_key(), successor) 202 | except nx.exception.NetworkXError: 203 | # when the node has no successors, the G.successors() 204 | # function through an error instead of an empty list 205 | # (e.g., the last node in the graph) 206 | pass 207 | 208 | for h in mpi_calls: 209 | self.add_edge(h, ghost_node) 210 | 211 | vc = [0] * (nprocs + 1) 212 | vc[nprocs] = ghost_node_count 213 | self.G.nodes[ghost_node.graph_key()]['vc'] = vc 214 | ghost_node_count += 1 215 | 216 | # Detect cycles of the graph 217 | # correct code should contain no cycles. 218 | # incorrect code, e.g., with unmatched collective calls 219 | # may have cycles 220 | # e.g., pnetcdf/test/testcases/test-varm.c 221 | # it uses ncmpi_wait() call, which may result in 222 | # rank0 calling MPI_File_write_at_all(), and 223 | # rank1 calling MPI_File_write_all() 224 | # 225 | # Our verification algorithm assumes the graph 226 | # has no code, so we need do this check first. 227 | def check_cycles(self): 228 | has_cycles = False 229 | try: 230 | cycle = nx.find_cycle(self.G) 231 | print("Generated graph contain cycles. Original code may have bugs.") 232 | 233 | simplified_cycle = [] 234 | for edge in cycle: 235 | c1, c2 = edge[0], edge[1] 236 | rank1 = self.key2rank(c1) 237 | rank2 = self.key2rank(c2) 238 | if (rank1 != rank2): 239 | simplified_cycle.append(edge) 240 | print(simplified_cycle) 241 | has_cycles = True 242 | except nx.exception.NetworkXNoCycle: 243 | pass 244 | return has_cycles 245 | 246 | -------------------------------------------------------------------------------- /tools/verifyio/visualize.py: -------------------------------------------------------------------------------- 1 | import graphviz 2 | 3 | def rank_of_node(node_key): 4 | if(node_key.startswith("-1")): 5 | return -1 6 | 7 | return int(node_key.split("-")[0]) 8 | 9 | def plot_graph2(G, mpi_size): 10 | 11 | edges_of_rank = {} 12 | other_edges = [] 13 | 14 | for rank in [-1]+range(mpi_size): 15 | edges_of_rank[rank] = [] 16 | 17 | for (h, t) in G.edges(): 18 | h_rank = rank_of_node(h) 19 | t_rank = rank_of_node(t) 20 | if h_rank == t_rank: 21 | edges_of_rank[h_rank].append((h,t)) 22 | else: 23 | other_edges.append((h, t)) 24 | 25 | vizG = graphviz.Digraph('G', filename='test.gv', format="pdf") 26 | for rank in [-1]+range(mpi_size): 27 | with vizG.subgraph(name='cluster_'+str(rank)) as sg: 28 | if rank %2 == 0: 29 | sg.attr(style='filled', color='lightgrey') 30 | sg.node_attr.update(style='filled', color='white') 31 | else: 32 | #sg.attr(color='blue') 33 | sg.node_attr['style'] = 'filled' 34 | 35 | sg.edges(edges_of_rank[rank]) 36 | sg.attr(label='Proc '+str(rank)) 37 | 38 | for (h, t) in other_edges: 39 | h_rank = rank_of_node(h) 40 | t_rank = rank_of_node(t) 41 | constraint = "true" 42 | if h_rank == -1 or t_rank == -1: 43 | constraint = "false" 44 | vizG.edge(h, t, constraint=constraint) 45 | 46 | vizG.view() 47 | 48 | def plot_graph(G, mpi_size): 49 | 50 | nodes_of_rank = {} 51 | for rank in [-1]+range(mpi_size): 52 | nodes_of_rank[rank] = [] 53 | 54 | for n in G.nodes(): 55 | rank = rank_of_node(n) 56 | nodes_of_rank[rank].append(n) 57 | 58 | vizG = graphviz.Digraph('G', filename='test.gv', format="pdf") 59 | for level in range(len(nodes_of_rank[0])): 60 | with vizG.subgraph() as sg: 61 | sg.attr(rank=str(level)) 62 | for rank in range(mpi_size): 63 | if level < len(nodes_of_rank[rank]): 64 | sg.node(nodes_of_rank[rank][level]) 65 | 66 | for (h, t) in G.edges(): 67 | h_rank = rank_of_node(h) 68 | t_rank = rank_of_node(t) 69 | #if h_rank != -1 and t_rank != -1: 70 | vizG.edge(h, t) 71 | 72 | vizG.view() 73 | --------------------------------------------------------------------------------