├── .clang-format
├── .github
    └── workflows
    │   ├── clang-format.yml
    │   ├── ubuntu-20.04.yml
    │   ├── ubuntu-22.04.yml
    │   ├── windows-2019.yml
    │   └── windows-2022.yml
├── .gitignore
├── CMakeLists.txt
├── LICENSE
├── README.md
├── benchmarks
    ├── CMakeLists.txt
    └── spmv
    │   ├── CMakeLists.txt
    │   ├── parameters.hxx
    │   └── work_oriented.cu
├── cmake
    ├── FetchCXXOpts.cmake
    ├── FetchColors.cmake
    ├── FetchGoogleTest.cmake
    ├── FetchModernGPU.cmake
    ├── FetchNVBench.cmake
    └── FetchThrustCUB.cmake
├── datasets
    ├── chesapeake
    │   └── chesapeake.mtx
    └── suitesparse.txt
├── docker
    ├── README.md
    └── dockerfile
├── docs
    ├── .gitkeep
    ├── _config.yml
    ├── abstraction.md
    ├── background.md
    └── loadbalancing_api.md
├── examples
    ├── CMakeLists.txt
    ├── range
    │   ├── CMakeLists.txt
    │   └── range.cu
    ├── saxpy
    │   ├── CMakeLists.txt
    │   └── saxpy.cu
    ├── spmm
    │   ├── CMakeLists.txt
    │   ├── helpers.hxx
    │   └── thread_mapped.cu
    └── spmv
    │   ├── CMakeLists.txt
    │   ├── group_mapped.cu
    │   ├── helpers.hxx
    │   ├── merge_path.cu
    │   ├── original.cu
    │   ├── thread_mapped.cu
    │   └── work_oriented.cu
├── include
    └── loops
    │   ├── algorithms
    │       ├── spmm
    │       │   └── thread_mapped.cuh
    │       └── spmv
    │       │   ├── group_mapped.cuh
    │       │   ├── merge_path_flat.cuh
    │       │   ├── original.cuh
    │       │   ├── thread_mapped.cuh
    │       │   └── work_oriented.cuh
    │   ├── container
    │       ├── container.hxx
    │       ├── coo.hxx
    │       ├── coordinate.hxx
    │       ├── csc.hxx
    │       ├── csr.hxx
    │       ├── detail
    │       │   ├── convert.hxx
    │       │   ├── mmio.cpp
    │       │   └── mmio.hxx
    │       ├── formats.hxx
    │       ├── market.hxx
    │       ├── matrix.cuh
    │       └── vector.hxx
    │   ├── error.hxx
    │   ├── memory.hxx
    │   ├── range.hxx
    │   ├── schedule.hxx
    │   ├── schedule
    │       ├── group_mapped.hxx
    │       ├── merge_path_flat.hxx
    │       ├── thread_mapped.hxx
    │       └── work_oriented.hxx
    │   ├── stride_ranges.hxx
    │   └── util
    │       ├── device.hxx
    │       ├── equal.hxx
    │       ├── filepath.hxx
    │       ├── generate.hxx
    │       ├── launch.hxx
    │       ├── math.hxx
    │       ├── sample.hxx
    │       ├── search.hxx
    │       └── timer.hxx
├── plots
    ├── data
    │   ├── cusparse.csv
    │   ├── group_mapped.csv
    │   ├── heuristics.csv
    │   ├── merge_path.csv
    │   ├── thread_mapped.csv
    │   └── versus_cub.csv
    ├── performance_evaluation.ipynb
    └── requirements.txt
├── scripts
    ├── README.md
    ├── format.sh
    └── run.sh
└── unittests
    └── CMakeLists.txt


/.clang-format:
--------------------------------------------------------------------------------
1 | # Defines the Chromium style for automatic reformatting.
2 | # http://clang.llvm.org/docs/ClangFormatStyleOptions.html
3 | BasedOnStyle: Chromium
4 | # This defaults to 'Auto'. Explicitly set it for a while, so that
5 | # 'vector<vector<int> >' in existing files gets formatted to
6 | # 'vector<vector<int>>'. ('Auto' means that clang-format will only use
7 | # 'int>>' if the file already contains at least one such instance.)
8 | Standard: Cpp11
9 | SortIncludes: false


--------------------------------------------------------------------------------
/.github/workflows/clang-format.yml:
--------------------------------------------------------------------------------
 1 | name: clang-format
 2 | 
 3 | # Controls when the workflow will run
 4 | on:
 5 |   # Triggers the workflow on push or pull request events but only for the main branch
 6 |   push:
 7 |     branches: [ main, dev ]
 8 | 
 9 | jobs:
10 |   formatting-check:
11 |     name: Format
12 |     runs-on: ubuntu-22.04
13 |     steps:
14 |     - uses: actions/checkout@v3
15 |       with:
16 |         ref: ${{ github.event.pull_request.head.ref }}
17 | 
18 |     - name: Install clang-format
19 |       run: |
20 |         pip install clang-format \
21 |         && clang-format --version
22 |       shell: bash
23 | 
24 |     - name: Format files
25 |       run: ${{github.workspace}}/scripts/format.sh ${{github.workspace}} y
26 |       shell: bash
27 | 
28 |     - name: Commit changes
29 |       uses: EndBug/add-and-commit@v9
30 |       with:
31 |         committer_name: GitHub Actions
32 |         committer_email: 41898282+github-actions[bot]@users.noreply.github.com
33 |         message: ':octocat: Applied clang-format. [skip build]'
34 |         add: '["*.cpp", "*.cu", "*.cuh", "*.hxx"]'


--------------------------------------------------------------------------------
/.github/workflows/ubuntu-20.04.yml:
--------------------------------------------------------------------------------
 1 | name: ubuntu-20.04
 2 | 
 3 | # Controls when the workflow will run
 4 | on:
 5 |   # Triggers the workflow on push or pull request events but only for the main branch
 6 |   push:
 7 |     branches: [ main, dev ]
 8 |   pull_request:
 9 |     branches: [ main, dev ]
10 | 
11 |   # Allows you to run this workflow manually from the Actions tab
12 |   workflow_dispatch:
13 |   
14 | env:
15 |   BUILD_TYPE: Release
16 |   ARCHITECTURES: 75
17 | 
18 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel
19 | jobs:
20 |   # This workflow contains a single job called "build"
21 |   build:
22 |     # https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
23 |     if: "!contains(github.event.commits[0].message, '[skip build]')"
24 |     runs-on: ubuntu-20.04
25 | 
26 |     # Steps represent a sequence of tasks that will be executed as part of the job
27 |     steps:
28 |       # Fetch CUDA toolkit using Jimver/cuda-toolkit@v0.2.7
29 |       - name: Fetch CUDA toolkit
30 |         uses: Jimver/cuda-toolkit@v0.2.7
31 |         id: cuda-toolkit
32 |         with:
33 |           cuda: '11.7.0'
34 |           linux-local-args: '["--toolkit"]'
35 | 
36 |       # Runs a single command using the runners shell
37 |       - name: Check nvcc version
38 |         run: nvcc -V
39 |         
40 |       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
41 |       - uses: actions/checkout@v3
42 |       
43 |       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
44 |       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
45 |       - name: Configure cmake
46 |         run: cmake -B ${{github.workspace}}/build -D LOOPS_BUILD_TESTS=ON -D CMAKE_CUDA_ARCHITECTURES=${{env.ARCHITECTURES}} -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
47 |         
48 |       # Build your program with the given configuration
49 |       - name: Build all applications
50 |         run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
51 | 


--------------------------------------------------------------------------------
/.github/workflows/ubuntu-22.04.yml:
--------------------------------------------------------------------------------
 1 | name: ubuntu-22.04
 2 | 
 3 | # Controls when the workflow will run
 4 | on:
 5 |   # Triggers the workflow on push or pull request events but only for the main branch
 6 |   push:
 7 |     branches: [ main, dev ]
 8 |   pull_request:
 9 |     branches: [ main, dev ]
10 | 
11 |   # Allows you to run this workflow manually from the Actions tab
12 |   workflow_dispatch:
13 |   
14 | env:
15 |   BUILD_TYPE: Release
16 |   ARCHITECTURES: 75
17 | 
18 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel
19 | jobs:
20 |   # This workflow contains a single job called "build"
21 |   build:
22 |     # https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
23 |     if: "!contains(github.event.commits[0].message, '[skip build]')"
24 |     runs-on: ubuntu-22.04
25 | 
26 |     # Steps represent a sequence of tasks that will be executed as part of the job
27 |     steps:
28 |       # Fetch CUDA toolkit using Jimver/cuda-toolkit@v0.2.7
29 |       - name: Fetch CUDA toolkit
30 |         uses: Jimver/cuda-toolkit@v0.2.7
31 |         id: cuda-toolkit
32 |         with:
33 |           cuda: '11.7.0'
34 |           linux-local-args: '["--toolkit"]'
35 | 
36 |       # Runs a single command using the runners shell
37 |       - name: Check nvcc version
38 |         run: nvcc -V
39 |         
40 |       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
41 |       - uses: actions/checkout@v3
42 |       
43 |       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
44 |       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
45 |       - name: Configure cmake
46 |         run: cmake -B ${{github.workspace}}/build -D LOOPS_BUILD_TESTS=ON -D CMAKE_CUDA_ARCHITECTURES=${{env.ARCHITECTURES}} -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
47 |         
48 |       # Build your program with the given configuration
49 |       - name: Build all applications
50 |         run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
51 | 


--------------------------------------------------------------------------------
/.github/workflows/windows-2019.yml:
--------------------------------------------------------------------------------
 1 | name: windows-2019
 2 | 
 3 | # Controls when the workflow will run
 4 | on:
 5 |   # Triggers the workflow on push or pull request events but only for the main branch
 6 |   push:
 7 |     branches: [ main, dev ]
 8 |   pull_request:
 9 |     branches: [ main, dev ]
10 | 
11 |   # Allows you to run this workflow manually from the Actions tab
12 |   workflow_dispatch:
13 | 
14 | env:
15 |   BUILD_TYPE: Release
16 |   ARCHITECTURES: 75
17 | 
18 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel
19 | jobs:
20 |   # This workflow contains a single job called "build"
21 |   build:
22 |     # https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
23 |     if: "!contains(github.event.commits[0].message, '[skip build]')"
24 |     runs-on: windows-2019
25 | 
26 |     # Steps represent a sequence of tasks that will be executed as part of the job
27 |     steps:
28 |       # Fetch CUDA toolkit using Jimver/cuda-toolkit@v0.2.7
29 |       - name: Fetch CUDA toolkit
30 |         uses: Jimver/cuda-toolkit@v0.2.7
31 |         id: cuda-toolkit
32 |         with:
33 |           cuda: '11.7.0'
34 |           linux-local-args: '["--toolkit"]'
35 | 
36 |       # Runs a single command using the runners shell
37 |       - name: Check nvcc version
38 |         run: nvcc -V
39 |         
40 |       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
41 |       - uses: actions/checkout@v3
42 |       
43 |       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
44 |       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
45 |       - name: Configure cmake
46 |         run: cmake -B ${{github.workspace}}/build -D LOOPS_BUILD_TESTS=ON -D CMAKE_CUDA_ARCHITECTURES=${{env.ARCHITECTURES}} -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
47 | 
48 |       # See issue: https://github.com/NVIDIA/thrust/issues/1328
49 |       # And... https://github.com/gunrock/essentials/issues/92
50 |       - name: Remove CUB symbolic link
51 |         run: rm ${{github.workspace}}/externals/thrust-src/cub
52 |         
53 |       # Build your program with the given configuration
54 |       - name: Build all applications
55 |         run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
56 | 


--------------------------------------------------------------------------------
/.github/workflows/windows-2022.yml:
--------------------------------------------------------------------------------
 1 | name: windows-2022
 2 | 
 3 | # Controls when the workflow will run
 4 | on:
 5 |   # Triggers the workflow on push or pull request events but only for the main branch
 6 |   push:
 7 |     branches: [ main, dev ]
 8 |   pull_request:
 9 |     branches: [ main, dev ]
10 | 
11 |   # Allows you to run this workflow manually from the Actions tab
12 |   workflow_dispatch:
13 | 
14 | env:
15 |   BUILD_TYPE: Release
16 |   ARCHITECTURES: 75
17 | 
18 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel
19 | jobs:
20 |   # This workflow contains a single job called "build"
21 |   build:
22 |     # https://github.blog/changelog/2021-02-08-github-actions-skip-pull-request-and-push-workflows-with-skip-ci/
23 |     if: "!contains(github.event.commits[0].message, '[skip build]')"
24 |     runs-on: windows-2022
25 | 
26 |     # Steps represent a sequence of tasks that will be executed as part of the job
27 |     steps:
28 |       # Fetch CUDA toolkit using Jimver/cuda-toolkit@v0.2.7
29 |       - name: Fetch CUDA toolkit
30 |         uses: Jimver/cuda-toolkit@v0.2.7
31 |         id: cuda-toolkit
32 |         with:
33 |           cuda: '11.7.0'
34 |           linux-local-args: '["--toolkit"]'
35 | 
36 |       # Runs a single command using the runners shell
37 |       - name: Check nvcc version
38 |         run: nvcc -V
39 |         
40 |       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
41 |       - uses: actions/checkout@v3
42 |       
43 |       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
44 |       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
45 |       - name: Configure cmake
46 |         run: cmake -B ${{github.workspace}}/build -D LOOPS_BUILD_TESTS=ON -D CMAKE_CUDA_ARCHITECTURES=${{env.ARCHITECTURES}} -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
47 | 
48 |       # See issue: https://github.com/NVIDIA/thrust/issues/1328
49 |       # And... https://github.com/gunrock/essentials/issues/92
50 |       - name: Remove CUB symbolic link
51 |         run: rm ${{github.workspace}}/externals/thrust-src/cub
52 |         
53 |       # Build your program with the given configuration
54 |       - name: Build all applications
55 |         run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Ignore vscode
 2 | *.vscode*
 3 | 
 4 | # Ignore vim undo file
 5 | *.un~
 6 | 
 7 | # Compiled Object files
 8 | *.slo
 9 | *.lo
10 | *.o
11 | 
12 | # Compiled Dynamic libraries
13 | *.so
14 | *.dylib
15 | 
16 | # Compiled Static libraries
17 | *.lai
18 | *.la
19 | *.a
20 | 
21 | # Ignore graph dataset files
22 | *.graph
23 | *.mtx*
24 | 
25 | # Ignore tarball
26 | *.tar.gz
27 | 
28 | # Ignore bin directory
29 | bin/
30 | 
31 | # Ignore eval directory
32 | eval/
33 | 
34 | # Ignore vim temp files
35 | *.sw~
36 | 
37 | # Ignore build directory
38 | build
39 | 
40 | # Ignore external fetched content
41 | externals/*
42 | 
43 | # Ignore checkpoints ipynb
44 | */*.ipynb_checkpoints/*
45 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
  1 | ####################################################
  2 | ############ INSTALLING CORRECT CMAKE ##############
  3 | ####################################################
  4 | # Installing correct cmake version is easy!
  5 | # 1) Find the respective version here;
  6 | # https://github.com/Kitware/CMake/releases,
  7 | # and 2) replace the [x.xx.x] in the following
  8 | # commands with the version number (remove the
  9 | # brackets). For example, if you are installing
 10 | # CMake 3.22.1, replace [x.xx.x] with 3.22.1:
 11 | 
 12 | # wget https://github.com/Kitware/CMake/releases/download/v[x.xx.x]/cmake-[x.xx.x]-linux-x86_64.sh
 13 | # chmod +x ./cmake-[x.xx.x]-linux-x86_64.sh
 14 | # ./cmake-[x.xx.x]-linux-x86_64.sh
 15 | # sudo mv cmake-[x.xx.x]-linux-x86_64 /opt/cmake
 16 | # sudo ln -s /opt/cmake/bin/* /usr/local/bin/
 17 | cmake_minimum_required(VERSION 3.24 FATAL_ERROR)
 18 | 
 19 | # Select "Release" as the default build type.
 20 | # This can be altered by setting -DCMAKE_BUILD_TYPE
 21 | # in the command-line interface to Release or Debug.
 22 | # No reason to set CMAKE_CONFIGURATION_TYPES if it's 
 23 | # not a multiconfig generator. Also no reason mess 
 24 | # with CMAKE_BUILD_TYPE if it's a multiconfig generator.
 25 | # https://stackoverflow.com/a/31548693/5729690
 26 | get_property(isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 27 | if(isMultiConfig)
 28 |     set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) 
 29 | else()
 30 |     if(NOT CMAKE_BUILD_TYPE)
 31 |         message(STATUS "Defaulting to Release build type")
 32 |         set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE)
 33 |     endif()
 34 |     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY HELPSTRING "Choose the type of build")
 35 |     # set the valid options for cmake-gui drop-down list
 36 |     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug;Release")
 37 | endif()
 38 | 
 39 | ####################################################
 40 | ############### SET SM ARCHITECTURE ################
 41 | ####################################################
 42 | 
 43 | ## Note: This applies to NVBench as well.
 44 | ## Can be used for applications by extracting the
 45 | ## CUDA_ARCHITECTURES property from loops project.
 46 | ## see: get_target_properties()
 47 | ## see: https://github.com/RadeonOpenCompute/rocminfo/blob/master/rocm_agent_enumerator#L12
 48 | if(NOT CMAKE_CUDA_ARCHITECTURES)
 49 |   set(CMAKE_CUDA_ARCHITECTURES 70)
 50 |   message(STATUS "Using default GPU Architecture (CMAKE_CUDA_ARCHITECTURES): ${CMAKE_CUDA_ARCHITECTURES}")
 51 | else()
 52 |   message(STATUS "GPU Architecture (CMAKE_CUDA_ARCHITECTURES): ${CMAKE_CUDA_ARCHITECTURES}")
 53 | endif()
 54 | 
 55 | project(loops
 56 |   LANGUAGES CXX C CUDA
 57 | )
 58 | 
 59 | # begin /* Dependencies directory */
 60 | set(PROJECT_DEPS_DIR externals)
 61 | # end /* Dependencies directory */
 62 | 
 63 | # begin /* Include cmake modules */
 64 | include(${PROJECT_SOURCE_DIR}/cmake/FetchColors.cmake)
 65 | include(${PROJECT_SOURCE_DIR}/cmake/FetchThrustCUB.cmake)
 66 | include(${PROJECT_SOURCE_DIR}/cmake/FetchCXXOpts.cmake)
 67 | # end /* Include cmake modules */
 68 | 
 69 | ## Set the directory where the binaries will be stored
 70 | set(EXECUTABLE_OUTPUT_PATH
 71 |   ${PROJECT_BINARY_DIR}/bin
 72 |   CACHE PATH
 73 |   "Directory where all executables will be stored")
 74 | 
 75 | ## Set the directory where the libraries will be stored
 76 | set(LIBRARY_OUTPUT_PATH
 77 |   ${PROJECT_BINARY_DIR}/lib
 78 |   CACHE PATH
 79 |   "Directory where all the libraries will be stored")
 80 | 
 81 | ## Export compile commands
 82 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 83 | set(CMAKE_VERBOSE_MAKEFILE OFF)
 84 | 
 85 | ############ ADD LIBRARY: LOOPS (HEADER-ONLY) ############
 86 | add_library(loops INTERFACE)
 87 | 
 88 | ####################################################
 89 | ############### SET TARGET PROPERTIES ##############
 90 | ####################################################
 91 | set_target_properties(loops 
 92 |   PROPERTIES
 93 |     CXX_STANDARD 17
 94 |     CXX_STANDARD_REQUIRED ON
 95 |     CXX_EXTENSIONS OFF # Should this be turned on for MSVC?
 96 |     CUDA_STANDARD 17
 97 |     CUDA_STANDARD_REQUIRED ON
 98 |     CUDA_EXTENSIONS OFF
 99 |     CUDA_RESOLVE_DEVICE_SYMBOLS ON
100 |     CUDA_SEPARABLE_COMPILATION ON
101 |     CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
102 |     # CUDA_PTX_COMPILATION ON # Can only be applied to OBJ.
103 | )
104 | 
105 | ####################################################
106 | ############ TARGET COMPILER DEFINITIONS ###########
107 | ####################################################
108 | target_compile_definitions(loops
109 |   INTERFACE 
110 |     SM_TARGET=${CMAKE_CUDA_ARCHITECTURES}
111 |     LOOPS_VERSION=${LOOPS_VERSION}
112 | )
113 | 
114 | message(STATUS "Loops CUDA Architecture: ${CMAKE_CUDA_ARCHITECTURES}")
115 | 
116 | ####################################################
117 | ############ TARGET COMPILE FEATURES ###############
118 | ####################################################
119 | # Turn C++ Standard 17 ON.
120 | target_compile_features(loops INTERFACE cxx_std_17)
121 | # set(CMAKE_CXX_EXTENSIONS OFF)
122 | 
123 | set(LOOPS_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
124 | 
125 | ####################################################
126 | ############ TARGET INCLUDE DIRECTORIES ############
127 | ####################################################
128 | target_include_directories(loops
129 |   INTERFACE ${LOOPS_INCLUDE_DIR}
130 |   INTERFACE ${CXXOPTS_INCLUDE_DIR}
131 |   INTERFACE ${CUB_INCLUDE_DIR}
132 |   INTERFACE ${THRUST_INCLUDE_DIR}
133 |   INTERFACE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
134 | )
135 | 
136 | ####################################################
137 | ############ TARGET LINK LIBRARIES #################
138 | ####################################################
139 | target_link_libraries(loops
140 |   INTERFACE curand
141 |   INTERFACE cuda
142 | )
143 | 
144 | ####################################################
145 | ################# TARGET SOURCES ###################
146 | ####################################################
147 | target_sources(loops
148 |   INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/loops/container/detail/mmio.cpp"
149 | )
150 | 
151 | ####################################################
152 | ############## SET CXX & CUDA FLAGS ################
153 | ####################################################
154 | set(CXX_FLAGS 
155 |   $<$<CXX_COMPILER_ID:MSVC>:
156 |     /W4
157 |   >
158 |   $<$<CXX_COMPILER_ID:GNU>:
159 |     -Wall 
160 |     # -Wextra
161 |     -Wno-unused-result
162 |     -Wno-unused-local-typedefs
163 |     -Wno-strict-aliasing
164 |     -Wno-unused-function
165 |     -Wno-format-security
166 |     # -Werror
167 |     # -vvv
168 |   >
169 | )
170 | 
171 | set(CUDA_RELEASE_FLAGS
172 |   --expt-extended-lambda 
173 |   --expt-relaxed-constexpr
174 |   --use_fast_math
175 | )
176 | 
177 | set(CUDA_DEBUG_FLAGS
178 |   --expt-extended-lambda 
179 |   --expt-relaxed-constexpr
180 |   --ptxas-options -v
181 |   --debug # Host debug
182 |   --device-debug # Device debug
183 | )
184 | 
185 | ####################################################
186 | ############ TARGET COMPILE OPTIONS ################
187 | ####################################################
188 | target_compile_options(loops INTERFACE 
189 |   $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CONFIG:DEBUG>>:${CXX_FLAGS}>
190 |   $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CONFIG:DEBUG>>:${CUDA_DEBUG_FLAGS}>
191 |   $<$<AND:$<COMPILE_LANGUAGE:CUDA>,$<CONFIG:RELEASE>>:${CUDA_RELEASE_FLAGS}>
192 | )
193 | 
194 | ####################################################
195 | ############ BUILD EXAMPLE APPLICATIONS ############
196 | ####################################################
197 | option(LOOPS_BUILD_EXAMPLES
198 |   "If on, builds the example applications."
199 |   ON)
200 | 
201 | # Subdirectories for examples, testing and documentation
202 | if(LOOPS_BUILD_EXAMPLES)
203 |   add_subdirectory(examples)
204 | endif(LOOPS_BUILD_EXAMPLES)
205 | 
206 | ####################################################
207 | ################ BUILD UNIT TESTS  #################
208 | ####################################################
209 | option(LOOPS_BUILD_TESTS
210 |   "If on, builds the unit tests."
211 |   OFF)
212 | 
213 | # Subdirectories for examples, testing and documentation
214 | if(LOOPS_BUILD_TESTS)
215 |   include(${PROJECT_SOURCE_DIR}/cmake/FetchGoogleTest.cmake)
216 |   # add_subdirectory(unittests)
217 | endif(LOOPS_BUILD_TESTS)
218 | 
219 | ####################################################
220 | ################ BUILD BENCHMARKS  #################
221 | ####################################################
222 | option(LOOPS_BUILD_BENCHMARKS
223 |   "If on, builds loops with benchmarking support."
224 |   OFF)
225 | 
226 | # Subdirectories for examples, testing and documentation
227 | if(LOOPS_BUILD_BENCHMARKS)
228 |   # ... see https://github.com/NVIDIA/nvbench/issues/66
229 |   set(NVBench_ENABLE_NVML OFF)
230 |   include(${PROJECT_SOURCE_DIR}/cmake/FetchNVBench.cmake)
231 |   add_subdirectory(benchmarks)
232 | endif(LOOPS_BUILD_BENCHMARKS)


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright 2022 Regents of the University of California
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # 🐧 `loops`: Expressing Parallel Irregular Computations
  2 | 
  3 | [![ubuntu-20.04](https://github.com/gunrock/loops/actions/workflows/ubuntu-20.04.yml/badge.svg)](https://github.com/gunrock/loops/actions/workflows/ubuntu-20.04.yml) [![ubuntu-22.04](https://github.com/gunrock/loops/actions/workflows/ubuntu-22.04.yml/badge.svg)](https://github.com/gunrock/loops/actions/workflows/ubuntu-22.04.yml) [![windows-2019](https://github.com/gunrock/loops/actions/workflows/windows-2019.yml/badge.svg)](https://github.com/gunrock/loops/actions/workflows/windows-2019.yml) [![windows-2022](https://github.com/gunrock/loops/actions/workflows/windows-2022.yml/badge.svg)](https://github.com/gunrock/loops/actions/workflows/windows-2022.yml) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7465053.svg)](https://doi.org/10.5281/zenodo.7465053)
  4 | 
  5 | We propose an open-source GPU load-balancing framework for applications that exhibit irregular parallelism. The set of applications and algorithms we consider are fundamental to computing tasks ranging from sparse machine learning, large numerical simulations, and on through to graph analytics. The underlying data and data structures that drive these applications present access patterns that naturally don't map well to the GPU's architecture that is designed with dense and regular patterns in mind. 
  6 | 
  7 | Prior to the work we present and propose here, the only way to unleash the GPU's full power on these problems has been to workload balance through tightly coupled load-balancing techniques. Our proposed load-balancing abstraction decouples load balancing from work processing and aims to support both static and dynamic schedules with a programmable interface to implement new load-balancing schedules in the future. 
  8 | 
  9 | With our open-source framework, we hope to not only improve programmers' productivity when developing irregular-parallel algorithms on the GPU but also improve the overall performance characteristics for such applications by allowing a quick path to experimentation with a variety of existing load-balancing techniques. Consequently, we also hope that by separating the concerns of load-balancing from work processing within our abstraction, managing and extending existing code to future architectures becomes easier.
 10 | 
 11 | ## Requirements
 12 | - **OS:** Ubuntu 18.04, 20.04, Windows
 13 | - **Hardware:** NVIDIA GPU (Volta or newer)
 14 | - **Software:** CUDA 11.7 or above and cmake 3.20.1 or above.
 15 | - **CUDA Architecture:** SM 70 or above (see [GPUs supported](https://en.wikipedia.org/wiki/CUDA#GPUs_supported)), this is specified using cmake's command: `-DCMAKE_CUDA_ARCHITECTURES=70`. Alternatively, set the CUDA architecture version in the `CMakeLists.txt` file directly: [CMakeLists.txt#72](https://github.com/gunrock/loops/blob/main/CMakeLists.txt#L72).
 16 | 
 17 | ## Getting Started
 18 | Before building `loops` make sure you have CUDA Toolkit and cmake installed on your system, and exported in `PATH` of your system. Other external dependencies such as `NVIDIA/thrust`, `NVIDIA/cub`, etc. are automatically fetched using cmake.
 19 | 
 20 | ```bash
 21 | git clone https://github.com/gunrock/loops.git
 22 | cd loops
 23 | mkdir build && cd build
 24 | cmake -DCMAKE_CUDA_ARCHITECTURES=70 .. # Volta = 70, Turing = 75, ...
 25 | make -j$(nproc)
 26 | bin/loops.spmv.merge_path -m ../datasets/chesapeake/chesapeake.mtx
 27 | ```
 28 | 
 29 | ### Building Specific Algorithms
 30 | 
 31 | ```bash
 32 | make loops.spmv.<algorithm>
 33 | ```
 34 | Replaced the `<algorithm>` with one of the following algorithm names to build a specific SpMV algorithm instead of all of them:
 35 | - `original`
 36 | - `thread_mapped`
 37 | - `group_mapped`
 38 | - `work_oriented`
 39 | - `merge_path`
 40 | 
 41 | An example of the above: `make loops.spmv.merge_path`.
 42 | 
 43 | ## Datasets
 44 | 
 45 | To download the SuiteSparse Matrix Collection[^1], simply run the following command. We recommend using a `tmux` session, because downloading the entire collection can take a significant time. Uncompress the dataset by running the following command in the dataset's directory `find . -name '*.tar.gz' -execdir tar -xzvf '{}' \;
 46 | ` The total downloaded size of the dataset is nontrivial: uncompressed + compressed = 887GB.
 47 | ```bash
 48 | wget --recursive --no-parent --force-directories -l inf -X RB,mat \ 
 49 | --accept "*.tar.gz" "https://suitesparse-collection-website.herokuapp.com/"
 50 | ```
 51 | 
 52 | - `--recursive` recursively download
 53 | - `--no-parent` prevent wget from starting to fetch links in the parent of the website
 54 | - `--l inf` keep downloading for an infinite level
 55 | - `-X RB,mat` ignore subdirectories RB and mat, since I am only downloading matrix market MM, you can choose to download any of the others or remove this entirely to download all formats
 56 | - `--accept` accept the following extension only
 57 | - `--force-directories` create a hierarchy of directories, even if one would not have been created otherwise
 58 | 
 59 | [^1]: Timothy A. Davis and Yifan Hu. 2011. The University of Florida Sparse Matrix Collection. ACM Transactions on Mathematical Software 38, 1, Article 1 (December 2011), 25 pages. DOI: https://doi.org/10.1145/2049662.2049663
 60 | 
 61 | ## Experimentation
 62 | If CUDA and cmake are already setup, follow the [Getting Started](#getting-started) instructions. Or, you may prefer to set up the entire project using docker, and for that we have provided a docker file and instructions on how to use it in [/docker](https://github.com/gunrock/loops/tree/main/docker) directory.
 63 | 
 64 | ### Sanity Check
 65 | Run the following command in the cmake's `build` folder:
 66 | ```bash
 67 | bin/loops.spmv.merge_path -m ../datasets/chesapeake/chesapeake.mtx \
 68 |  --validate -v
 69 | ```
 70 | You should approximately see the following output:
 71 | ```bash
 72 | ~/loops/build$ bin/loops.spmv.merge_path \
 73 |  -m ../datasets/chesapeake/chesapeake.mtx --validate -v
 74 | Elapsed (ms):   0.063328
 75 | Matrix:         chesapeake.mtx
 76 | Dimensions:     39 x 39 (340)
 77 | Errors:         0
 78 | ```
 79 | ## Reproducing Results
 80 | > Find pre-generated results in [plots/](https://github.com/gunrock/loops/blob/main/plots/) directory along with `performance_evaluation.ipynb` notebook to recreate the plots (labeled figures) found in the paper.
 81 | 
 82 | 1. In the run script, update the `DATASET_DIR` to point to the path of all the downloaded datasets (set to the path of the directory containing `MM` directory, and inside the `MM` it has subdirectories with `.mtx` files): [scripts/run.sh](https://github.com/gunrock/loops/blob/main/scripts/run.sh). Additionally, you may change the path to `DATASET_FILES_NAME` containing the list of all the datasets (default points to [datasets/suitesparse.txt](https://github.com/gunrock/loops/blob/main/datasets/suitesparse.txt)).
 83 | 2. Fire up the complete run using `run.sh` found in `scripts` directory, `cd scripts && ./run.sh`, note one complete run can take up to 3 days (goes over the entire suitesparse matrix collection dataset four times with four different algorithms, the main bottleneck is loading files from disk.)
 84 | 3. **Warning!** Some runs on the matrices are expected to fail as they are not in proper MatrixMarket Format although labeled as `.mtx`. These matrices and the ones that do not fit on the GPU will result in runtime exceptions or `offset_t` type overflow and can be safely ignored.
 85 | 4. To run *N* number of datasets simply adjust the stop condition here (default set to `10`): [scripts/run.sh#L22](https://github.com/gunrock/loops/blob/main/scripts/run.sh#L22), or remove this if-condition entirely to run on all available `.mtx` files: [scripts/run.sh#L22-L26](https://github.com/gunrock/loops/blob/main/scripts/run.sh#L22-L26).
 86 | 
 87 | Expected output from the above runs are `csv` files in the same directory as the `run.sh`, these can replace the existing `csv` files within `plots/data`, and a [python jupyter notebook](https://jupyter.org/install) can be fired up to evaluate the results. Python notebook includes instructions on generating plots. See sample output of one of the `csv` files below:
 88 | 
 89 | ```csv
 90 | kernel,dataset,rows,cols,nnzs,elapsed
 91 | merge-path,144,144649,144649,2148786,0.0720215
 92 | merge-path,08blocks,300,300,592,0.0170898
 93 | merge-path,1138_bus,1138,1138,4054,0.0200195
 94 | ```
 95 | 
 96 | ## How to Cite Loops
 97 | Thank you for citing our work.
 98 | 
 99 | ```bibtex
100 | @inproceedings{Osama:2023:APM,
101 |   author       = {Muhammad Osama and Serban D. Porumbescu and John D. Owens},
102 |   title        = {A Programming Model for {GPU} Load Balancing},
103 |   booktitle    = {Proceedings of the 28th ACM SIGPLAN Symposium on
104 |                   Principles and Practice of Parallel Programming},
105 |   series       = {PPoPP 2023},
106 |   year         = 2023,
107 |   month        = feb # "\slash " # mar,
108 |   acceptance   = {31 of 131 submissions, 23.7\%},
109 |   code         = {https://github.com/gunrock/loops},
110 |   doi          = {10.1145/3572848.3577434},
111 | }
112 | ```
113 | 
114 | ```bibtex
115 | @software{Osama:2022:LAP:Code,
116 |   author       = {Muhammad Osama and Serban D. Porumbescu and John D. Owens},
117 |   title        = {Loops: A Programming Model for GPU Load Balancing},
118 |   month        = dec,
119 |   year         = 2022,
120 |   publisher    = {Zenodo},
121 |   version      = {v0.1.0-alpha},
122 |   doi          = {10.5281/zenodo.7465053},
123 |   url          = {https://doi.org/10.5281/zenodo.7465053}
124 | }
125 | ```
126 | 


--------------------------------------------------------------------------------
/benchmarks/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | # begin /* Add benchmarks' subdirectories */
2 | add_subdirectory(spmv)
3 | # end /* Add benchmarks' subdirectories */


--------------------------------------------------------------------------------
/benchmarks/spmv/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | set(BENCHMARK_SOURCES
 2 |   work_oriented.cu
 3 | )
 4 | 
 5 | foreach(SOURCE IN LISTS BENCHMARK_SOURCES)
 6 |   get_filename_component(BENCHMARK_NAME ${SOURCE} NAME_WLE)
 7 |   string(PREPEND BENCHMARK_NAME "loops.bench.spmv.")
 8 |   add_executable(${BENCHMARK_NAME} ${SOURCE})
 9 |   target_link_libraries(${BENCHMARK_NAME}
10 |     PRIVATE loops
11 |     # PRIVATE nvbench::main
12 |     PRIVATE nvbench::nvbench
13 |   )
14 | 
15 |   set_target_properties(${BENCHMARK_NAME}
16 |     PROPERTIES
17 |       CXX_STANDARD 17
18 |       CUDA_STANDARD 17
19 |       CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
20 |   )
21 |   
22 |   message(STATUS "Benchmark Added: ${BENCHMARK_NAME}")
23 | endforeach()


--------------------------------------------------------------------------------
/benchmarks/spmv/parameters.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file parameters.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief SpMV + NVBench parameters.
 5 |  * @version 0.1
 6 |  * @date 2022-07-18
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <string>
15 | #include <iostream>
16 | #include <cxxopts.hpp>
17 | 
18 | #include <loops/util/filepath.hxx>
19 | #include <nvbench/nvbench.cuh>
20 | 
21 | std::string filename;
22 | 
23 | struct parameters_t {
24 |   /**
25 |    * @brief Construct a new parameters object and parse command line arguments.
26 |    *
27 |    * @param argc Number of command line arguments.
28 |    * @param argv Command line arguments.
29 |    */
30 |   parameters_t(int argc, char** argv)
31 |       : m_options(argv[0], "SPMV Benchmarking"), m_argc(argc) {
32 |     m_options.allow_unrecognised_options();
33 |     // Add command line options
34 |     m_options.add_options()("h,help", "Print help")  // help
35 |         ("m,market", "Matrix file",
36 |          cxxopts::value<std::string>());  // mtx
37 | 
38 |     // Parse command line arguments.
39 |     auto result = m_options.parse(argc, argv);
40 | 
41 |     // Print help if requested
42 |     if (result.count("help")) {
43 |       m_help = true;
44 |       std::cout << m_options.help({""});
45 |       std::cout << "  [optional nvbench args]" << std::endl << std::endl;
46 |       const char* argh[1] = {"-h"};
47 |       NVBENCH_MAIN_BODY(1, argh);
48 |     }
49 | 
50 |     // Get matrix market file or error if not specified.
51 |     else {
52 |       if (result.count("market") == 1) {
53 |         this->m_filename = result["market"].as<std::string>();
54 |         filename = m_filename;
55 |         if (!loops::is_market(m_filename)) {
56 |           std::cout << m_options.help({""});
57 |           std::cout << "  [optional nvbench args]" << std::endl << std::endl;
58 |           std::exit(0);
59 |         }
60 | 
61 |         // Remove loops parameters and pass the rest to nvbench.
62 |         for (int i = 0; i < argc; i++) {
63 |           if (strcmp(argv[i], "--market") == 0 || strcmp(argv[i], "-m") == 0) {
64 |             i++;
65 |             continue;
66 |           }
67 |           m_args.push_back(argv[i]);
68 |         }
69 | 
70 |       } else {
71 |         std::cout << m_options.help({""});
72 |         std::cout << "  [optional nvbench args]" << std::endl << std::endl;
73 |         std::exit(0);
74 |       }
75 |     }
76 |   }
77 | 
78 |   /// Helpers for NVBENCH_MAIN_BODY call.
79 |   int nvbench_argc() { return m_argc - 2; }
80 |   auto nvbench_argv() { return m_args.data(); }
81 | 
82 |  private:
83 |   std::string m_filename;           ///< Matrix market file.
84 |   cxxopts::Options m_options;       ///< Command line options.
85 |   std::vector<const char*> m_args;  ///< Command line arguments.
86 |   bool m_help = false;              ///< Help flag.
87 |   int m_argc;                       ///< Number of command line arguments.
88 | };


--------------------------------------------------------------------------------
/benchmarks/spmv/work_oriented.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file work_oriented.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Benchmark for Sparse Matrix-Vector Multiplication.
 5 |  * @version 0.1
 6 |  * @date 2022-07-18
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "parameters.hxx"
13 | 
14 | #include <loops/util/generate.hxx>
15 | #include <loops/container/container.hxx>
16 | #include <loops/algorithms/spmv/work_oriented.cuh>
17 | 
18 | #define LOOPS_CUPTI_SUPPORTED 0
19 | 
20 | using namespace loops;
21 | 
22 | template <typename value_t>
23 | void work_oriented_bench(nvbench::state& state, nvbench::type_list<value_t>) {
24 |   using index_t = int;
25 |   using offset_t = int;
26 |   using type_t = value_t;
27 | 
28 |   matrix_market_t<index_t, offset_t, type_t> mtx;
29 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(filename));
30 | 
31 |   vector_t<type_t> x(csr.rows);
32 |   vector_t<type_t> y(csr.rows);
33 | 
34 |   generate::random::uniform_distribution(x.begin(), x.end(), type_t(1.0),
35 |                                          type_t(10.0));
36 | 
37 | #if LOOPS_CUPTI_SUPPORTED
38 |   /// Add CUPTI metrics to collect for the state.
39 |   state.collect_dram_throughput();
40 |   state.collect_l1_hit_rates();
41 |   state.collect_l2_hit_rates();
42 |   state.collect_loads_efficiency();
43 |   state.collect_stores_efficiency();
44 | #endif
45 | 
46 |   /// Execute the benchmark.
47 |   state.exec(nvbench::exec_tag::sync, [&csr, &x, &y](nvbench::launch& launch) {
48 |     algorithms::spmv::work_oriented(csr, x, y);
49 |   });
50 | }
51 | 
52 | // Define a type_list to use for the type axis:
53 | using value_types = nvbench::type_list<int, float, double>;
54 | NVBENCH_BENCH_TYPES(work_oriented_bench, NVBENCH_TYPE_AXES(value_types));
55 | 
56 | int main(int argc, char** argv) {
57 |   parameters_t params(argc, argv);
58 |   NVBENCH_MAIN_BODY(params.nvbench_argc(), params.nvbench_argv());
59 | }


--------------------------------------------------------------------------------
/cmake/FetchCXXOpts.cmake:
--------------------------------------------------------------------------------
 1 | include(FetchContent)
 2 | set(FETCHCONTENT_QUIET ON)
 3 | 
 4 | message(STATUS "Cloning External Project: CXXOPTS")
 5 | get_filename_component(FC_BASE "../externals"
 6 |                 REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 7 | set(FETCHCONTENT_BASE_DIR ${FC_BASE})
 8 | 
 9 | FetchContent_Declare(
10 |   cxxopts
11 |     GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git
12 |     GIT_TAG        v3.0.0
13 | )
14 | 
15 | FetchContent_GetProperties(cxxopts)
16 | if(NOT cxxopts_POPULATED)
17 |   FetchContent_Populate(
18 |     cxxopts
19 |   )
20 | endif()
21 | set(CXXOPTS_INCLUDE_DIR "${cxxopts_SOURCE_DIR}/include")


--------------------------------------------------------------------------------
/cmake/FetchColors.cmake:
--------------------------------------------------------------------------------
 1 | if(NOT WIN32)
 2 |   string(ASCII 27 Esc)
 3 |   set(ColourReset "${Esc}[m")
 4 |   set(ColourBold  "${Esc}[1m")
 5 |   set(Red         "${Esc}[31m")
 6 |   set(Green       "${Esc}[32m")
 7 |   set(Yellow      "${Esc}[33m")
 8 |   set(Blue        "${Esc}[34m")
 9 |   set(Magenta     "${Esc}[35m")
10 |   set(Cyan        "${Esc}[36m")
11 |   set(White       "${Esc}[37m")
12 |   set(BoldRed     "${Esc}[1;31m")
13 |   set(BoldGreen   "${Esc}[1;32m")
14 |   set(BoldYellow  "${Esc}[1;33m")
15 |   set(BoldBlue    "${Esc}[1;34m")
16 |   set(BoldMagenta "${Esc}[1;35m")
17 |   set(BoldCyan    "${Esc}[1;36m")
18 |   set(BoldWhite   "${Esc}[1;37m")
19 | endif()


--------------------------------------------------------------------------------
/cmake/FetchGoogleTest.cmake:
--------------------------------------------------------------------------------
 1 | include(FetchContent)
 2 | set(FETCHCONTENT_QUIET ON)
 3 | 
 4 | message(STATUS "Cloning External Project: GoogleTests")
 5 | get_filename_component(FC_BASE "../externals"
 6 |                 REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 7 | set(FETCHCONTENT_BASE_DIR ${FC_BASE})
 8 | 
 9 | FetchContent_Declare(
10 |     googletest
11 |     GIT_REPOSITORY https://github.com/google/googletest.git
12 |     GIT_TAG        release-1.11.0
13 | )
14 | 
15 | # For Windows: Prevent overriding the parent project's compiler/linker settings
16 | if (MSVC)
17 |     set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
18 | endif()
19 | 
20 | FetchContent_MakeAvailable(googletest)
21 | add_library(gtest::main ALIAS gtest_main)
22 | 
23 | 


--------------------------------------------------------------------------------
/cmake/FetchModernGPU.cmake:
--------------------------------------------------------------------------------
 1 | include(FetchContent)
 2 | set(FETCHCONTENT_QUIET ON)
 3 | 
 4 | message(STATUS "Cloning External Project: ModernGPU")
 5 | get_filename_component(FC_BASE "../externals"
 6 |                 REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 7 | set(FETCHCONTENT_BASE_DIR ${FC_BASE})
 8 | 
 9 | FetchContent_Declare(
10 |     moderngpu
11 |     GIT_REPOSITORY https://github.com/moderngpu/moderngpu.git
12 |     GIT_TAG        master
13 | )
14 | 
15 | FetchContent_GetProperties(moderngpu)
16 | if(NOT moderngpu_POPULATED)
17 |   FetchContent_Populate(
18 |     moderngpu
19 |   )
20 | endif()
21 | set(MODERNGPU_INCLUDE_DIR "${moderngpu_SOURCE_DIR}/src")
22 | 


--------------------------------------------------------------------------------
/cmake/FetchNVBench.cmake:
--------------------------------------------------------------------------------
 1 | include(FetchContent)
 2 | set(FETCHCONTENT_QUIET ON)
 3 | 
 4 | message(STATUS "Cloning External Project: NVBench")
 5 | get_filename_component(FC_BASE "../externals"
 6 |                 REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 7 | set(FETCHCONTENT_BASE_DIR ${FC_BASE})
 8 | 
 9 | FetchContent_Declare(
10 |     nvbench
11 |     GIT_REPOSITORY https://github.com/NVIDIA/nvbench.git
12 |     GIT_TAG        main
13 | )
14 | 
15 | FetchContent_GetProperties(nvbench)
16 | if(NOT nvbench_POPULATED)
17 |   FetchContent_Populate(
18 |     nvbench
19 |   )
20 | endif()
21 | 
22 | # Exposing nvbench's source and include directory
23 | set(NVBENCH_INCLUDE_DIR "${nvbench_SOURCE_DIR}")
24 | set(NVBENCH_BUILD_DIR "${nvbench_BINARY_DIR}")
25 | 
26 | # Add subdirectory ::nvbench
27 | add_subdirectory(${NVBENCH_INCLUDE_DIR} ${NVBENCH_BUILD_DIR})


--------------------------------------------------------------------------------
/cmake/FetchThrustCUB.cmake:
--------------------------------------------------------------------------------
 1 | include(FetchContent)
 2 | set(FETCHCONTENT_QUIET ON)
 3 | 
 4 | message(STATUS "Cloning External Project: Thrust and CUB")
 5 | get_filename_component(FC_BASE "../externals"
 6 |                 REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 7 | set(FETCHCONTENT_BASE_DIR ${FC_BASE})
 8 | 
 9 | FetchContent_Declare(
10 |     thrust
11 |     GIT_REPOSITORY https://github.com/thrust/thrust.git
12 |     GIT_TAG        1.17.2
13 | )
14 | 
15 | FetchContent_GetProperties(thrust)
16 | if(NOT thrust_POPULATED)
17 |   FetchContent_Populate(
18 |     thrust
19 |   )
20 | endif()
21 | set(THRUST_INCLUDE_DIR "${thrust_SOURCE_DIR}")
22 | # Windows doesn't support symblink, so make sure we link to the real library.
23 | set(CUB_INCLUDE_DIR "${thrust_SOURCE_DIR}/dependencies/cub")


--------------------------------------------------------------------------------
/docker/README.md:
--------------------------------------------------------------------------------
 1 | # Docker Instructions
 2 | ## Using the provided `dockerfile`, simply run the following commands.
 3 | ```bash
 4 | docker build . -t loops
 5 | docker run -it loops:latest /bin/bash
 6 | ```
 7 | 
 8 | ## Alternatively pull directly from [hub.docker.com](https://hub.docker.com/repository/docker/neoblizz/loops).
 9 | 
10 | ```bash
11 | docker pull neoblizz/loops:v0.1
12 | ```
13 | 
14 | ## Once within the docker...
15 | ```bash
16 | cd loops/build
17 | bin/loops.spmv.merge_path -m ../datasets/chesapeake/chesapeake.mtx
18 | ```


--------------------------------------------------------------------------------
/docker/dockerfile:
--------------------------------------------------------------------------------
 1 | # Or directly get from docker:
 2 | # docker pull neoblizz/loops:v0.1
 3 | FROM nvidia/cuda:11.7.0-devel-ubuntu20.04
 4 | 
 5 | # Install dependencies
 6 | RUN apt-get update
 7 | RUN apt-get install -y build-essential
 8 | RUN apt-get install -y wget vim git gcc g++
 9 | 
10 | RUN wget https://github.com/Kitware/CMake/releases/download/v3.25.0/cmake-3.25.0-linux-x86_64.sh
11 | RUN chmod +x ./cmake-3.25.0-linux-x86_64.sh
12 | RUN mkdir /opt/cmake-3.25.0/
13 | RUN ./cmake-3.25.0-linux-x86_64.sh --skip-license --prefix=/opt/cmake-3.25.0
14 | RUN ln -s /opt/cmake-3.25.0/bin/* /usr/local/bin/
15 | 
16 | 
17 | # Build and install
18 | RUN git clone https://github.com/gunrock/loops.git
19 | RUN cd loops && mkdir build && cd build && cmake .. && make -j$(nproc)


--------------------------------------------------------------------------------
/docs/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gunrock/loops/6169cf64d06e17b24b7a687fe0baf7ba2347002b/docs/.gitkeep


--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | theme: jekyll-theme-minimal


--------------------------------------------------------------------------------
/docs/abstraction.md:
--------------------------------------------------------------------------------
 1 | [README](/README.md) > **Abstraction**
 2 | 
 3 | # Abstraction
 4 | 
 5 | The simple idea behind our load-balancing abstraction is to represent sparse formats as atoms, tiles and set functional abstraction elements described in the "Function and Set Notation" below. Once represented as such, we can develop load-balancing algorithms that create balanced ranges of atoms and tiles and map them to processor ids. This information can be abstracted to the user with a simple API (such as ranged-for-loops) to capture user-defined computations. Some benefits of this approach are: (1) the user-defined computation remains largely the same for many different static or dynamic load-balancing schedules, (2) these schedules can now be extended to other computations and (3) dramatically reduces code complexity.
 6 | 
 7 | ## As function and set notation.
 8 | 
 9 | Given a sparse-irregular problem $S$ made of many subsets called tiles, $T$. $T_i$ is defined as a collection of atoms, where an atom is the smallest possible processing element (for example, a nonzero element within a sparse-matrix). Using a scheduler, our abstraction's goal is to create a new set, $M$, which maps the processor ids (thread ids for a given kernel execution) $P_{id}$ to a group of subsets of $T$: $M = \{ P_{id}, T_i ... T_j \}$, map of processor ids to tiles, and the scheduler responsible for creating the maps: $L(S) = \{ M_0, ..., M_m\}$.
10 | 
11 | ## As three domains: data, schedule and computation.
12 | 
13 | ![illustration](https://user-images.githubusercontent.com/9790745/168728299-6b125b44-894a-49bb-92fd-ee85aaa80ae4.png)


--------------------------------------------------------------------------------
/docs/background.md:
--------------------------------------------------------------------------------
 1 | [README](/README.md) > **Background**
 2 | 
 3 | # Background
 4 | **DARPA** announced [**Software Defined Hardware (SDH)**](https://www.darpa.mil/program/software-defined-hardware)[^2], a program that aims "*to build runtime-reconfigurable hardware and software that enables near ASIC performance without sacrificing programmability for data-intensive algorithms.*" **NVIDIA** leading the charge on the program, internally called, [**Symphony**](https://blogs.nvidia.com/blog/2018/07/24/darpa-research-post-moores-law/). Our work is a small but important piece of this larger puzzle. The "data-intensive algorithms" part of the program includes domains like Machine Learning, Graph Processing, Sparse-Matrix-Vector algorithms, etc. where there is a large amount of data available to be processed. And the problems being addressed are either already based on irregular data structures and workloads, or are trending towards it (such as sparse machine learning problems). For these irregular workload computations to be successful, we require efficient load-balancing schemes targetting specialized hardware such as the GPUs or Symphony.
 5 | 
 6 | [^2]: [DARPA Selects Teams to Unleash Power of Specialized, Reconfigurable Computing Hardware](https://www.darpa.mil/news-events/2018-07-24a)
 7 | 
 8 | ## A small (and important) piece of a larger puzzle.
 9 | The predominant approach today to addressing irregularity is to build application-dependent solutions. These are not portable between applications. This is a shame because We believe the underlying techniques that are currently used to address irregularity have the potential to be expressed in a generic, portable, powerful way. We build a generic open-source library for load balancing that will expose high-performance, intuitive load-balancing strategies to any irregular-parallel application.
10 | 
11 | ## Load-balancing problem, and a silver lining.
12 | Today's GPUs follow a Single Instruction Multiple Data (SIMD) model, where different work components (for example a node in a graph) are mapped to a single thread. Each thread runs a copy of the program and threads run in parallel (this is a simple explanation, there are other work units in NVIDIA's GPUs such as warps, cooperative thread arrays, streaming multiprocessors etc.). Let's take a graph problem as an example to understand load imbalance. One key operation in graph problems is traversal, given a set of vertices, a traversal operation visits all the neighboring vertices of the input. If we naïvely map each input vertex to a GPU thread it can result in a massive imbalance of work. As some threads within the GPU will get a lot more work than others, causing inefficient utilization of hardware resources. In our example; this could happen for a social-network graph where one input vertex may have millions of connections while other input vertices in the same traversal pass may only have tens of neighbors.
13 | 
14 | The silver lining here is that there are more intelligent workload mappings that address this problem the load imbalance problem for various types of graphs and other irregular workloads. We extend these previously tightly-coupled scheduling algorithms to an abstraction.
15 | 


--------------------------------------------------------------------------------
/docs/loadbalancing_api.md:
--------------------------------------------------------------------------------
  1 | [README](/README.md) > **Load-Balancing API**
  2 | 
  3 | # Load-Balancing API
  4 | 
  5 | SpMV problem-specific kernel parameters.
  6 | 
  7 | ```cpp
  8 | template <std::size_t threads_per_block,
  9 |           typename index_t,
 10 |           typename offset_t,
 11 |           typename type_t>
 12 | __global__ void __launch_bounds__(threads_per_block, 2)
 13 |           spmv(std::size_t rows,
 14 |                std::size_t cols,
 15 |                std::size_t nnz,
 16 |                offset_t* offsets,
 17 |                index_t* indices,
 18 |                const type_t* values,
 19 |                const type_t* x,
 20 |                type_t* y) {
 21 | ```
 22 | 
 23 | ### (1) Define and configure load-balancing schedule.
 24 | Allocates any temporary memory required for load-balancing, as well as constructs a schedule per processors partition (defined using cooperative groups).
 25 | ```cpp
 26 |   using setup_t = schedule::setup<schedule::algroithms_t::tile_mapped,
 27 |                                   threads_per_block, 32, index_t, offset_t>;
 28 | 
 29 |   /// Allocate temporary storage for the schedule.
 30 |   using storage_t = typename setup_t::storage_t;
 31 |   __shared__ storage_t temporary_storage;
 32 | 
 33 |   /// Construct the schedule.
 34 |   setup_t config(temporary_storage, offsets, rows, nnz);
 35 |   auto p = config.partition();
 36 | ```
 37 | 
 38 | ### (2) Load-balanced ranged loops. (also see; [C++ ranges](https://en.cppreference.com/w/cpp/header/ranges))
 39 | In this example, we define two iteration spaces; virtual and real. Virtual spaces allow us to balance atoms and tiles onto the processor ids and link directly to the real iteration space, which returns the exact atom or tile being processed. The code below loops over all balanced number of atoms fetches the tile corresponding to the atom being processed and allows user to define their computation.
 40 | ```cpp
 41 |   for (auto virtual_atom : config.atom_accessor(p)) {
 42 |     auto virtual_tile = config.tile_accessor(virtual_atom, p);
 43 | 
 44 |     if (!(config.is_valid_accessor(virtual_tile, p)))
 45 |       continue;
 46 | 
 47 |     auto row = config.tile_id(virtual_tile, p);
 48 | 
 49 |     auto nz_idx = config.atom_id(virtual_atom, row, virtual_tile, p);
 50 | ```
 51 | 
 52 | ### (3) User-defined computation.
 53 | Once the user has access to the atom, tile, and the processor id, they implement the desired computation on the given tuple. In this example, we use a simple `atomicAdd` to perform SpMV (can be improved).
 54 | ```cpp
 55 |     atomicAdd(&(y[row]), values[nz_idx] * x[indices[nz_idx]]);
 56 |   }
 57 | }
 58 | ```
 59 | 
 60 | [**work_oriented.cuh**](https://github.com/neoblizz/loops/blob/main/include/loops/algorithms/spmv/work_oriented.cuh) (another example)
 61 | 
 62 | ```cpp
 63 | #include <loops/schedule.hxx>
 64 | 
 65 | template <std::size_t threads_per_block,
 66 |           typename index_t,
 67 |           typename offset_t,
 68 |           typename type_t>
 69 | __global__ void __launch_bounds__(threads_per_block, 2)
 70 |     __work_oriented(std::size_t rows,
 71 |                     std::size_t cols,
 72 |                     std::size_t nnz,
 73 |                     offset_t* offsets,
 74 |                     index_t* indices,
 75 |                     const type_t* values,
 76 |                     const type_t* x,
 77 |                     type_t* y) {
 78 |   using setup_t =
 79 |       schedule::setup<schedule::algorithms_t::work_oriented, threads_per_block,
 80 |                       1, index_t, offset_t, std::size_t, std::size_t>;
 81 | 
 82 |   setup_t config(offsets, rows, nnz);
 83 |   auto map = config.init();
 84 | 
 85 |   /// Accumulate the complete tiles.
 86 |   type_t sum = 0;
 87 |   for (auto row : config.tiles(map)) {
 88 |     for (auto nz : config.atoms(row, map)) {
 89 |       sum += values[nz] * x[indices[nz]];
 90 |     }
 91 |     y[row] = sum;
 92 |     sum = 0;
 93 |   }
 94 | 
 95 |   /// Process remaining tiles.
 96 |   for (auto row : config.remainder_tiles(map)) {
 97 |     for (auto nz : config.remainder_atoms(map)) {
 98 |       sum += values[nz] * x[indices[nz]];
 99 |     }
100 |     /// Accumulate the remainder.
101 |     if (sum != 0)
102 |       atomicAdd(&(y[row]), sum);
103 |   }
104 | }
105 | ```


--------------------------------------------------------------------------------
/examples/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | # begin /* Add examples' subdirectories */
2 | add_subdirectory(range)
3 | add_subdirectory(saxpy)
4 | add_subdirectory(spmv)
5 | add_subdirectory(spmm)
6 | # end /* Add examples' subdirectories */


--------------------------------------------------------------------------------
/examples/range/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | # begin /* Set the application name. */
 2 | set(APPLICATION_NAME range)
 3 | # end /* Set the application name. */
 4 | 
 5 | # begin /* Add CUDA executables */
 6 | add_executable(${APPLICATION_NAME})
 7 | 
 8 | set(SOURCE_LIST 
 9 |     ${APPLICATION_NAME}.cu
10 | )
11 | 
12 | target_sources(${APPLICATION_NAME} PRIVATE ${SOURCE_LIST})
13 | target_link_libraries(${APPLICATION_NAME} 
14 |     PRIVATE loops
15 |     # PRIVATE nvToolsExt
16 | )
17 | 
18 | set_target_properties(${APPLICATION_NAME} 
19 |     PROPERTIES 
20 |         CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
21 | ) # XXX: Find a better way to inherit loops properties.
22 | 
23 | message(STATUS "Example Added: ${APPLICATION_NAME}")
24 | # end /* Add CUDA executables */


--------------------------------------------------------------------------------
/examples/range/range.cu:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <vector>
 3 | 
 4 | #include <loops/range.hxx>
 5 | 
 6 | using namespace loops;
 7 | 
 8 | int main() {
 9 |   for (auto i : range(1, 5))
10 |     std::cout << i << std::endl;
11 | 
12 |   for (auto u : range(0u))
13 |     if (u == 3u)
14 |       break;
15 |     else
16 |       std::cout << u << std::endl;
17 | 
18 |   for (auto c : range('a', 'd'))
19 |     std::cout << c << std::endl;
20 | 
21 |   for (auto u : range(20u, 29u).step(2u))
22 |     std::cout << u << std::endl;
23 | 
24 |   for (auto i : range(100).step(-3))
25 |     if (i < 90)
26 |       break;
27 |     else
28 |       std::cout << i << std::endl;
29 | 
30 |   std::vector<int> x{1, 2, 3};
31 |   for (auto i : indices(x))
32 |     std::cout << i << std::endl;
33 | 
34 |   for (auto i : indices({"foo", "bar"}))
35 |     std::cout << i << std::endl;
36 | 
37 |   for (auto i : indices("foobar").step(2))
38 |     std::cout << i << std::endl;
39 | }


--------------------------------------------------------------------------------
/examples/saxpy/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | # begin /* Set the application name. */
 2 | set(APPLICATION_NAME saxpy)
 3 | # end /* Set the application name. */
 4 | 
 5 | # begin /* Add CUDA executables */
 6 | add_executable(${APPLICATION_NAME})
 7 | 
 8 | set(SOURCE_LIST 
 9 |     ${APPLICATION_NAME}.cu
10 | )
11 | 
12 | target_sources(${APPLICATION_NAME} PRIVATE ${SOURCE_LIST})
13 | target_link_libraries(${APPLICATION_NAME} 
14 |     PRIVATE loops
15 |     # PRIVATE nvToolsExt
16 | )
17 | 
18 | set_target_properties(${APPLICATION_NAME} 
19 |     PROPERTIES 
20 |         CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
21 | ) # XXX: Find a better way to inherit loops properties.
22 | 
23 | message(STATUS "Example Added: ${APPLICATION_NAME}")
24 | # end /* Add CUDA executables */


--------------------------------------------------------------------------------
/examples/saxpy/saxpy.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file saxpy.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Simple CUDA example of saxpy (Single-Precision A·X Plus Y) using
 5 |  * ranged loops.
 6 |  * @version 0.1
 7 |  * @date 2022-02-02
 8 |  *
 9 |  * @copyright Copyright (c) 2022
10 |  *
11 |  */
12 | #include <loops/container/vector.hxx>
13 | #include <loops/stride_ranges.hxx>
14 | #include <loops/util/generate.hxx>
15 | 
16 | #include <thrust/copy.h>
17 | 
18 | template <typename type_t>
19 | __global__ void saxpy(int n, type_t a, type_t* x, type_t* y) {
20 |   /// Equivalent to:
21 |   /// i = blockIdx.x * blockDim.x + threadIdx.x; (init)
22 |   /// i < n; (boundary condition)
23 |   /// i += gridDim.x * blockDim.x. (step)
24 |   for (auto i : loops::grid_stride_range(0, n)) {
25 |     y[i] += a * x[i];
26 |   }
27 | }
28 | 
29 | int main() {
30 |   using type_t = float;
31 |   constexpr int N = 1 << 20;
32 |   constexpr type_t alpha = 2.0f;
33 | 
34 |   // Create thrust device vectors.
35 |   loops::vector_t<type_t> x(N);
36 |   loops::vector_t<type_t> y(N);
37 | 
38 |   // Generate random numbers between [0, 1].
39 |   loops::generate::random::uniform_distribution(x.begin(), x.end(), 0, 1);
40 |   loops::generate::random::uniform_distribution(y.begin(), y.end(), 0, 1);
41 | 
42 |   // Launch kernel with a given configuration.
43 |   constexpr std::size_t threads_per_block = 256;
44 |   std::size_t blocks_per_grid = (N + threads_per_block - 1) / threads_per_block;
45 |   saxpy<<<blocks_per_grid, threads_per_block>>>(N, alpha, x.data().get(),
46 |                                                 y.data().get());
47 | 
48 |   // Print the x and y vectors.
49 |   std::cout << "x = ";
50 |   thrust::copy(x.begin(), (x.size() < 10) ? x.end() : x.begin() + 10,
51 |                std::ostream_iterator<type_t>(std::cout, " "));
52 |   std::cout << std::endl;
53 | 
54 |   std::cout << "y = ";
55 |   thrust::copy(y.begin(), (y.size() < 10) ? y.end() : y.begin() + 10,
56 |                std::ostream_iterator<type_t>(std::cout, " "));
57 |   std::cout << std::endl;
58 | }


--------------------------------------------------------------------------------
/examples/spmm/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | # begin /* Add application */
 2 | set(SOURCES
 3 |   thread_mapped.cu
 4 | )
 5 | 
 6 | foreach(SOURCE IN LISTS SOURCES)
 7 |   get_filename_component(TEST_NAME ${SOURCE} NAME_WLE)
 8 |   add_executable(loops.spmm.${TEST_NAME} ${SOURCE})
 9 |   target_link_libraries(loops.spmm.${TEST_NAME} PRIVATE loops)
10 |   set_target_properties(loops.spmm.${TEST_NAME} 
11 |     PROPERTIES 
12 |       CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
13 |   )
14 |   message(STATUS "Example Added: loops.spmm.${TEST_NAME}")
15 | endforeach()
16 | # end /* Add application */


--------------------------------------------------------------------------------
/examples/spmm/helpers.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file helpers.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Header file for SpMM.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/util/generate.hxx>
15 | #include <loops/container/formats.hxx>
16 | #include <loops/container/vector.hxx>
17 | #include <loops/container/market.hxx>
18 | #include <loops/util/filepath.hxx>
19 | #include <loops/util/equal.hxx>
20 | #include <loops/util/device.hxx>
21 | #include <loops/util/timer.hxx>
22 | #include <loops/memory.hxx>
23 | #include <cxxopts.hpp>
24 | 
25 | #include <algorithm>
26 | #include <iostream>
27 | 
28 | struct parameters_t {
29 |   std::string filename;
30 |   bool validate;
31 |   bool verbose;
32 |   cxxopts::Options options;
33 | 
34 |   /**
35 |    * @brief Construct a new parameters object and parse command line arguments.
36 |    *
37 |    * @param argc Number of command line arguments.
38 |    * @param argv Command line arguments.
39 |    */
40 |   parameters_t(int argc, char** argv)
41 |       : options(argv[0], "Sparse Matrix-Matrix Multiplication") {
42 |     // Add command line options
43 |     options.add_options()("h,help", "Print help")                   // help
44 |         ("m,market", "Matrix file", cxxopts::value<std::string>())  // mtx
45 |         ("validate", "CPU validation")                              // validate
46 |         ("v,verbose", "Verbose output");                            // verbose
47 | 
48 |     // Parse command line arguments
49 |     auto result = options.parse(argc, argv);
50 | 
51 |     if (result.count("help") || (result.count("market") == 0)) {
52 |       std::cout << options.help({""}) << std::endl;
53 |       std::exit(0);
54 |     }
55 | 
56 |     if (result.count("market") == 1) {
57 |       filename = result["market"].as<std::string>();
58 |       if (loops::is_market(filename)) {
59 |       } else {
60 |         std::cout << options.help({""}) << std::endl;
61 |         std::exit(0);
62 |       }
63 |     } else {
64 |       std::cout << options.help({""}) << std::endl;
65 |       std::exit(0);
66 |     }
67 | 
68 |     if (result.count("validate") == 1) {
69 |       validate = true;
70 |     } else {
71 |       validate = false;
72 |     }
73 | 
74 |     if (result.count("verbose") == 1) {
75 |       verbose = true;
76 |     } else {
77 |       verbose = false;
78 |     }
79 |   }
80 | };
81 | 


--------------------------------------------------------------------------------
/examples/spmm/thread_mapped.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file thread_mapped.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Matrix Multiplication example.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "helpers.hxx"
13 | #include <loops/algorithms/spmm/thread_mapped.cuh>
14 | 
15 | using namespace loops;
16 | 
17 | int main(int argc, char** argv) {
18 |   using index_t = int;
19 |   using offset_t = int;
20 |   using type_t = float;
21 | 
22 |   // ... I/O parameters, mtx, etc.
23 |   parameters_t parameters(argc, argv);
24 | 
25 |   matrix_market_t<index_t, offset_t, type_t> mtx;
26 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(parameters.filename));
27 | 
28 |   // Input and output matrices.
29 |   std::size_t n = 10;
30 |   matrix_t<type_t> B(csr.cols, n);
31 |   matrix_t<type_t> C(csr.rows, n);
32 | 
33 |   // Generate random numbers between [0, 10].
34 |   generate::random::uniform_distribution(B.m_data.begin(), B.m_data.end(), 1,
35 |                                          10);
36 | 
37 |   // Run the benchmark.
38 |   util::timer_t timer;
39 |   timer.start();
40 |   algorithms::spmm::thread_mapped(csr, B, C);
41 |   timer.stop();
42 | 
43 |   std::cout << "Elapsed (ms):\t" << timer.milliseconds() << std::endl;
44 | }


--------------------------------------------------------------------------------
/examples/spmv/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | # begin /* Add application */
 2 | set(SOURCES
 3 |   original.cu  
 4 |   thread_mapped.cu
 5 |   group_mapped.cu
 6 |   work_oriented.cu
 7 |   merge_path.cu
 8 | )
 9 | 
10 | foreach(SOURCE IN LISTS SOURCES)
11 |   get_filename_component(TEST_NAME ${SOURCE} NAME_WLE)
12 |   string(PREPEND TEST_NAME "loops.spmv.")
13 |   add_executable(${TEST_NAME} ${SOURCE})
14 |   target_link_libraries(${TEST_NAME}
15 |     PRIVATE loops
16 |   )
17 | 
18 |   set_target_properties(${TEST_NAME}
19 |     PROPERTIES
20 |       CXX_STANDARD 17
21 |       CUDA_STANDARD 17
22 |       CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}
23 |   )
24 |   
25 |   message(STATUS "Example Added: ${TEST_NAME}")
26 | endforeach()
27 | # end /* Add application */


--------------------------------------------------------------------------------
/examples/spmv/group_mapped.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file group_mapped.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication example.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "helpers.hxx"
13 | #include <loops/algorithms/spmv/group_mapped.cuh>
14 | 
15 | using namespace loops;
16 | 
17 | int main(int argc, char** argv) {
18 |   using index_t = int;
19 |   using offset_t = int;
20 |   using type_t = float;
21 | 
22 |   // ... I/O parameters, mtx, etc.
23 |   parameters_t parameters(argc, argv);
24 |   matrix_market_t<index_t, offset_t, type_t> mtx;
25 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(parameters.filename));
26 | 
27 |   // Input and output vectors.
28 |   vector_t<type_t> x(csr.cols);
29 |   vector_t<type_t> y(csr.rows);
30 | 
31 |   // Generate random numbers between [0, 1].
32 |   generate::random::uniform_distribution(x.begin(), x.end(), 1, 10);
33 | 
34 |   // Run the benchmark.
35 |   util::timer_t timer;
36 |   timer.start();
37 |   algorithms::spmv::group_mapped(csr, x, y);
38 |   timer.stop();
39 | 
40 |   std::cout << "group_mapped," << mtx.dataset << "," << csr.rows << ","
41 |             << csr.cols << "," << csr.nnzs << "," << timer.milliseconds()
42 |             << std::endl;
43 | 
44 |   // Validation.
45 |   if (parameters.validate)
46 |     cpu::validate(parameters, csr, x, y);
47 | }


--------------------------------------------------------------------------------
/examples/spmv/helpers.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file helpers.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Header file for SpMV.
  5 |  * @version 0.1
  6 |  * @date 2022-02-03
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include <loops/util/generate.hxx>
 15 | #include <loops/container/formats.hxx>
 16 | #include <loops/container/vector.hxx>
 17 | #include <loops/container/market.hxx>
 18 | #include <loops/util/filepath.hxx>
 19 | #include <loops/util/equal.hxx>
 20 | #include <loops/util/device.hxx>
 21 | #include <loops/util/timer.hxx>
 22 | #include <loops/memory.hxx>
 23 | #include <cxxopts.hpp>
 24 | 
 25 | #include <algorithm>
 26 | #include <iostream>
 27 | 
 28 | struct parameters_t {
 29 |   std::string filename;
 30 |   bool validate;
 31 |   bool verbose;
 32 |   cxxopts::Options options;
 33 | 
 34 |   /**
 35 |    * @brief Construct a new parameters object and parse command line arguments.
 36 |    *
 37 |    * @param argc Number of command line arguments.
 38 |    * @param argv Command line arguments.
 39 |    */
 40 |   parameters_t(int argc, char** argv)
 41 |       : options(argv[0], "Sparse Matrix-Vector Multiplication") {
 42 |     // Add command line options
 43 |     options.add_options()("h,help", "Print help")                   // help
 44 |         ("m,market", "Matrix file", cxxopts::value<std::string>())  // mtx
 45 |         ("validate", "CPU validation")                              // validate
 46 |         ("v,verbose", "Verbose output");                            // verbose
 47 | 
 48 |     // Parse command line arguments
 49 |     auto result = options.parse(argc, argv);
 50 | 
 51 |     if (result.count("help") || (result.count("market") == 0)) {
 52 |       std::cout << options.help({""}) << std::endl;
 53 |       std::exit(0);
 54 |     }
 55 | 
 56 |     if (result.count("market") == 1) {
 57 |       filename = result["market"].as<std::string>();
 58 |       if (loops::is_market(filename)) {
 59 |       } else {
 60 |         std::cout << options.help({""}) << std::endl;
 61 |         std::exit(0);
 62 |       }
 63 |     } else {
 64 |       std::cout << options.help({""}) << std::endl;
 65 |       std::exit(0);
 66 |     }
 67 | 
 68 |     if (result.count("validate") == 1) {
 69 |       validate = true;
 70 |     } else {
 71 |       validate = false;
 72 |     }
 73 | 
 74 |     if (result.count("verbose") == 1) {
 75 |       verbose = true;
 76 |     } else {
 77 |       verbose = false;
 78 |     }
 79 |   }
 80 | };
 81 | 
 82 | namespace cpu {
 83 | 
 84 | using namespace loops;
 85 | using namespace loops::memory;
 86 | 
 87 | /**
 88 |  * @brief CPU SpMV implementation.
 89 |  *
 90 |  * @tparam index_t
 91 |  * @tparam offset_t
 92 |  * @tparam type_t
 93 |  * @param csr device CSR matrix.
 94 |  * @param x device input vector.
 95 |  * @return loops::vector_t<type_t, memory_space_t::host> device output vector.
 96 |  */
 97 | template <typename index_t, typename offset_t, typename type_t>
 98 | loops::vector_t<type_t, memory_space_t::host> reference(
 99 |     loops::csr_t<index_t, offset_t, type_t, memory_space_t::device>& csr,
100 |     loops::vector_t<type_t, memory_space_t::device>& x) {
101 |   // Copy data to CPU.
102 |   loops::csr_t<index_t, offset_t, type_t, memory_space_t::host> csr_h(csr);
103 |   loops::vector_t<type_t, memory_space_t::host> x_h(x);
104 |   loops::vector_t<type_t, memory_space_t::host> y_h(x_h.size());
105 | 
106 |   for (auto row = 0; row < csr_h.rows; ++row) {
107 |     type_t sum = 0;
108 |     for (auto nz = csr_h.offsets[row]; nz < csr_h.offsets[row + 1]; ++nz) {
109 |       sum += csr_h.values[nz] * x_h[csr_h.indices[nz]];
110 |     }
111 |     y_h[row] = sum;
112 |   }
113 | 
114 |   return y_h;
115 | }
116 | 
117 | /**
118 |  * @brief Validation for SpMV.
119 |  *
120 |  * @tparam index_t Column indices type.
121 |  * @tparam offset_t Row offset type.
122 |  * @tparam type_t Value type.
123 |  * @param parameters Parameters.
124 |  * @param csr CSR matrix.
125 |  * @param x Input vector.
126 |  * @param y Output vector.
127 |  */
128 | template <typename index_t, typename offset_t, typename type_t>
129 | void validate(parameters_t& parameters,
130 |               csr_t<index_t, offset_t, type_t>& csr,
131 |               vector_t<type_t>& x,
132 |               vector_t<type_t>& y) {
133 |   // Validation code, can be safely ignored.
134 |   auto h_y = reference(csr, x);
135 | 
136 |   std::size_t errors = util::equal(
137 |       y.data().get(), h_y.data(), csr.rows,
138 |       [](const type_t a, const type_t b) { return std::abs(a - b) > 1e-2; },
139 |       parameters.verbose);
140 | 
141 |   std::cout << "Matrix:\t\t" << extract_filename(parameters.filename)
142 |             << std::endl;
143 |   std::cout << "Dimensions:\t" << csr.rows << " x " << csr.cols << " ("
144 |             << csr.nnzs << ")" << std::endl;
145 |   std::cout << "Errors:\t\t" << errors << std::endl;
146 | }
147 | 
148 | }  // namespace cpu


--------------------------------------------------------------------------------
/examples/spmv/merge_path.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file merge_path.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication example.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "helpers.hxx"
13 | #include <loops/algorithms/spmv/merge_path_flat.cuh>
14 | 
15 | using namespace loops;
16 | 
17 | int main(int argc, char** argv) {
18 |   using index_t = int;
19 |   using offset_t = int;
20 |   using type_t = float;
21 | 
22 |   // ... I/O parameters, mtx, etc.
23 |   parameters_t parameters(argc, argv);
24 | 
25 |   matrix_market_t<index_t, offset_t, type_t> mtx;
26 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(parameters.filename));
27 | 
28 |   // Input and output vectors.
29 |   vector_t<type_t> x(csr.cols);
30 |   vector_t<type_t> y(csr.rows);
31 | 
32 |   // Generate random numbers between [0, 1].
33 |   generate::random::uniform_distribution(x.begin(), x.end(), 1, 10);
34 |   // thrust::fill(x.begin(), x.end(), 2);
35 | 
36 |   // Run the benchmark.
37 |   float elapsed = 0.0f;
38 |   int num_runs = 1;
39 |   for (int i = 0; i < num_runs; i++) {
40 |     auto timer = algorithms::spmv::merge_path_flat(csr, x, y);
41 |     elapsed += timer.milliseconds();
42 |   }
43 | 
44 |   std::cout << "merge_path_flat," << mtx.dataset << "," << csr.rows << ","
45 |             << csr.cols << "," << csr.nnzs << "," << elapsed << std::endl;
46 | 
47 |   // Validation.
48 |   if (parameters.validate)
49 |     cpu::validate(parameters, csr, x, y);
50 | }


--------------------------------------------------------------------------------
/examples/spmv/original.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file original.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication example.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "helpers.hxx"
13 | #include <loops/algorithms/spmv/original.cuh>
14 | 
15 | using namespace loops;
16 | 
17 | int main(int argc, char** argv) {
18 |   using index_t = int;
19 |   using offset_t = int;
20 |   using type_t = float;
21 | 
22 |   // ... I/O parameters, mtx, etc.
23 |   parameters_t parameters(argc, argv);
24 | 
25 |   matrix_market_t<index_t, offset_t, type_t> mtx;
26 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(parameters.filename));
27 | 
28 |   // Input and output vectors.
29 |   vector_t<type_t> x(csr.cols);
30 |   vector_t<type_t> y(csr.rows);
31 | 
32 |   // Generate random numbers between [0, 1].
33 |   generate::random::uniform_distribution(x.begin(), x.end(), 1, 10);
34 | 
35 |   // Run the benchmark.
36 |   util::timer_t timer;
37 |   timer.start();
38 |   algorithms::spmv::original(csr, x, y);
39 |   timer.stop();
40 | 
41 |   std::cout << "original," << mtx.dataset << "," << csr.rows << "," << csr.cols
42 |             << "," << csr.nnzs << "," << timer.milliseconds() << std::endl;
43 | 
44 |   // Validation.
45 |   if (parameters.validate)
46 |     cpu::validate(parameters, csr, x, y);
47 | }


--------------------------------------------------------------------------------
/examples/spmv/thread_mapped.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file thread_mapped.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication example.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "helpers.hxx"
13 | #include <loops/algorithms/spmv/thread_mapped.cuh>
14 | 
15 | using namespace loops;
16 | 
17 | int main(int argc, char** argv) {
18 |   using index_t = int;
19 |   using offset_t = int;
20 |   using type_t = float;
21 | 
22 |   // ... I/O parameters, mtx, etc.
23 |   parameters_t parameters(argc, argv);
24 | 
25 |   matrix_market_t<index_t, offset_t, type_t> mtx;
26 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(parameters.filename));
27 | 
28 |   // Input and output vectors.
29 |   vector_t<type_t> x(csr.cols);
30 |   vector_t<type_t> y(csr.rows);
31 | 
32 |   // Generate random numbers between [0, 1].
33 |   generate::random::uniform_distribution(x.begin(), x.end(), 1, 10);
34 | 
35 |   // Run the benchmark.
36 |   util::timer_t timer;
37 |   timer.start();
38 |   algorithms::spmv::thread_mapped(csr, x, y);
39 |   timer.stop();
40 | 
41 |   std::cout << "thread_mapped," << mtx.dataset << "," << csr.rows << ","
42 |             << csr.cols << "," << csr.nnzs << "," << timer.milliseconds()
43 |             << std::endl;
44 | 
45 |   // Validation.
46 |   if (parameters.validate)
47 |     cpu::validate(parameters, csr, x, y);
48 | }


--------------------------------------------------------------------------------
/examples/spmv/work_oriented.cu:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file work_oriented.cu
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication example.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include "helpers.hxx"
13 | #include <loops/algorithms/spmv/work_oriented.cuh>
14 | 
15 | using namespace loops;
16 | 
17 | int main(int argc, char** argv) {
18 |   using index_t = int;
19 |   using offset_t = int;
20 |   using type_t = float;
21 | 
22 |   // ... I/O parameters, mtx, etc.
23 |   parameters_t parameters(argc, argv);
24 | 
25 |   matrix_market_t<index_t, offset_t, type_t> mtx;
26 |   csr_t<index_t, offset_t, type_t> csr(mtx.load(parameters.filename));
27 | 
28 |   // Input and output vectors.
29 |   vector_t<type_t> x(csr.cols);
30 |   vector_t<type_t> y(csr.rows);
31 | 
32 |   // Generate random numbers between [0, 1].
33 |   generate::random::uniform_distribution(x.begin(), x.end(), 1, 10);
34 | 
35 |   // Run the benchmark.
36 |   util::timer_t timer;
37 |   timer.start();
38 |   algorithms::spmv::work_oriented(csr, x, y);
39 |   timer.stop();
40 | 
41 |   std::cout << "work_oriented," << mtx.dataset << "," << csr.rows << ","
42 |             << csr.cols << "," << csr.nnzs << "," << timer.milliseconds()
43 |             << std::endl;
44 | 
45 |   // Validation.
46 |   if (parameters.validate)
47 |     cpu::validate(parameters, csr, x, y);
48 | }


--------------------------------------------------------------------------------
/include/loops/algorithms/spmm/thread_mapped.cuh:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file thread_mapped.cuh
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Matrix Multiplication kernels.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/stride_ranges.hxx>
15 | #include <loops/schedule.hxx>
16 | #include <loops/container/formats.hxx>
17 | #include <loops/container/vector.hxx>
18 | #include <loops/container/matrix.cuh>
19 | #include <loops/util/launch.hxx>
20 | #include <loops/util/device.hxx>
21 | #include <loops/memory.hxx>
22 | #include <iostream>
23 | 
24 | namespace loops {
25 | namespace algorithms {
26 | namespace spmm {
27 | 
28 | template <typename setup_t,
29 |           typename index_t,
30 |           typename offset_t,
31 |           typename type_t>
32 | __global__ void __thread_mapped(setup_t config,
33 |                                 const std::size_t a_rows,
34 |                                 const std::size_t a_cols,
35 |                                 const std::size_t a_nnz,
36 |                                 const offset_t* offsets,
37 |                                 const index_t* indices,
38 |                                 const type_t* values,
39 |                                 const matrix_t<type_t> B,
40 |                                 matrix_t<type_t> C) {
41 |   for (auto row : config.tiles()) {
42 |     for (auto col :
43 |          custom_stride_range(std::size_t(0), B.cols, std::size_t(1))) {
44 |       type_t sum = 0;
45 |       for (auto nz : config.atoms(row)) {
46 |         sum += values[nz] * B(indices[nz], col);
47 |       }
48 | 
49 |       // Output
50 |       C(row, col) = sum;
51 |     }
52 |   }
53 | }
54 | 
55 | /**
56 |  * @brief Sparse-Matrix Matrix Multiplication API.
57 |  *
58 |  * @tparam index_t Type of column indices.
59 |  * @tparam offset_t Type of row offsets.
60 |  * @tparam type_t Type of values.
61 |  * @param csr CSR matrix (GPU).
62 |  * @param n Number of columns in the B-matrix.
63 |  * @param B Input matrix B (GPU).
64 |  * @param C Output matrix C (GPU).
65 |  * @param stream CUDA stream.
66 |  */
67 | template <typename index_t, typename offset_t, typename type_t>
68 | void thread_mapped(csr_t<index_t, offset_t, type_t>& csr,
69 |                    matrix_t<type_t>& B,
70 |                    matrix_t<type_t>& C,
71 |                    cudaStream_t stream = 0) {
72 |   // Create a schedule.
73 |   constexpr std::size_t block_size = 128;
74 | 
75 |   /// Set-up kernel launch parameters and run the kernel.
76 | 
77 |   // Create a schedule.
78 |   using setup_t = schedule::setup<schedule::algorithms_t::thread_mapped, 1, 1,
79 |                                   index_t, offset_t>;
80 |   setup_t config(csr.offsets.data().get(), csr.rows, csr.nnzs);
81 | 
82 |   std::size_t grid_size = (csr.rows + block_size - 1) / block_size;
83 |   launch::non_cooperative(
84 |       stream, __thread_mapped<setup_t, index_t, offset_t, type_t>, grid_size,
85 |       block_size, config, csr.rows, csr.cols, csr.nnzs,
86 |       csr.offsets.data().get(), csr.indices.data().get(),
87 |       csr.values.data().get(), B, C);
88 | 
89 |   cudaStreamSynchronize(stream);
90 | }
91 | 
92 | }  // namespace spmm
93 | }  // namespace algorithms
94 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/algorithms/spmv/group_mapped.cuh:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file group_mapped.cuh
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Sparse Matrix-Vector Multiplication kernels.
  5 |  * @version 0.1
  6 |  * @date 2022-02-03
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include <loops/schedule.hxx>
 15 | #include <loops/container/formats.hxx>
 16 | #include <loops/container/vector.hxx>
 17 | #include <loops/util/launch.hxx>
 18 | #include <loops/util/device.hxx>
 19 | #include <loops/memory.hxx>
 20 | #include <iostream>
 21 | 
 22 | namespace loops {
 23 | namespace algorithms {
 24 | namespace spmv {
 25 | 
 26 | template <std::size_t threads_per_block,
 27 |           typename index_t,
 28 |           typename offset_t,
 29 |           typename type_t>
 30 | __global__ void __launch_bounds__(threads_per_block, 2)
 31 |     __group_mapped(std::size_t rows,
 32 |                    std::size_t cols,
 33 |                    std::size_t nnz,
 34 |                    offset_t* offsets,
 35 |                    index_t* indices,
 36 |                    const type_t* values,
 37 |                    const type_t* x,
 38 |                    type_t* y) {
 39 |   using setup_t = schedule::block_mapped<threads_per_block, index_t, offset_t>;
 40 | 
 41 |   /// Allocate temporary storage for the schedule.
 42 |   using storage_t = typename setup_t::storage_t;
 43 |   __shared__ storage_t temporary_storage;
 44 | 
 45 |   /// Construct the schedule.
 46 |   setup_t config(temporary_storage, offsets, rows, nnz);
 47 |   auto p = config.partition();
 48 | 
 49 |   for (auto virtual_atom : config.atom_accessor(p)) {
 50 |     auto virtual_tile = config.tile_accessor(virtual_atom, p);
 51 | 
 52 |     if (!(config.is_valid_accessor(virtual_tile, p)))
 53 |       continue;
 54 | 
 55 |     auto row = config.tile_id(virtual_tile, p);
 56 | 
 57 |     auto nz_idx = config.atom_id(virtual_atom, row, virtual_tile, p);
 58 |     atomicAdd(&(y[row]), values[nz_idx] * x[indices[nz_idx]]);
 59 |   }
 60 | }
 61 | 
 62 | /**
 63 |  * @brief Sparse-Matrix Vector Multiplication API.
 64 |  *
 65 |  * @tparam index_t Type of column indices.
 66 |  * @tparam offset_t Type of row offsets.
 67 |  * @tparam type_t Type of values.
 68 |  * @param csr CSR matrix (GPU).
 69 |  * @param x Input vector x (GPU).
 70 |  * @param y Output vector y (GPU).
 71 |  * @param stream CUDA stream.
 72 |  */
 73 | template <typename index_t, typename offset_t, typename type_t>
 74 | void group_mapped(csr_t<index_t, offset_t, type_t>& csr,
 75 |                   vector_t<type_t>& x,
 76 |                   vector_t<type_t>& y,
 77 |                   cudaStream_t stream = 0) {
 78 |   // Create a schedule.
 79 |   constexpr std::size_t block_size = 128;
 80 | 
 81 |   /// Set-up kernel launch parameters and run the kernel.
 82 | 
 83 |   /// Traditional kernel launch, this is nice for tile mapped scheduling, which
 84 |   /// will allow blocks to be scheduled in and out as needed. And will rely on
 85 |   /// NVIDIA's hardware schedule to schedule the blocks efficiently.
 86 |   std::size_t grid_size = (csr.rows + block_size - 1) / block_size;
 87 |   launch::non_cooperative(
 88 |       stream, __group_mapped<block_size, index_t, offset_t, type_t>, grid_size,
 89 |       block_size, csr.rows, csr.cols, csr.nnzs, csr.offsets.data().get(),
 90 |       csr.indices.data().get(), csr.values.data().get(), x.data().get(),
 91 |       y.data().get());
 92 | 
 93 |   /// Cooperative kernel launch; requires a fixed number of blocks per grid to
 94 |   /// be launched, this number can be determined by using CUDA's occupancy API
 95 |   /// to figure out how many blocks will run concurrently at all times per SM.
 96 |   /// And then we simply loop over the entire work within the kernel.
 97 |   // launch::cooperative(stream, __group_mapped<block_size, index_t, offset_t,
 98 |   // type_t>,
 99 |   //                     grid_size, block_size, csr.rows, csr.cols, csr.nnzs,
100 |   //                     csr.offsets.data().get(), csr.indices.data().get(),
101 |   //                     csr.values.data().get(), x.data().get(),
102 |   //                     y.data().get());
103 | 
104 |   cudaStreamSynchronize(stream);
105 | }
106 | 
107 | }  // namespace spmv
108 | }  // namespace algorithms
109 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/algorithms/spmv/merge_path_flat.cuh:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file work_oriented.cuh
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Sparse Matrix-Vector Multiplication example.
  5 |  * @version 0.1
  6 |  * @date 2022-02-03
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include <loops/schedule.hxx>
 15 | #include <loops/util/math.hxx>
 16 | #include <loops/container/formats.hxx>
 17 | #include <loops/container/vector.hxx>
 18 | #include <loops/util/launch.hxx>
 19 | #include <loops/util/device.hxx>
 20 | #include <loops/memory.hxx>
 21 | #include <iostream>
 22 | 
 23 | #include <cub/block/block_scan.cuh>
 24 | 
 25 | namespace loops {
 26 | namespace algorithms {
 27 | namespace spmv {
 28 | 
 29 | /**
 30 |  * @brief Flat Merge-Path SpMV kernel.
 31 |  *
 32 |  * @tparam threads_per_block Number of threads per block.
 33 |  * @tparam items_per_thread Number of items per thread to process.
 34 |  * @tparam index_t Type of column indices.
 35 |  * @tparam offset_t Type of row offsets.
 36 |  * @tparam type_t Type of values.
 37 |  */
 38 | template <std::size_t threads_per_block,
 39 |           std::size_t items_per_thread,
 40 |           typename meta_t,
 41 |           typename index_t,
 42 |           typename offset_t,
 43 |           typename type_t>
 44 | __global__ void __launch_bounds__(int(threads_per_block))
 45 |     __merge_path_flat(meta_t meta,
 46 |                       std::size_t rows,
 47 |                       std::size_t cols,
 48 |                       std::size_t nnz,
 49 |                       offset_t* offsets,
 50 |                       index_t* indices,
 51 |                       const type_t* values,
 52 |                       const type_t* x,
 53 |                       type_t* y) {
 54 |   using setup_t = schedule::setup<schedule::algorithms_t::merge_path_flat,
 55 |                                   threads_per_block, items_per_thread, index_t,
 56 |                                   offset_t, std::size_t, std::size_t>;
 57 | 
 58 |   /// Allocate temporary storage for the schedule.
 59 |   using storage_t = typename setup_t::storage_t;
 60 |   __shared__ storage_t temporary_storage;
 61 | 
 62 |   /// Construct the schedule.
 63 |   setup_t config(meta, temporary_storage, offsets, rows, nnz);
 64 |   auto map = config.init();
 65 | 
 66 |   if (!config.is_valid_accessor(map))
 67 |     return;
 68 | 
 69 | /// Flat Merge-Path loop from 0..items_per_thread.
 70 | #pragma unroll
 71 |   for (auto item : config.virtual_idx()) {
 72 |     auto nz = config.atom_idx(item, map);
 73 |     auto row = config.tile_idx(map);
 74 |     type_t nonzero = values[nz] * x[indices[nz]];
 75 |     if (config.atoms_counting_it[map.y] <
 76 |         temporary_storage.tile_end_offset[map.x]) {
 77 |       atomicAdd(&(y[row]), nonzero);
 78 |       map.y++;
 79 |     } else {
 80 |       map.x++;
 81 |     }
 82 |   }
 83 | }
 84 | 
 85 | /**
 86 |  * @brief Sparse-Matrix Vector Multiplication API.
 87 |  *
 88 |  * @tparam index_t Type of column indices.
 89 |  * @tparam offset_t Type of row offsets.
 90 |  * @tparam type_t Type of values.
 91 |  * @param csr CSR matrix (GPU).
 92 |  * @param x Input vector x (GPU).
 93 |  * @param y Output vector y (GPU).
 94 |  * @param stream CUDA stream.
 95 |  */
 96 | template <typename index_t, typename offset_t, typename type_t>
 97 | util::timer_t merge_path_flat(csr_t<index_t, offset_t, type_t>& csr,
 98 |                               vector_t<type_t>& x,
 99 |                               vector_t<type_t>& y,
100 |                               cudaStream_t stream = 0) {
101 |   // Create a schedule.
102 |   constexpr std::size_t block_size = sizeof(type_t) > 4 ? 64 : 128;
103 |   constexpr std::size_t items_per_thread = sizeof(type_t) > 4 ? 3 : 5;
104 | 
105 |   using preprocessor_t =
106 |       schedule::merge_path::preprocess_t<block_size, items_per_thread, index_t,
107 |                                          offset_t, std::size_t, std::size_t>;
108 | 
109 |   /// Light-weight preprocess that does not modify the data, just creates an
110 |   /// array with pre-calculated per block coordinates to reduce the work in the
111 |   /// actual kernel.
112 |   preprocessor_t meta(csr.offsets.data().get(), csr.rows, csr.nnzs, stream);
113 | 
114 |   /// Set-up kernel launch parameters and run the kernel.
115 |   int max_dim_x;
116 |   int num_merge_tiles =
117 |       math::ceil_div(csr.rows + csr.nnzs, block_size * items_per_thread);
118 |   int device_ordinal = device::get();
119 |   cudaDeviceGetAttribute(&max_dim_x, cudaDevAttrMaxGridDimX, device_ordinal);
120 | 
121 |   util::timer_t timer;
122 |   timer.start();
123 | 
124 |   // Launch main kernel that uses merge-path schedule.
125 |   int within_bounds = min(num_merge_tiles, max_dim_x);
126 |   int overflow = math::ceil_div(num_merge_tiles, max_dim_x);
127 |   dim3 grid_size(within_bounds, overflow, 1);
128 |   launch::non_cooperative(
129 |       stream,
130 |       __merge_path_flat<block_size, items_per_thread, preprocessor_t, index_t,
131 |                         offset_t, type_t>,
132 |       grid_size, block_size, meta, csr.rows, csr.cols, csr.nnzs,
133 |       csr.offsets.data().get(), csr.indices.data().get(),
134 |       csr.values.data().get(), x.data().get(), y.data().get());
135 |   cudaStreamSynchronize(stream);
136 |   timer.stop();
137 | 
138 |   return timer;
139 | }
140 | 
141 | }  // namespace spmv
142 | }  // namespace algorithms
143 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/algorithms/spmv/original.cuh:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file original.cuh
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication kernels.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/schedule.hxx>
15 | #include <loops/container/formats.hxx>
16 | #include <loops/container/vector.hxx>
17 | #include <loops/util/launch.hxx>
18 | #include <loops/util/device.hxx>
19 | #include <loops/memory.hxx>
20 | #include <iostream>
21 | 
22 | namespace loops {
23 | namespace algorithms {
24 | namespace spmv {
25 | 
26 | template <typename index_t, typename offset_t, typename type_t>
27 | __global__ void __original(const std::size_t rows,
28 |                            const std::size_t cols,
29 |                            const std::size_t nnz,
30 |                            const offset_t* offsets,
31 |                            const index_t* indices,
32 |                            const type_t* values,
33 |                            const type_t* x,
34 |                            type_t* y) {
35 |   for (auto row = blockIdx.x * blockDim.x + threadIdx.x;
36 |        row < rows;                    // boundary condition
37 |        row += gridDim.x * blockDim.x  // step
38 |   ) {
39 |     type_t sum = 0;
40 |     for (offset_t nz = offsets[row]; nz < offsets[row + 1]; ++nz)
41 |       sum += values[nz] * x[indices[nz]];
42 | 
43 |     // Output
44 |     y[row] = sum;
45 |   }
46 | }
47 | 
48 | /**
49 |  * @brief Sparse-Matrix Vector Multiplication API.
50 |  *
51 |  * @tparam index_t Type of column indices.
52 |  * @tparam offset_t Type of row offsets.
53 |  * @tparam type_t Type of values.
54 |  * @param csr CSR matrix (GPU).
55 |  * @param x Input vector x (GPU).
56 |  * @param y Output vector y (GPU).
57 |  * @param stream CUDA stream.
58 |  */
59 | template <typename index_t, typename offset_t, typename type_t>
60 | void original(csr_t<index_t, offset_t, type_t>& csr,
61 |               vector_t<type_t>& x,
62 |               vector_t<type_t>& y,
63 |               cudaStream_t stream = 0) {
64 |   // Create a schedule.
65 |   constexpr std::size_t block_size = 128;
66 | 
67 |   /// Set-up kernel launch parameters and run the kernel.
68 |   std::size_t grid_size = (csr.rows + block_size - 1) / block_size;
69 |   launch::non_cooperative(stream, __original<index_t, offset_t, type_t>,
70 |                           grid_size, block_size, csr.rows, csr.cols, csr.nnzs,
71 |                           csr.offsets.data().get(), csr.indices.data().get(),
72 |                           csr.values.data().get(), x.data().get(),
73 |                           y.data().get());
74 | 
75 |   cudaStreamSynchronize(stream);
76 | }
77 | 
78 | }  // namespace spmv
79 | }  // namespace algorithms
80 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/algorithms/spmv/thread_mapped.cuh:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file thread_mapped.cuh
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Sparse Matrix-Vector Multiplication kernels.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/schedule.hxx>
15 | #include <loops/container/formats.hxx>
16 | #include <loops/container/vector.hxx>
17 | #include <loops/util/launch.hxx>
18 | #include <loops/util/device.hxx>
19 | #include <loops/memory.hxx>
20 | #include <iostream>
21 | 
22 | namespace loops {
23 | namespace algorithms {
24 | namespace spmv {
25 | 
26 | template <typename setup_t,
27 |           typename index_t,
28 |           typename offset_t,
29 |           typename type_t>
30 | __global__ void __thread_mapped(setup_t config,
31 |                                 const std::size_t rows,
32 |                                 const std::size_t cols,
33 |                                 const std::size_t nnz,
34 |                                 const offset_t* offsets,
35 |                                 const index_t* indices,
36 |                                 const type_t* values,
37 |                                 const type_t* x,
38 |                                 type_t* y) {
39 |   /// Equivalent to:
40 |   /// row = blockIdx.x * blockDim.x + threadIdx.x; (init)
41 |   /// row < rows; (boundary condition)
42 |   /// row += gridDim.x * blockDim.x. (step)
43 |   for (auto row : config.tiles()) {
44 |     type_t sum = 0;
45 | 
46 |     /// Equivalent to:
47 |     /// for (offset_t nz = offset; nz < end; ++nz)
48 |     for (auto nz : config.atoms(row)) {
49 |       sum += values[nz] * x[indices[nz]];
50 |     }
51 | 
52 |     // Output
53 |     y[row] = sum;
54 |   }
55 | }
56 | 
57 | /**
58 |  * @brief Sparse-Matrix Vector Multiplication API.
59 |  *
60 |  * @tparam index_t Type of column indices.
61 |  * @tparam offset_t Type of row offsets.
62 |  * @tparam type_t Type of values.
63 |  * @param csr CSR matrix (GPU).
64 |  * @param x Input vector x (GPU).
65 |  * @param y Output vector y (GPU).
66 |  * @param stream CUDA stream.
67 |  */
68 | template <typename index_t, typename offset_t, typename type_t>
69 | void thread_mapped(csr_t<index_t, offset_t, type_t>& csr,
70 |                    vector_t<type_t>& x,
71 |                    vector_t<type_t>& y,
72 |                    cudaStream_t stream = 0) {
73 |   // Create a schedule.
74 |   constexpr std::size_t block_size = 128;
75 | 
76 |   /// Set-up kernel launch parameters and run the kernel.
77 | 
78 |   // Create a schedule.
79 |   using setup_t = schedule::setup<schedule::algorithms_t::thread_mapped, 1, 1,
80 |                                   index_t, offset_t>;
81 |   setup_t config(csr.offsets.data().get(), csr.rows, csr.nnzs);
82 | 
83 |   std::size_t grid_size = (csr.rows + block_size - 1) / block_size;
84 |   launch::non_cooperative(
85 |       stream, __thread_mapped<setup_t, index_t, offset_t, type_t>, grid_size,
86 |       block_size, config, csr.rows, csr.cols, csr.nnzs,
87 |       csr.offsets.data().get(), csr.indices.data().get(),
88 |       csr.values.data().get(), x.data().get(), y.data().get());
89 | 
90 |   cudaStreamSynchronize(stream);
91 | }
92 | 
93 | }  // namespace spmv
94 | }  // namespace algorithms
95 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/algorithms/spmv/work_oriented.cuh:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file work_oriented.cuh
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Sparse Matrix-Vector Multiplication example.
  5 |  * @version 0.1
  6 |  * @date 2022-02-03
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include <loops/schedule.hxx>
 15 | #include <loops/container/formats.hxx>
 16 | #include <loops/container/vector.hxx>
 17 | #include <loops/util/launch.hxx>
 18 | #include <loops/util/device.hxx>
 19 | #include <loops/memory.hxx>
 20 | #include <iostream>
 21 | 
 22 | namespace loops {
 23 | namespace algorithms {
 24 | namespace spmv {
 25 | 
 26 | /**
 27 |  * @brief Work oriented SpMV kernel.
 28 |  *
 29 |  * @tparam threads_per_block Number of threads per block.
 30 |  * @tparam index_t Type of column indices.
 31 |  * @tparam offset_t Type of row offsets.
 32 |  * @tparam type_t Type of values.
 33 |  */
 34 | template <std::size_t threads_per_block,
 35 |           typename index_t,
 36 |           typename offset_t,
 37 |           typename type_t>
 38 | __global__ void __launch_bounds__(threads_per_block, 2)
 39 |     __work_oriented(std::size_t rows,
 40 |                     std::size_t cols,
 41 |                     std::size_t nnz,
 42 |                     offset_t* offsets,
 43 |                     index_t* indices,
 44 |                     const type_t* values,
 45 |                     const type_t* x,
 46 |                     type_t* y) {
 47 |   using setup_t =
 48 |       schedule::setup<schedule::algorithms_t::work_oriented, threads_per_block,
 49 |                       1, index_t, offset_t, std::size_t, std::size_t>;
 50 | 
 51 |   setup_t config(offsets, rows, nnz);
 52 |   auto map = config.init();
 53 | 
 54 |   /// Accumulate the complete tiles.
 55 |   type_t sum = 0;
 56 |   for (auto row : config.tiles(map)) {
 57 |     for (auto nz : config.atoms(row, map)) {
 58 |       sum += values[nz] * x[indices[nz]];
 59 |     }
 60 |     y[row] = sum;
 61 |     sum = 0;
 62 |   }
 63 | 
 64 |   // Interesting use of syncthreads to ensure all remaining tiles get processed
 65 |   // at the same time, possibly causing less thread divergence among the threads
 66 |   // in the same warp.
 67 |   __syncthreads();
 68 | 
 69 |   /// Process remaining tiles.
 70 |   for (auto row : config.remainder_tiles(map)) {
 71 |     for (auto nz : config.remainder_atoms(map)) {
 72 |       sum += values[nz] * x[indices[nz]];
 73 |     }
 74 |     /// Accumulate the remainder.
 75 |     if (sum != 0)
 76 |       atomicAdd(&(y[row]), sum);
 77 |   }
 78 | }
 79 | 
 80 | /**
 81 |  * @brief Sparse-Matrix Vector Multiplication API.
 82 |  *
 83 |  * @tparam index_t Type of column indices.
 84 |  * @tparam offset_t Type of row offsets.
 85 |  * @tparam type_t Type of values.
 86 |  * @param csr CSR matrix (GPU).
 87 |  * @param x Input vector x (GPU).
 88 |  * @param y Output vector y (GPU).
 89 |  * @param stream CUDA stream.
 90 |  */
 91 | template <typename index_t, typename offset_t, typename type_t>
 92 | void work_oriented(csr_t<index_t, offset_t, type_t>& csr,
 93 |                    vector_t<type_t>& x,
 94 |                    vector_t<type_t>& y,
 95 |                    cudaStream_t stream = 0) {
 96 |   // Create a schedule.
 97 |   constexpr std::size_t block_size = 128;
 98 | 
 99 |   /// Set-up kernel launch parameters and run the kernel.
100 | 
101 |   /// Launch 2 x (SM Count) number of blocks.
102 |   /// Weirdly enough, a really high number here might cause it to fail.
103 |   loops::device::properties_t props;
104 |   std::size_t grid_size = 2 * props.multi_processor_count();
105 | 
106 |   launch::non_cooperative(
107 |       stream, __work_oriented<block_size, index_t, offset_t, type_t>, grid_size,
108 |       block_size, csr.rows, csr.cols, csr.nnzs, csr.offsets.data().get(),
109 |       csr.indices.data().get(), csr.values.data().get(), x.data().get(),
110 |       y.data().get());
111 | 
112 |   cudaStreamSynchronize(stream);
113 | }
114 | 
115 | }  // namespace spmv
116 | }  // namespace algorithms
117 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/container.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file container.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Includes all the container headers.
 5 |  * @version 0.1
 6 |  * @date 2022-07-25
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include <loops/container/formats.hxx>
13 | #include <loops/container/market.hxx>
14 | #include <loops/container/vector.hxx>
15 | #include <loops/container/matrix.cuh>


--------------------------------------------------------------------------------
/include/loops/container/coo.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file coo.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Interface for Coordinate format.
  5 |  * @version 0.1
  6 |  * @date 2022-07-21
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include <loops/container/formats.hxx>
 15 | #include <loops/container/detail/convert.hxx>
 16 | #include <loops/container/vector.hxx>
 17 | #include <loops/memory.hxx>
 18 | 
 19 | #include <thrust/sort.h>
 20 | #include <thrust/tuple.h>
 21 | #include <thrust/unique.h>
 22 | #include <thrust/execution_policy.h>
 23 | #include <thrust/iterator/zip_iterator.h>
 24 | 
 25 | namespace loops {
 26 | 
 27 | using namespace memory;
 28 | 
 29 | /**
 30 |  * @brief Coordinate (COO) format.
 31 |  *
 32 |  * @tparam index_t
 33 |  * @tparam value_t
 34 |  */
 35 | template <typename index_t,
 36 |           typename value_t,
 37 |           memory_space_t space = memory_space_t::device>
 38 | struct coo_t {
 39 |   std::size_t rows;
 40 |   std::size_t cols;
 41 |   std::size_t nnzs;
 42 | 
 43 |   vector_t<index_t, space> row_indices;  /// I
 44 |   vector_t<index_t, space> col_indices;  /// J
 45 |   vector_t<value_t, space> values;       /// V
 46 | 
 47 |   /**
 48 |    * @brief Construct a new coo object with everything initialized to zero.
 49 |    *
 50 |    */
 51 |   coo_t() : rows(0), cols(0), nnzs(0), row_indices(), col_indices(), values() {}
 52 | 
 53 |   /**
 54 |    * @brief Construct a new coo object with the given dimensions.
 55 |    *
 56 |    * @param r Number of rows.
 57 |    * @param c Number of columns.
 58 |    * @param nnz Number of non-zero elements.
 59 |    */
 60 |   coo_t(std::size_t r, std::size_t c, std::size_t nnz)
 61 |       : rows(r),
 62 |         cols(c),
 63 |         nnzs(nnz),
 64 |         row_indices(nnz),
 65 |         col_indices(nnz),
 66 |         values(nnz) {}
 67 | 
 68 |   /**
 69 |    * @brief Construct a new coo from another coo object on host/device.
 70 |    *
 71 |    * @param rhs coo_t<index_t, value_t, rhs_space>
 72 |    */
 73 |   template <auto rhs_space>
 74 |   coo_t(const coo_t<index_t, value_t, rhs_space>& rhs)
 75 |       : rows(rhs.rows),
 76 |         cols(rhs.cols),
 77 |         nnzs(rhs.nnzs),
 78 |         row_indices(rhs.row_indices),
 79 |         col_indices(rhs.col_indices),
 80 |         values(rhs.values) {}
 81 | 
 82 |   /**
 83 |    * @brief Construct a new coo object from compressed sparse format (CSR).
 84 |    *
 85 |    * @param csr csr_t<index_t, offset_t, value_t, rhs_space>
 86 |    */
 87 |   template <auto rhs_space, typename offset_t>
 88 |   coo_t(const csr_t<index_t, offset_t, value_t, rhs_space>& csr)
 89 |       : rows(csr.rows),
 90 |         cols(csr.cols),
 91 |         nnzs(csr.nnzs),
 92 |         row_indices(csr.nnzs),
 93 |         col_indices(csr.col_indices),
 94 |         values(csr.values) {
 95 |     /// TODO: Do not need this copy for all cases.
 96 |     vector_t<offset_t, space> _row_offsets = csr.offsets;
 97 |     detail::offsets_to_indices(_row_offsets, row_indices);
 98 |   }
 99 | 
100 |   /**
101 |    * @brief Sorts the coordinate matrix by row indices.
102 |    *
103 |    */
104 |   void sort_by_row() {
105 |     auto begin = thrust::make_zip_iterator(
106 |         thrust::make_tuple(row_indices.begin(), col_indices.begin()));
107 |     auto end = thrust::make_zip_iterator(
108 |         thrust::make_tuple(row_indices.end(), col_indices.end()));
109 |     sort(begin, end);
110 |   }
111 | 
112 |   /**
113 |    * @brief Sorts the coordinate matrix by column indices.
114 |    *
115 |    */
116 |   void sort_by_column() {
117 |     auto begin = thrust::make_zip_iterator(
118 |         thrust::make_tuple(col_indices.begin(), row_indices.begin()));
119 |     auto end = thrust::make_zip_iterator(
120 |         thrust::make_tuple(col_indices.end(), row_indices.end()));
121 |     sort(begin, end);
122 |   }
123 | 
124 |   /**
125 |    * @brief Sorts, removes I,J pairs.
126 |    *
127 |    */
128 |   void remove_duplicates() {
129 |     // Sort by row indices.
130 |     this->sort_by_row();
131 |     auto begin = thrust::make_zip_iterator(
132 |         thrust::make_tuple(row_indices.begin(), col_indices.begin()));
133 |     auto end = thrust::make_zip_iterator(
134 |         thrust::make_tuple(row_indices.end(), col_indices.end()));
135 | 
136 |     // Remove duplicates.
137 |     auto new_it = thrust::unique_by_key(begin, end, values.begin());
138 |     auto first_it = thrust::get<1>(new_it);
139 |     nnzs = thrust::distance(values.begin(), first_it);
140 | 
141 |     // Resize vectors to new size.
142 |     row_indices.resize(nnzs);
143 |     col_indices.resize(nnzs);
144 |     values.resize(nnzs);
145 |   }
146 | 
147 |  private:
148 |   /**
149 |    * @brief Sorting helper, uses zip iterator as begin and end.
150 |    *
151 |    * @par Example Zip iterator:
152 |    *  auto begin = thrust::make_zip_iterator(
153 |    *    thrust::make_tuple(col_indices.begin(), row_indices.begin()));
154 |    *  auto end = thrust::make_zip_iterator(
155 |    *    thrust::make_tuple(col_indices.end(), row_indices.end()));
156 |    *
157 |    * @tparam begin_it_t Begin iterator type.
158 |    * @tparam end_it_t End iterator type.
159 |    * @param begin Begin iterator (zip iterator of row and col indices).
160 |    * @param end End iterator (zip iterator of row and col indices).
161 |    */
162 |   template <typename begin_it_t, typename end_it_t>
163 |   void sort(begin_it_t& begin, end_it_t& end) {
164 |     thrust::sort_by_key(begin, end, values.begin());
165 |   }
166 | };  // struct coo_t
167 | 
168 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/coordinate.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file coordinate.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Simple coordinate with x and y.
 5 |  * @version 0.1
 6 |  * @date 2022-11-12
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | namespace loops {
14 | template <typename index_t>
15 | struct coordinate_t {
16 |   index_t x;
17 |   index_t y;
18 | };
19 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/csc.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file csc.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Interface for Compressed Sparse-Column format.
 5 |  * @version 0.1
 6 |  * @date 2022-07-21
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/container/vector.hxx>
15 | #include <loops/memory.hxx>
16 | 
17 | namespace loops {
18 | 
19 | using namespace memory;
20 | 
21 | /**
22 |  * @brief Compressed Sparse Column (CSC) format.
23 |  *
24 |  * @tparam index_t
25 |  * @tparam offset_t
26 |  * @tparam value_t
27 |  */
28 | template <typename index_t,
29 |           typename offset_t,
30 |           typename value_t,
31 |           memory_space_t space = memory_space_t::device>
32 | struct csc_t {
33 |   std::size_t rows;
34 |   std::size_t cols;
35 |   std::size_t nnzs;
36 | 
37 |   vector_t<offset_t, space> offsets;  /// Aj
38 |   vector_t<index_t, space> indices;   /// Ap
39 |   vector_t<value_t, space> values;    /// Ax
40 | 
41 |   /**
42 |    * @brief Construct a new csc object with everything initialized to zero.
43 |    *
44 |    */
45 |   csc_t() : rows(0), cols(0), nnzs(0), offsets(), indices(), values() {}
46 | 
47 |   /**
48 |    * @brief Construct a new csc object with the given dimensions.
49 |    *
50 |    * @param r Number of rows.
51 |    * @param c Number of columns.
52 |    * @param nnz Number of non-zero elements.
53 |    */
54 |   csc_t(std::size_t r, std::size_t c, std::size_t nnz)
55 |       : rows(r),
56 |         cols(c),
57 |         nnzs(nnz),
58 |         offsets(r + 1),
59 |         indices(nnz),
60 |         values(nnz) {}
61 | 
62 |   /**
63 |    * @brief Construct a new csc from another csc object on host/device.
64 |    *
65 |    * @param rhs csc_t<index_t, offset_t, value_t, rhs_space>
66 |    */
67 |   template <auto rhs_space>
68 |   csc_t(const csc_t<index_t, offset_t, value_t, rhs_space>& rhs)
69 |       : rows(rhs.rows),
70 |         cols(rhs.cols),
71 |         nnzs(rhs.nnzs),
72 |         offsets(rhs.offsets),
73 |         indices(rhs.indices),
74 |         values(rhs.values) {}
75 | 
76 |   /**
77 |    * @brief Construct a new csc object from coordinate format (COO).
78 |    * @note This constructor creates a copy of the input COO matrix.
79 |    *
80 |    * @param coo coo_t<index_t, value_t, auto>
81 |    */
82 |   template <auto rhs_space>
83 |   csc_t(const coo_t<index_t, value_t, rhs_space>& coo)
84 |       : rows(coo.rows), cols(coo.cols), nnzs(coo.nnzs), offsets(coo.cols + 1) {
85 |     coo_t<index_t, value_t, space> __(coo);
86 |     __.sort_by_column();
87 |     indices = std::move(__.row_indices);
88 |     values = std::move(__.values);
89 |     detail::indices_to_offsets(__.col_indices, offsets);
90 |   }
91 | 
92 | };  // struct csc_t
93 | 
94 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/csr.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file csr.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Interface for Compressed Sparse-Row format.
 5 |  * @version 0.1
 6 |  * @date 2022-07-21
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/container/vector.hxx>
15 | #include <loops/container/formats.hxx>
16 | #include <loops/container/detail/convert.hxx>
17 | #include <loops/memory.hxx>
18 | 
19 | #include <thrust/transform.h>
20 | 
21 | namespace loops {
22 | 
23 | using namespace memory;
24 | 
25 | /**
26 |  * @brief Compressed Sparse Row (CSR) format.
27 |  *
28 |  * @tparam index_t Type of the nonzero elements indices.
29 |  * @tparam offset_t Type of the row offsets.
30 |  * @tparam value_t Type of the nonzero elements values.
31 |  */
32 | template <typename index_t,
33 |           typename offset_t,
34 |           typename value_t,
35 |           memory_space_t space = memory_space_t::device>
36 | struct csr_t {
37 |   std::size_t rows;
38 |   std::size_t cols;
39 |   std::size_t nnzs;
40 | 
41 |   vector_t<offset_t, space> offsets;  /// Ap
42 |   vector_t<index_t, space> indices;   /// Aj
43 |   vector_t<value_t, space> values;    /// Ax
44 | 
45 |   /**
46 |    * @brief Construct a new csr object with everything initialized to zero.
47 |    *
48 |    */
49 |   csr_t() : rows(0), cols(0), nnzs(0), offsets(), indices(), values() {}
50 | 
51 |   /**
52 |    * @brief Construct a new csr object with the given dimensions.
53 |    *
54 |    * @param r Number of rows.
55 |    * @param c Number of columns.
56 |    * @param nnz Number of non-zero elements.
57 |    */
58 |   csr_t(std::size_t r, std::size_t c, std::size_t nnz)
59 |       : rows(r),
60 |         cols(c),
61 |         nnzs(nnz),
62 |         offsets(r + 1),
63 |         indices(nnz),
64 |         values(nnz) {}
65 | 
66 |   /**
67 |    * @brief Construct a new csr from another csr object on host/device.
68 |    *
69 |    * @param rhs csr_t<index_t, offset_t, value_t, rhs_space>
70 |    */
71 |   template <auto rhs_space>
72 |   csr_t(const csr_t<index_t, offset_t, value_t, rhs_space>& rhs)
73 |       : rows(rhs.rows),
74 |         cols(rhs.cols),
75 |         nnzs(rhs.nnzs),
76 |         offsets(rhs.offsets),
77 |         indices(rhs.indices),
78 |         values(rhs.values) {}
79 | 
80 |   /**
81 |    * @brief Construct a new csr object from coordinate format (COO).
82 |    * @note This constructor creates a copy of the input COO matrix.
83 |    *
84 |    * @param coo coo_t<index_t, value_t, auto>
85 |    */
86 |   template <auto rhs_space>
87 |   csr_t(const coo_t<index_t, value_t, rhs_space>& coo)
88 |       : rows(coo.rows), cols(coo.cols), nnzs(coo.nnzs), offsets(coo.rows + 1) {
89 |     coo_t<index_t, value_t, space> __(coo);
90 |     __.sort_by_row();
91 |     indices = std::move(__.col_indices);
92 |     values = std::move(__.values);
93 |     detail::indices_to_offsets(__.row_indices, offsets);
94 |   }
95 | };  // struct csr_t
96 | 
97 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/detail/convert.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file convert.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Conversion functions for containers.
 5 |  * @version 0.1
 6 |  * @date 2022-07-19
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <thrust/binary_search.h>
15 | #include <thrust/scan.h>
16 | #include <thrust/fill.h>
17 | #include <thrust/scatter.h>
18 | #include <thrust/execution_policy.h>
19 | #include <thrust/iterator/counting_iterator.h>
20 | #include <thrust/iterator/zip_iterator.h>
21 | #include <thrust/tuple.h>
22 | #include <loops/memory.hxx>
23 | 
24 | namespace loops {
25 | namespace detail {
26 | using namespace memory;
27 | 
28 | /**
29 |  * @brief Convert offsets to indices.
30 |  *
31 |  * @tparam index_v_t The type of vector indices.
32 |  * @tparam offset_v_t The type of vector offsets.
33 |  * @param offsets The offsets.
34 |  * @param indices The indices.
35 |  */
36 | template <typename index_v_t, typename offset_v_t>
37 | void offsets_to_indices(const offset_v_t& offsets, index_v_t& indices) {
38 |   using offset_t = typename offset_v_t::value_type;
39 |   using index_t = typename index_v_t::value_type;
40 | 
41 |   // Convert compressed offsets into uncompressed indices.
42 |   thrust::fill(indices.begin(), indices.end(), offset_t(0));
43 | 
44 |   thrust::scatter_if(
45 |       thrust::counting_iterator<offset_t>(0),  // begin iterator
46 |       thrust::counting_iterator<offset_t>(offsets.size() - 1),  // end iterator
47 |       offsets.begin(),  // where to scatter
48 |       thrust::make_transform_iterator(
49 |           thrust::make_zip_iterator(
50 |               thrust::make_tuple(offsets.begin(), offsets.begin() + 1)),
51 |           [=] __host__ __device__(const thrust::tuple<offset_t, offset_t>& t) {
52 |             thrust::not_equal_to<offset_t> comp;
53 |             return comp(thrust::get<0>(t), thrust::get<1>(t));
54 |           }),
55 |       indices.begin());
56 | 
57 |   thrust::inclusive_scan(indices.begin(), indices.end(), indices.begin(),
58 |                          thrust::maximum<offset_t>());
59 | }
60 | 
61 | /**
62 |  * @brief Converts "indices"-based array to "offsets"-based array.
63 |  *
64 |  * @tparam index_v_t The type of vector indices.
65 |  * @tparam offset_v_t The type of vector offsets.
66 |  * @param indices The indices.
67 |  * @param offsets The offsets.
68 |  */
69 | template <typename index_v_t, typename offset_v_t>
70 | void indices_to_offsets(const index_v_t& indices, offset_v_t& offsets) {
71 |   using offset_t = typename offset_v_t::value_type;
72 |   using index_t = typename index_v_t::value_type;
73 | 
74 |   // Convert uncompressed indices into compressed offsets.
75 |   thrust::lower_bound(
76 |       indices.begin(), indices.end(), thrust::counting_iterator<offset_t>(0),
77 |       thrust::counting_iterator<offset_t>(offsets.size()), offsets.begin());
78 | }
79 | 
80 | }  // namespace detail
81 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/detail/mmio.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file mmio.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Matrix-Market file format header file, see mmio.cpp for implementation
  5 |  * details.
  6 |  * @version 0.1
  7 |  * @date 2020-10-12
  8 |  *
  9 |  * @copyright Copyright (c) 2020
 10 |  *
 11 |  */
 12 | 
 13 | #pragma once
 14 | 
 15 | #if defined(__cplusplus)
 16 | extern "C" {
 17 | #endif
 18 | 
 19 | /*
 20 |  *   Matrix Market I/O library for ANSI C
 21 |  *
 22 |  *   See http://math.nist.gov/MatrixMarket for details.
 23 |  *
 24 |  *
 25 |  */
 26 | 
 27 | #ifndef MM_IO_H
 28 | #define MM_IO_H
 29 | 
 30 | #define MM_MAX_LINE_LENGTH 1025
 31 | #define MatrixMarketBanner "%%MatrixMarket"
 32 | #define MM_MAX_TOKEN_LENGTH 64
 33 | 
 34 | typedef char MM_typecode[4];
 35 | 
 36 | char* mm_typecode_to_str(MM_typecode matcode);
 37 | 
 38 | int mm_read_banner(FILE* f, MM_typecode* matcode);
 39 | int mm_read_mtx_crd_size(FILE* f,
 40 |                          std::size_t* M,
 41 |                          std::size_t* N,
 42 |                          std::size_t* nz);
 43 | 
 44 | int mm_read_mtx_array_size(FILE* f, std::size_t* M, std::size_t* N);
 45 | 
 46 | int mm_write_banner(FILE* f, MM_typecode matcode);
 47 | int mm_write_mtx_crd_size(FILE* f,
 48 |                           std::size_t M,
 49 |                           std::size_t N,
 50 |                           std::size_t nz);
 51 | int mm_write_mtx_array_size(FILE* f, std::size_t M, std::size_t N);
 52 | 
 53 | /********************* MM_typecode query fucntions ***************************/
 54 | 
 55 | #define mm_is_matrix(typecode) ((typecode)[0] == 'M')
 56 | 
 57 | #define mm_is_sparse(typecode) ((typecode)[1] == 'C')
 58 | #define mm_is_coordinate(typecode) ((typecode)[1] == 'C')
 59 | #define mm_is_dense(typecode) ((typecode)[1] == 'A')
 60 | #define mm_is_array(typecode) ((typecode)[1] == 'A')
 61 | 
 62 | #define mm_is_complex(typecode) ((typecode)[2] == 'C')
 63 | #define mm_is_real(typecode) ((typecode)[2] == 'R')
 64 | #define mm_is_pattern(typecode) ((typecode)[2] == 'P')
 65 | #define mm_is_integer(typecode) ((typecode)[2] == 'I')
 66 | 
 67 | #define mm_is_symmetric(typecode) ((typecode)[3] == 'S')
 68 | #define mm_is_general(typecode) ((typecode)[3] == 'G')
 69 | #define mm_is_skew(typecode) ((typecode)[3] == 'K')
 70 | #define mm_is_hermitian(typecode) ((typecode)[3] == 'H')
 71 | 
 72 | int mm_is_valid(MM_typecode matcode); /* too complex for a macro */
 73 | 
 74 | /********************* MM_typecode modify fucntions ***************************/
 75 | 
 76 | #define mm_set_matrix(typecode) ((*typecode)[0] = 'M')
 77 | #define mm_set_coordinate(typecode) ((*typecode)[1] = 'C')
 78 | #define mm_set_array(typecode) ((*typecode)[1] = 'A')
 79 | #define mm_set_dense(typecode) mm_set_array(typecode)
 80 | #define mm_set_sparse(typecode) mm_set_coordinate(typecode)
 81 | 
 82 | #define mm_set_complex(typecode) ((*typecode)[2] = 'C')
 83 | #define mm_set_real(typecode) ((*typecode)[2] = 'R')
 84 | #define mm_set_pattern(typecode) ((*typecode)[2] = 'P')
 85 | #define mm_set_integer(typecode) ((*typecode)[2] = 'I')
 86 | 
 87 | #define mm_set_symmetric(typecode) ((*typecode)[3] = 'S')
 88 | #define mm_set_general(typecode) ((*typecode)[3] = 'G')
 89 | #define mm_set_skew(typecode) ((*typecode)[3] = 'K')
 90 | #define mm_set_hermitian(typecode) ((*typecode)[3] = 'H')
 91 | 
 92 | #define mm_clear_typecode(typecode) \
 93 |   ((*typecode)[0] = (*typecode)[1] = (*typecode)[2] = ' ', (*typecode)[3] = 'G')
 94 | 
 95 | #define mm_initialize_typecode(typecode) mm_clear_typecode(typecode)
 96 | 
 97 | /********************* Matrix Market error codes ***************************/
 98 | 
 99 | #define MM_COULD_NOT_READ_FILE 11
100 | #define MM_PREMATURE_EOF 12
101 | #define MM_NOT_MTX 13
102 | #define MM_NO_HEADER 14
103 | #define MM_UNSUPPORTED_TYPE 15
104 | #define MM_LINE_TOO_LONG 16
105 | #define MM_COULD_NOT_WRITE_FILE 17
106 | 
107 | /**
108 |  * @brief Matrix Market internal definitions.
109 |  *
110 |  *                  MM_matrix_typecode: 4-character sequence
111 |  *
112 |  * |                 | ojbect   | sparse/dense | data type | storage scheme |
113 |  * |-----------------|----------|--------------|-----------|----------------|
114 |  * | string position | [0]      | [1]          | [2]       | [3]            |
115 |  * | Matrix typecode | M(atrix) | C(oord)      | R(eal)    | G(eneral)      |
116 |  * |                 |          | A(rray)      | C(omplex) | H(ermitian)    |
117 |  * |                 |          |              | P(attern) | S(ymmetric)    |
118 |  * |                 |          |              | I(nteger) | K(kew)         |
119 |  *
120 |  */
121 | #define MM_MTX_STR "matrix"
122 | #define MM_ARRAY_STR "array"
123 | #define MM_DENSE_STR "array"
124 | #define MM_COORDINATE_STR "coordinate"
125 | #define MM_SPARSE_STR "coordinate"
126 | #define MM_COMPLEX_STR "complex"
127 | #define MM_REAL_STR "real"
128 | #define MM_INT_STR "integer"
129 | #define MM_GENERAL_STR "general"
130 | #define MM_SYMM_STR "symmetric"
131 | #define MM_HERM_STR "hermitian"
132 | #define MM_SKEW_STR "skew-symmetric"
133 | #define MM_PATTERN_STR "pattern"
134 | 
135 | /*  high level routines */
136 | 
137 | int mm_write_mtx_crd(char fname[],
138 |                      std::size_t M,
139 |                      std::size_t N,
140 |                      std::size_t nz,
141 |                      std::size_t I[],
142 |                      std::size_t J[],
143 |                      double val[],
144 |                      MM_typecode matcode);
145 | 
146 | int mm_read_mtx_crd_data(FILE* f,
147 |                          std::size_t M,
148 |                          std::size_t N,
149 |                          std::size_t nz,
150 |                          std::size_t I[],
151 |                          std::size_t J[],
152 |                          double val[],
153 |                          MM_typecode matcode);
154 | 
155 | int mm_read_mtx_crd_entry(FILE* f,
156 |                           std::size_t* I,
157 |                           std::size_t* J,
158 |                           double* real,
159 |                           double* img,
160 |                           MM_typecode matcode);
161 | 
162 | int mm_read_unsymmetric_sparse(const char* fname,
163 |                                std::size_t* M_,
164 |                                std::size_t* N_,
165 |                                std::size_t* nz_,
166 |                                double** val_,
167 |                                std::size_t** I_,
168 |                                std::size_t** J_);
169 | 
170 | #endif
171 | 
172 | #if defined(__cplusplus)
173 | }
174 | #endif


--------------------------------------------------------------------------------
/include/loops/container/formats.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file formats.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief
 5 |  * @version 0.1
 6 |  * @date 2020-10-05
 7 |  *
 8 |  * @copyright Copyright (c) 2020
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/memory.hxx>
15 | 
16 | namespace loops {
17 | 
18 | using namespace memory;
19 | 
20 | // Forward decleration
21 | template <typename index_t, typename value_t, memory_space_t space>
22 | struct coo_t;
23 | 
24 | template <typename index_t,
25 |           typename offset_t,
26 |           typename value_t,
27 |           memory_space_t space>
28 | struct csr_t;
29 | 
30 | template <typename index_t,
31 |           typename offset_t,
32 |           typename value_t,
33 |           memory_space_t space>
34 | struct csc_t;
35 | 
36 | }  // namespace loops
37 | 
38 | #include <loops/container/coo.hxx>
39 | #include <loops/container/csc.hxx>
40 | #include <loops/container/csr.hxx>


--------------------------------------------------------------------------------
/include/loops/container/market.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file matrix_market.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Matrix Market format reader.
  5 |  * @see http://math.nist.gov/MatrixMarket/
  6 |  * @version 0.1
  7 |  * @date 2020-10-09
  8 |  *
  9 |  * @copyright Copyright (c) 2020
 10 |  *
 11 |  */
 12 | 
 13 | #pragma once
 14 | 
 15 | #include <string>
 16 | #include <limits>
 17 | 
 18 | #include <loops/container/detail/mmio.hxx>
 19 | 
 20 | #include <loops/util/filepath.hxx>
 21 | #include <loops/container/formats.hxx>
 22 | #include <loops/memory.hxx>
 23 | #include <loops/error.hxx>
 24 | 
 25 | namespace loops {
 26 | 
 27 | using namespace memory;
 28 | 
 29 | /**
 30 |  * @brief Matrix Market format supports two kind of formats, a sparse coordinate
 31 |  * format and a dense array format.
 32 |  *
 33 |  */
 34 | enum matrix_market_format_t { coordinate, array };
 35 | 
 36 | /**
 37 |  * @brief Data type defines the type of data presented in the file, things like,
 38 |  * are they real numbers, complex (real and imaginary), pattern (do not have
 39 |  * weights/nonzero-values), etc.
 40 |  *
 41 |  */
 42 | enum matrix_market_data_t { real, complex, pattern, integer };
 43 | 
 44 | /**
 45 |  * @brief Storage scheme defines the storage structure, symmetric matrix for
 46 |  * example will be symmetric over the diagonal. Skew is skew symmetric. Etc.
 47 |  *
 48 |  */
 49 | enum matrix_market_storage_scheme_t { general, hermitian, symmetric, skew };
 50 | 
 51 | /**
 52 |  * @brief Reads a MARKET graph from an input-stream
 53 |  * into a specified sparse format
 54 |  *
 55 |  * Here is an example of the matrix market format
 56 |  * +----------------------------------------------+
 57 |  * |%%MatrixMarket matrix coordinate real general | <--- header line
 58 |  * |%                                             | <--+
 59 |  * |% comments                                    |    |-- 0 or more comments
 60 |  * |%                                             | <--+
 61 |  * |  M N L                                       | <--- rows, columns, entries
 62 |  * |  I1 J1 A(I1, J1)                             | <--+
 63 |  * |  I2 J2 A(I2, J2)                             |    |
 64 |  * |  I3 J3 A(I3, J3)                             |    |-- L lines
 65 |  * |     . . .                                    |    |
 66 |  * |  IL JL A(IL, JL)                             | <--+
 67 |  * +----------------------------------------------+
 68 |  *
 69 |  * Indices are 1-based i.2. A(1,1) is the first element.
 70 |  */
 71 | template <typename index_t, typename offset_t, typename type_t>
 72 | struct matrix_market_t {
 73 |   // typedef FILE* file_t;
 74 |   // typedef MM_typecode matrix_market_code_t;
 75 | 
 76 |   using file_t = FILE*;
 77 |   using matrix_market_code_t = MM_typecode;
 78 | 
 79 |   std::string filename;
 80 |   std::string dataset;
 81 | 
 82 |   // Dataset characteristics
 83 |   matrix_market_code_t code;              // (ALL INFORMATION)
 84 |   matrix_market_format_t format;          // Sparse coordinate or dense array
 85 |   matrix_market_data_t data;              // Data type
 86 |   matrix_market_storage_scheme_t scheme;  // Storage scheme
 87 | 
 88 |   matrix_market_t() {}
 89 |   ~matrix_market_t() {}
 90 | 
 91 |   /**
 92 |    * @brief Loads the given .mtx file into a coordinate format, and returns the
 93 |    * coordinate array. This needs to be further extended to support dense
 94 |    * arrays, those are the only two formats mtx are written in.
 95 |    *
 96 |    * @param _filename input file name (.mtx)
 97 |    * @return coordinate sparse format
 98 |    */
 99 |   auto load(std::string _filename) {
100 |     filename = _filename;
101 |     dataset = extract_dataset(extract_filename(filename));
102 | 
103 |     file_t file;
104 | 
105 |     // Load MTX information
106 |     if ((file = fopen(filename.c_str(), "r")) == NULL) {
107 |       std::cerr << "File could not be opened: " << filename << std::endl;
108 |       exit(1);
109 |     }
110 | 
111 |     if (mm_read_banner(file, &code) != 0) {
112 |       std::cerr << "Could not process Matrix Market banner" << std::endl;
113 |       exit(1);
114 |     }
115 | 
116 |     std::size_t num_rows, num_columns, num_nonzeros;
117 |     if ((mm_read_mtx_crd_size(file, &num_rows, &num_columns, &num_nonzeros)) !=
118 |         0) {
119 |       std::cerr << "Could not read file info (M, N, NNZ)" << std::endl;
120 |       exit(1);
121 |     }
122 | 
123 |     error::throw_if_exception(
124 |         num_rows >= std::numeric_limits<index_t>::max() ||
125 |             num_columns >= std::numeric_limits<index_t>::max(),
126 |         "index_t overflow");
127 |     error::throw_if_exception(
128 |         num_nonzeros >= std::numeric_limits<offset_t>::max(),
129 |         "offset_t overflow");
130 | 
131 |     // mtx are generally written as coordinate format
132 |     coo_t<index_t, type_t, memory_space_t::host> coo(
133 |         (index_t)num_rows, (index_t)num_columns, (offset_t)num_nonzeros);
134 | 
135 |     if (mm_is_coordinate(code))
136 |       format = matrix_market_format_t::coordinate;
137 |     else
138 |       format = matrix_market_format_t::array;
139 | 
140 |     if (mm_is_pattern(code)) {
141 |       data = matrix_market_data_t::pattern;
142 | 
143 |       // pattern matrix defines sparsity pattern, but not values
144 |       for (index_t i = 0; i < num_nonzeros; ++i) {
145 |         std::size_t row_index{0}, col_index{0};
146 |         auto num_assigned = fscanf(file, " %zu %zu \n", &row_index, &col_index);
147 |         error::throw_if_exception(num_assigned != 2,
148 |                                   "Could not read edge from market file");
149 |         error::throw_if_exception(row_index == 0,
150 |                                   "Market file is zero-indexed");
151 |         error::throw_if_exception(col_index == 0,
152 |                                   "Market file is zero-indexed");
153 |         // set and adjust from 1-based to 0-based indexing
154 |         coo.row_indices[i] = (index_t)row_index - 1;
155 |         coo.col_indices[i] = (index_t)col_index - 1;
156 |         coo.values[i] = (type_t)1.0;  // use value 1.0 for all nonzero entries
157 |       }
158 |     } else if (mm_is_real(code) || mm_is_integer(code)) {
159 |       if (mm_is_real(code))
160 |         data = matrix_market_data_t::real;
161 |       else
162 |         data = matrix_market_data_t::integer;
163 | 
164 |       for (index_t i = 0; i < coo.nnzs; ++i) {
165 |         std::size_t row_index{0}, col_index{0};
166 |         double weight{0.0};
167 | 
168 |         auto num_assigned =
169 |             fscanf(file, " %zu %zu %lf \n", &row_index, &col_index, &weight);
170 | 
171 |         error::throw_if_exception(
172 |             num_assigned != 3, "Could not read weighted edge from market file");
173 |         error::throw_if_exception(row_index == 0,
174 |                                   "Market file is zero-indexed");
175 |         error::throw_if_exception(col_index == 0,
176 |                                   "Market file is zero-indexed");
177 | 
178 |         coo.row_indices[i] = (index_t)row_index - 1;
179 |         coo.col_indices[i] = (index_t)col_index - 1;
180 |         coo.values[i] = (type_t)weight;
181 |       }
182 |     } else {
183 |       std::cerr << "Unrecognized matrix market format type" << std::endl;
184 |       exit(1);
185 |     }
186 | 
187 |     if (mm_is_symmetric(code)) {  // duplicate off diagonal entries
188 |       scheme = matrix_market_storage_scheme_t::symmetric;
189 |       index_t off_diagonals = 0;
190 |       for (index_t i = 0; i < coo.nnzs; ++i) {
191 |         if (coo.row_indices[i] != coo.col_indices[i])
192 |           ++off_diagonals;
193 |       }
194 | 
195 |       index_t _nonzeros = 2 * off_diagonals + (coo.nnzs - off_diagonals);
196 | 
197 |       vector_t<index_t, memory_space_t::host> _I(_nonzeros);
198 |       vector_t<index_t, memory_space_t::host> _J(_nonzeros);
199 |       vector_t<type_t, memory_space_t::host> _V(_nonzeros);
200 | 
201 |       index_t ptr = 0;
202 |       for (index_t i = 0; i < coo.nnzs; ++i) {
203 |         if (coo.row_indices[i] != coo.col_indices[i]) {
204 |           _I[ptr] = coo.row_indices[i];
205 |           _J[ptr] = coo.col_indices[i];
206 |           _V[ptr] = coo.values[i];
207 |           ++ptr;
208 |           _J[ptr] = coo.row_indices[i];
209 |           _I[ptr] = coo.col_indices[i];
210 |           _V[ptr] = coo.values[i];
211 |           ++ptr;
212 |         } else {
213 |           _I[ptr] = coo.row_indices[i];
214 |           _J[ptr] = coo.col_indices[i];
215 |           _V[ptr] = coo.values[i];
216 |           ++ptr;
217 |         }
218 |       }
219 |       coo.row_indices = _I;
220 |       coo.col_indices = _J;
221 |       coo.values = _V;
222 |       coo.nnzs = _nonzeros;
223 |     }  // end symmetric case
224 | 
225 |     fclose(file);
226 |     return coo;
227 |   }
228 | };
229 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/matrix.cuh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <loops/container/vector.hxx>
 4 | #include <loops/memory.hxx>
 5 | 
 6 | namespace loops {
 7 | 
 8 | template <typename value_t, memory_space_t space = memory_space_t::device>
 9 | struct matrix_t {
10 |   std::size_t rows;
11 |   std::size_t cols;
12 | 
13 |   vector_t<value_t, space> m_data;
14 |   value_t* m_data_ptr;
15 | 
16 |   matrix_t() : rows(0), cols(0), m_data(), m_data_ptr(nullptr) {}
17 | 
18 |   matrix_t(std::size_t r, std::size_t c)
19 |       : rows(r),
20 |         cols(c),
21 |         m_data(r * c),
22 |         m_data_ptr(memory::raw_pointer_cast(m_data.data())) {}
23 | 
24 |   __host__ __device__ matrix_t(const matrix_t<value_t, space>& other)
25 |       : rows(other.rows), cols(other.cols), m_data_ptr(other.m_data_ptr) {}
26 | 
27 |   __host__ __device__ __forceinline__ value_t operator()(int r, int c) const {
28 |     std::size_t idx = (cols * r) + c;
29 |     return m_data_ptr[idx];
30 |   }
31 | 
32 |   __host__ __device__ __forceinline__ value_t& operator()(int r, int c) {
33 |     std::size_t idx = (cols * r) + c;
34 |     return m_data_ptr[idx];
35 |   }
36 | 
37 |   __host__ __device__ __forceinline__ value_t
38 |   operator[](std::size_t index) const {
39 |     std::size_t r = index / cols;
40 |     std::size_t c = index % cols;
41 |     std::size_t idx = (cols * r) + c;
42 |     return m_data_ptr[idx];
43 |   }
44 | 
45 |   __host__ __device__ __forceinline__ value_t& operator[](std::size_t index) {
46 |     std::size_t r = index / cols;
47 |     std::size_t c = index % cols;
48 |     std::size_t idx = (cols * r) + c;
49 |     return m_data_ptr[idx];
50 |   }
51 | };
52 | 
53 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/container/vector.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file vector.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief support for stl based vectors on gpu. Relies on thrust::host and
 5 |  * device vectors.
 6 |  *
 7 |  * @version 0.1
 8 |  * @date 2022-02-03
 9 |  *
10 |  * @copyright Copyright (c) 2022
11 |  *
12 |  */
13 | 
14 | #pragma once
15 | 
16 | #include <loops/memory.hxx>
17 | 
18 | // includes: thrust
19 | #include <thrust/device_vector.h>
20 | #include <thrust/host_vector.h>
21 | 
22 | namespace loops {
23 | using namespace memory;
24 | 
25 | /**
26 |  * @brief vector container for GPU and CPU.
27 |  *
28 |  * @tparam type_t data type of the vector
29 |  * @tparam space (@see loops::memory_space_t)
30 |  */
31 | template <typename type_t,
32 |           memory_space_t space = memory_space_t::device>
33 | using vector_t =
34 |     std::conditional_t<space == memory_space_t::host,  // condition
35 |                        thrust::host_vector<type_t>,    // host_type
36 |                        thrust::device_vector<type_t>   // device_type
37 |                        >;
38 | 
39 | template <typename type_t>
40 | using host_vector_t = thrust::host_vector<type_t>;
41 | template <typename type_t>
42 | using device_vector_t = thrust::device_vector<type_t>;
43 | 
44 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/error.hxx:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <exception>
 4 | #include <string>
 5 | #include <cuda_runtime_api.h>
 6 | 
 7 | namespace loops {
 8 | 
 9 | /**
10 |  * @namespace error
11 |  * Error utilities for exception handling within device and host code.
12 |  */
13 | namespace error {
14 | 
15 | typedef cudaError_t error_t;
16 | 
17 | /**
18 |  * @brief Exception class for errors in device code.
19 |  *
20 |  */
21 | struct exception_t : std::exception {
22 |   std::string report;
23 | 
24 |   exception_t(error_t _status, std::string _message = "") {
25 |     report = cudaGetErrorString(_status) + std::string("\t: ") + _message;
26 |   }
27 | 
28 |   exception_t(std::string _message = "") { report = _message; }
29 |   virtual const char* what() const noexcept { return report.c_str(); }
30 | };
31 | 
32 | /**
33 |  * @brief Throw an exception if the given error code is not cudaSuccess.
34 |  *
35 |  * @param status error_t error code (equivalent to cudaError_t).
36 |  * @param message custom message to be appended to the error message.
37 |  */
38 | inline void throw_if_exception(error_t status, std::string message = "") {
39 |   if (status != cudaSuccess)
40 |     throw exception_t(status, message);
41 | }
42 | 
43 | inline void throw_if_exception(bool is_exception, std::string message = "") {
44 |   if (is_exception)
45 |     throw exception_t(message);
46 | }
47 | 
48 | }  // namespace error
49 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/memory.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file memory.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <iostream>
15 | #include <memory>
16 | 
17 | #include <thrust/device_ptr.h>
18 | 
19 | namespace loops {
20 | namespace memory {
21 | 
22 | /**
23 |  * @brief memory space; cuda (device) or host.
24 |  * Can be extended to support uvm and multi-gpu.
25 |  *
26 |  * @todo change this enum to support cudaMemoryType
27 |  * (see ref;  std::underlying_type<cudaMemoryType>::type)
28 |  * instead of some random enums, we can rely
29 |  * on cudaMemoryTypeHost/Device/Unregistered/Managed
30 |  * for this.
31 |  *
32 |  */
33 | enum memory_space_t { device, host, managed };
34 | 
35 | /**
36 |  * @brief Wrapper around thrust::raw_pointer_cast() to accept .data() or raw
37 |  * pointer and return a raw pointer. Useful when we would like to return a raw
38 |  * pointer of either a thrust device vector or a host vector. Because thrust
39 |  * device vector's raw pointer is accessed by `.data().get()`, whereas thrust
40 |  * host vector's raw pointer is simply `data()`. So, when calling these
41 |  * functions on `.data()`, it can cast either a host or device vector.
42 |  *
43 |  * @tparam type_t
44 |  * @param pointer
45 |  * @return type_t*
46 |  */
47 | template <typename type_t>
48 | inline type_t* raw_pointer_cast(thrust::device_ptr<type_t> pointer) {
49 |   return thrust::raw_pointer_cast(pointer);
50 | }
51 | 
52 | /**
53 |  * @brief Wrapper around thrust::raw_pointer_cast() to accept .data() or raw
54 |  * pointer and return a raw pointer. Useful when we would like to return a raw
55 |  * pointer of either a thrust device vector or a host vector. Because thrust
56 |  * device vector's raw pointer is accessed by `.data().get()`, whereas thrust
57 |  * host vector's raw pointer is simply `data()`. So, when calling these
58 |  * functions on `.data()`, it can cast either a host or device vector.
59 |  *
60 |  * @tparam type_t
61 |  * @param pointer
62 |  * @return type_t*
63 |  */
64 | template <typename type_t>
65 | __host__ __device__ inline type_t* raw_pointer_cast(type_t* pointer) {
66 |   return thrust::raw_pointer_cast(pointer);
67 | }
68 | 
69 | }  // namespace memory
70 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/range.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file range.hxx
  3 |  * @brief Code is based on Mark Harris' work:
  4 |  * https://github.com/harrism/cpp11-range/blob/master/range.hpp
  5 |  * @version 0.1
  6 |  * @date 2022-02-02
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | #pragma once
 12 | 
 13 | #include <iterator>
 14 | #include <type_traits>
 15 | 
 16 | namespace loops {
 17 | namespace detail {
 18 | 
 19 | template <typename type_t>
 20 | struct range_iter_base : std::iterator<std::input_iterator_tag, type_t> {
 21 |   __host__ __device__ range_iter_base(type_t current) : current(current) {}
 22 | 
 23 |   __host__ __device__ type_t operator*() const { return current; }
 24 | 
 25 |   __host__ __device__ type_t const* operator->() const { return &current; }
 26 | 
 27 |   __host__ __device__ range_iter_base& operator++() {
 28 |     ++current;
 29 |     return *this;
 30 |   }
 31 | 
 32 |   __host__ __device__ range_iter_base operator++(int) {
 33 |     auto copy = *this;
 34 |     ++*this;
 35 |     return copy;
 36 |   }
 37 | 
 38 |   __host__ __device__ bool operator==(range_iter_base const& other) const {
 39 |     return current == other.current;
 40 |   }
 41 | 
 42 |   __host__ __device__ bool operator!=(range_iter_base const& other) const {
 43 |     return !(*this == other);
 44 |   }
 45 | 
 46 |  protected:
 47 |   type_t current;
 48 | };
 49 | 
 50 | }  // namespace detail
 51 | 
 52 | template <typename type_t>
 53 | struct range_proxy {
 54 |   struct iter : detail::range_iter_base<type_t> {
 55 |     __host__ __device__ iter(type_t current)
 56 |         : detail::range_iter_base<type_t>(current) {}
 57 |   };
 58 | 
 59 |   struct step_range_proxy {
 60 |     struct iter : detail::range_iter_base<type_t> {
 61 |       __host__ __device__ iter(type_t current, type_t step)
 62 |           : detail::range_iter_base<type_t>(current), step(step) {}
 63 | 
 64 |       using detail::range_iter_base<type_t>::current;
 65 | 
 66 |       __host__ __device__ iter& operator++() {
 67 |         current += step;
 68 |         return *this;
 69 |       }
 70 | 
 71 |       __host__ __device__ iter operator++(int) {
 72 |         auto copy = *this;
 73 |         ++*this;
 74 |         return copy;
 75 |       }
 76 | 
 77 |       // Loses commutativity. Iterator-based ranges are simply broken.
 78 |       __host__ __device__ bool operator==(iter const& other) const {
 79 |         return step > 0 ? current >= other.current : current < other.current;
 80 |       }
 81 | 
 82 |       __host__ __device__ bool operator!=(iter const& other) const {
 83 |         return !(*this == other);
 84 |       }
 85 | 
 86 |      private:
 87 |       type_t step;
 88 |     };
 89 | 
 90 |     __host__ __device__ step_range_proxy(type_t begin, type_t end, type_t step)
 91 |         : begin_(begin, step), end_(end, step) {}
 92 | 
 93 |     __host__ __device__ iter begin() const { return begin_; }
 94 | 
 95 |     __host__ __device__ iter end() const { return end_; }
 96 | 
 97 |    private:
 98 |     iter begin_;
 99 |     iter end_;
100 |   };
101 | 
102 |   __host__ __device__ range_proxy(type_t begin, type_t end)
103 |       : begin_(begin), end_(end) {}
104 | 
105 |   __host__ __device__ step_range_proxy step(type_t step) {
106 |     return {*begin_, *end_, step};
107 |   }
108 | 
109 |   __host__ __device__ iter begin() const { return begin_; }
110 | 
111 |   __host__ __device__ iter end() const { return end_; }
112 | 
113 |  private:
114 |   iter begin_;
115 |   iter end_;
116 | };
117 | 
118 | template <typename type_t>
119 | struct infinite_range_proxy {
120 |   struct iter : detail::range_iter_base<type_t> {
121 |     __host__ __device__ iter(type_t current = type_t())
122 |         : detail::range_iter_base<type_t>(current) {}
123 | 
124 |     __host__ __device__ bool operator==(iter const&) const { return false; }
125 | 
126 |     __host__ __device__ bool operator!=(iter const&) const { return true; }
127 |   };
128 | 
129 |   struct step_range_proxy {
130 |     struct iter : detail::range_iter_base<type_t> {
131 |       __host__ __device__ iter(type_t current = type_t(),
132 |                                type_t step = type_t())
133 |           : detail::range_iter_base<type_t>(current), step(step) {}
134 | 
135 |       using detail::range_iter_base<type_t>::current;
136 | 
137 |       __host__ __device__ iter& operator++() {
138 |         current += step;
139 |         return *this;
140 |       }
141 | 
142 |       __host__ __device__ iter operator++(int) {
143 |         auto copy = *this;
144 |         ++*this;
145 |         return copy;
146 |       }
147 | 
148 |       __host__ __device__ bool operator==(iter const&) const { return false; }
149 | 
150 |       __host__ __device__ bool operator!=(iter const&) const { return true; }
151 | 
152 |      private:
153 |       type_t step;
154 |     };
155 | 
156 |     __host__ __device__ step_range_proxy(type_t begin, type_t step)
157 |         : begin_(begin, step) {}
158 | 
159 |     __host__ __device__ iter begin() const { return begin_; }
160 | 
161 |     __host__ __device__ iter end() const { return iter(); }
162 | 
163 |    private:
164 |     iter begin_;
165 |   };
166 | 
167 |   __host__ __device__ infinite_range_proxy(type_t begin) : begin_(begin) {}
168 | 
169 |   __host__ __device__ step_range_proxy step(type_t step) {
170 |     return step_range_proxy(*begin_, step);
171 |   }
172 | 
173 |   __host__ __device__ iter begin() const { return begin_; }
174 | 
175 |   __host__ __device__ iter end() const { return iter(); }
176 | 
177 |  private:
178 |   iter begin_;
179 | };
180 | 
181 | template <typename type_t>
182 | __host__ __device__ range_proxy<type_t> range(type_t begin, type_t end) {
183 |   return {begin, end};
184 | }
185 | 
186 | template <typename type_t>
187 | __host__ __device__ infinite_range_proxy<type_t> range(type_t begin) {
188 |   return {begin};
189 | }
190 | 
191 | namespace traits {
192 | 
193 | template <typename C>
194 | struct has_size {
195 |   template <typename type_t>
196 |   static constexpr auto check(type_t*) -> typename std::is_integral<
197 |       decltype(std::declval<type_t const>().size())>::type;
198 | 
199 |   template <typename>
200 |   static constexpr auto check(...) -> std::false_type;
201 | 
202 |   using type = decltype(check<C>(0));
203 |   static constexpr bool value = type::value;
204 | };
205 | 
206 | }  // namespace traits
207 | 
208 | template <typename C,
209 |           typename = typename std::enable_if<traits::has_size<C>::value>>
210 | __host__ __device__ auto indices(C const& cont)
211 |     -> range_proxy<decltype(cont.size())> {
212 |   return {0, cont.size()};
213 | }
214 | 
215 | template <typename type_t, std::size_t N>
216 | __host__ __device__ range_proxy<std::size_t> indices(type_t (&)[N]) {
217 |   return {0, N};
218 | }
219 | 
220 | template <typename type_t>
221 | range_proxy<typename std::initializer_list<type_t>::size_type>
222 |     __host__ __device__ indices(std::initializer_list<type_t>&& cont) {
223 |   return {0, cont.size()};
224 | }
225 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/schedule.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file schedule.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Header file for the schedule class.
 5 |  * @version 0.1
 6 |  * @date 2022-02-04
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <cstddef>
15 | 
16 | namespace loops {
17 | namespace schedule {
18 | 
19 | /**
20 |  * @brief Load balancing algorithms.
21 |  *
22 |  */
23 | enum algorithms_t {
24 |   merge_path_flat,  /// < Merge-path flat scheduling algorithm.
25 |   work_oriented,    /// < Work oriented scheduling algorithm.
26 |   thread_mapped,    /// < Thread mapped scheduling algorithm.
27 |   group_mapped,     /// < Group mapped scheduling algorithm.
28 |   bucketing,        /// < Bucketing scheduling algorithm.
29 | };
30 | 
31 | template <algorithms_t scheme, typename atoms_t, typename atom_size_t>
32 | class atom_traits;
33 | 
34 | template <algorithms_t scheme, typename tiles_t, typename tile_size_t>
35 | class tile_traits;
36 | 
37 | /**
38 |  * @brief Schedule's setup interface.
39 |  *
40 |  * @tparam scheme The scheduling algorithm.
41 |  * @tparam threads_per_block Number of threads per block.
42 |  * @tparam threads_per_tile Number of threads per tile.
43 |  * @tparam tiles_t Type of the tiles.
44 |  * @tparam atoms_t Type of the atoms.
45 |  * @tparam tile_size_t Type of the tile size (default: std::size_t).
46 |  * @tparam atom_size_t Type of the atom size (default: std::size_t).
47 |  */
48 | template <algorithms_t scheme,
49 |           std::size_t threads_per_block,
50 |           std::size_t threads_per_tile,
51 |           typename tiles_t,
52 |           typename atoms_t,
53 |           typename tile_size_t = std::size_t,
54 |           typename atom_size_t = std::size_t>
55 | class setup;
56 | 
57 | }  // namespace schedule
58 | }  // namespace loops
59 | 
60 | #include <loops/schedule/thread_mapped.hxx>
61 | #include <loops/schedule/group_mapped.hxx>
62 | #include <loops/schedule/work_oriented.hxx>
63 | #include <loops/schedule/merge_path_flat.hxx>


--------------------------------------------------------------------------------
/include/loops/schedule/group_mapped.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file group_mapped.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Group-mapped schedule (map work to tiles, process using individual
  5 |  * threads within the tile.)
  6 |  * @version 0.1
  7 |  * @date 2022-02-04
  8 |  *
  9 |  * @copyright Copyright (c) 2022
 10 |  *
 11 |  */
 12 | 
 13 | #pragma once
 14 | 
 15 | #include <loops/stride_ranges.hxx>
 16 | 
 17 | #include <thrust/binary_search.h>
 18 | #include <thrust/execution_policy.h>
 19 | #include <thrust/distance.h>
 20 | 
 21 | #ifndef _CG_ABI_EXPERIMENTAL
 22 | #define _CG_ABI_EXPERIMENTAL
 23 | #endif
 24 | 
 25 | #include <cooperative_groups.h>
 26 | #include <cooperative_groups/scan.h>
 27 | 
 28 | namespace loops {
 29 | namespace schedule {
 30 | 
 31 | namespace cg = cooperative_groups;
 32 | #if CUDART_VERSION >= 12000
 33 | namespace cg_x = cg;
 34 | #else
 35 | namespace cg_x = cooperative_groups::experimental;
 36 | #endif
 37 | 
 38 | template <typename atoms_type, typename atom_size_type>
 39 | class atom_traits<algorithms_t::group_mapped, atoms_type, atom_size_type> {
 40 |  public:
 41 |   using atoms_t = atoms_type;
 42 |   using atoms_iterator_t = atoms_t*;
 43 |   using atom_size_t = atom_size_type;
 44 | 
 45 |   __host__ __device__ atom_traits() : size_(0), atoms_(nullptr) {}
 46 |   __host__ __device__ atom_traits(atom_size_t size)
 47 |       : size_(size), atoms_(nullptr) {}
 48 |   __host__ __device__ atom_traits(atom_size_t size, atoms_iterator_t atoms)
 49 |       : size_(size), atoms_(atoms) {}
 50 | 
 51 |   __host__ __device__ atom_size_t size() const { return size_; }
 52 |   __host__ __device__ atoms_iterator_t begin() { return atoms_; };
 53 |   __host__ __device__ atoms_iterator_t end() { return atoms_ + size_; };
 54 | 
 55 |  private:
 56 |   atom_size_t size_;
 57 |   atoms_iterator_t atoms_;
 58 | };
 59 | 
 60 | template <typename tiles_type, typename tile_size_type>
 61 | class tile_traits<algorithms_t::group_mapped, tiles_type, tile_size_type> {
 62 |  public:
 63 |   using tiles_t = tiles_type;
 64 |   using tiles_iterator_t = tiles_t*;
 65 |   using tile_size_t = tile_size_type;
 66 | 
 67 |   __host__ __device__ tile_traits() : size_(0), tiles_(nullptr) {}
 68 |   __host__ __device__ tile_traits(tile_size_t size, tiles_iterator_t tiles)
 69 |       : size_(size), tiles_(tiles) {}
 70 | 
 71 |   __host__ __device__ tile_size_t size() const { return size_; }
 72 |   __host__ __device__ tiles_iterator_t begin() { return tiles_; };
 73 |   __host__ __device__ tiles_iterator_t end() { return tiles_ + size_; };
 74 | 
 75 |  private:
 76 |   tile_size_t size_;
 77 |   tiles_iterator_t tiles_;
 78 | };
 79 | 
 80 | template <std::size_t THREADS_PER_BLOCK,
 81 |           std::size_t THREADS_PER_TILE,
 82 |           typename tiles_type,
 83 |           typename atoms_type,
 84 |           typename tile_size_type,
 85 |           typename atom_size_type>
 86 | class setup<algorithms_t::group_mapped,
 87 |             THREADS_PER_BLOCK,
 88 |             THREADS_PER_TILE,
 89 |             tiles_type,
 90 |             atoms_type,
 91 |             tile_size_type,
 92 |             atom_size_type> : public tile_traits<algorithms_t::group_mapped,
 93 |                                                  tiles_type,
 94 |                                                  tile_size_type>,
 95 |                               public atom_traits<algorithms_t::group_mapped,
 96 |                                                  atoms_type,
 97 |                                                  atom_size_type> {
 98 |  public:
 99 |   using tiles_t = tiles_type;
100 |   using atoms_t = atoms_type;
101 |   using tiles_iterator_t = tiles_t*;
102 |   using atoms_iterator_t = atoms_t*;
103 |   using tile_size_t = tile_size_type;
104 |   using atom_size_t = atom_size_type;
105 | 
106 |   using tile_traits_t =
107 |       tile_traits<algorithms_t::group_mapped, tiles_type, tile_size_type>;
108 |   using atom_traits_t =
109 |       atom_traits<algorithms_t::group_mapped, atoms_type, atom_size_type>;
110 | 
111 |   enum : unsigned int {
112 |     threads_per_block = THREADS_PER_BLOCK,
113 |     threads_per_tile = THREADS_PER_TILE,
114 |     tiles_per_block = THREADS_PER_BLOCK / THREADS_PER_TILE,
115 |   };
116 | 
117 |   /// Temporary storage buffer for schedule algorithm.
118 |   struct __align__(32) storage_t {
119 | #if CUDART_VERSION >= 12000
120 |     cg_x::block_tile_memory<threads_per_block> groups;
121 | #else
122 |     cg_x::block_tile_memory<4, threads_per_block> groups;
123 | #endif
124 |     atoms_t tile_aggregates[threads_per_block / threads_per_tile];
125 |     atoms_t atoms_offsets[threads_per_block];
126 |     tiles_t tiles_indices[threads_per_block];
127 |   };
128 | 
129 |   storage_t& buffer;
130 | 
131 |   /**
132 |    * @brief Construct a setup object for load balance schedule.
133 |    *
134 |    * @param tiles Tiles iterator.
135 |    * @param num_tiles Number of tiles.
136 |    * @param num_atoms Number of atoms.
137 |    */
138 |   __device__ __forceinline__ setup(storage_t& _buffer,
139 |                                    tiles_iterator_t _tiles,
140 |                                    tile_size_t _num_tiles,
141 |                                    atom_size_t _num_atoms)
142 |       : buffer(_buffer),
143 |         tile_traits_t(_num_tiles, _tiles),
144 |         atom_traits_t(_num_atoms) {}
145 | 
146 |   __device__ __forceinline__ auto partition() {
147 |     auto g = cg::this_grid();
148 |     auto b = cg_x::this_thread_block(buffer.groups);
149 |     auto p = cg_x::tiled_partition<threads_per_tile>(b);
150 | 
151 |     auto index = g.thread_rank();
152 |     auto tile_index = p.thread_rank() + (p.meta_group_rank() * p.size());
153 |     if (index < tile_traits_t::size()) {
154 |       buffer.tiles_indices[tile_index] = index;
155 |     } else {
156 |       buffer.tiles_indices[tile_index] = -1;
157 |     }
158 |     return p;
159 |   }
160 | 
161 |   template <typename partition_t>
162 |   __device__ step_range_t<atoms_t> atom_accessor(partition_t& p) {
163 |     atoms_t* p_st =
164 |         buffer.atoms_offsets + (p.meta_group_rank() * threads_per_tile);
165 |     auto g = cg::this_grid();
166 |     auto index = g.thread_rank();
167 |     atoms_t num_atoms = 0;
168 |     if (index < tile_traits_t::size()) {
169 |       num_atoms =
170 |           tile_traits_t::begin()[index + 1] - tile_traits_t::begin()[index];
171 |     }
172 | 
173 |     p_st[p.thread_rank()] = cg::exclusive_scan(p, num_atoms);
174 |     p.sync();
175 | 
176 |     if (p.thread_rank() == p.size() - 1) {
177 |       // Accumulate tiled aggregates.
178 |       buffer.tile_aggregates[p.meta_group_rank()] =
179 |           p_st[p.thread_rank()] + num_atoms;
180 |     }
181 | 
182 |     p.sync();
183 |     atoms_t aggregate_atoms = buffer.tile_aggregates[p.meta_group_rank()];
184 |     return custom_stride_range(atoms_t(p.thread_rank()), aggregate_atoms,
185 |                                atoms_t(p.size()));
186 |   }
187 | 
188 |   template <typename partition_t>
189 |   __device__ __forceinline__ int get_length(partition_t& p) {
190 |     auto g = cg::this_grid();
191 | 
192 |     auto thread_id = g.thread_rank();
193 |     auto local_id = p.thread_rank();
194 | 
195 |     int length = thread_id - local_id + p.size();
196 |     if (tile_traits_t::size() < length)
197 |       length = tile_traits_t::size();
198 | 
199 |     length -= thread_id - local_id;
200 |     return length;
201 |   }
202 | 
203 |   template <typename partition_t>
204 |   __device__ __forceinline__ tiles_t tile_accessor(atoms_t& virtual_atom,
205 |                                                    partition_t& p) {
206 |     int length = get_length(p);
207 |     atoms_t* p_st =
208 |         buffer.atoms_offsets + (p.meta_group_rank() * threads_per_tile);
209 |     auto it =
210 |         thrust::upper_bound(thrust::seq, p_st, p_st + length, virtual_atom);
211 |     auto x = thrust::distance(p_st, it) - 1;
212 |     return x;
213 |   }
214 | 
215 |   template <typename partition_t>
216 |   __device__ __forceinline__ bool is_valid_accessor(tiles_t& tile_id,
217 |                                                     partition_t& p) {
218 |     return tile_id < get_length(p);
219 |   }
220 | 
221 |   template <typename partition_t>
222 |   __device__ __forceinline__ tiles_t tile_id(tiles_t& v_tile_id,
223 |                                              partition_t& p) {
224 |     return buffer.tiles_indices[v_tile_id + (p.meta_group_rank() * p.size())];
225 |   }
226 | 
227 |   template <typename partition_t>
228 |   __device__ __forceinline__ atoms_t atom_id(atoms_t& v_atom,
229 |                                              tiles_t& tile_id,
230 |                                              tiles_t& v_tile_id,
231 |                                              partition_t& p) {
232 |     atoms_t* p_st =
233 |         buffer.atoms_offsets + (p.meta_group_rank() * threads_per_tile);
234 |     return tile_traits_t::begin()[tile_id] + v_atom - p_st[v_tile_id];
235 |   }
236 | 
237 | };  // namespace schedule
238 | 
239 | template <std::size_t threads_per_block, typename tiles_t, typename atoms_t>
240 | using warp_mapped =
241 |     setup<algorithms_t::group_mapped, threads_per_block, 32, tiles_t, atoms_t>;
242 | 
243 | template <std::size_t threads_per_block, typename tiles_t, typename atoms_t>
244 | using block_mapped = setup<algorithms_t::group_mapped,
245 |                            threads_per_block,
246 |                            threads_per_block,
247 |                            tiles_t,
248 |                            atoms_t>;
249 | 
250 | }  // namespace schedule
251 | }  // namespace loops
252 | 


--------------------------------------------------------------------------------
/include/loops/schedule/thread_mapped.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file thread_mapped.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief
  5 |  * @version 0.1
  6 |  * @date 2022-02-04
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include <loops/stride_ranges.hxx>
 15 | #include <loops/schedule.hxx>
 16 | 
 17 | namespace loops {
 18 | namespace schedule {
 19 | 
 20 | /**
 21 |  * @brief Traits for Atom.
 22 |  *
 23 |  * @todo Implement an atom iterator, right now it is based on CSR only. Can be
 24 |  * abstracted very simply by allowing UDF iterators.
 25 |  *
 26 |  * @tparam atoms_type Type of the atoms.
 27 |  * @tparam atom_size_type Type of the atom size.
 28 |  */
 29 | template <typename atoms_type, typename atom_size_type>
 30 | class atom_traits<algorithms_t::thread_mapped, atoms_type, atom_size_type> {
 31 |  public:
 32 |   using atoms_t = atoms_type;
 33 |   using atoms_iterator_t = atoms_t*;
 34 |   using atom_size_t = atom_size_type;
 35 | 
 36 |   __host__ __device__ atom_traits() : size_(0), atoms_(nullptr) {}
 37 |   __host__ __device__ atom_traits(atom_size_t size)
 38 |       : size_(size), atoms_(nullptr) {}
 39 |   __host__ __device__ atom_traits(atom_size_t size, atoms_iterator_t atoms)
 40 |       : size_(size), atoms_(atoms) {}
 41 | 
 42 |   __host__ __device__ atom_size_t size() const { return size_; }
 43 |   __host__ __device__ atoms_iterator_t begin() { return atoms_; };
 44 |   __host__ __device__ atoms_iterator_t end() { return atoms_ + size_; };
 45 | 
 46 |  private:
 47 |   atom_size_t size_;
 48 |   atoms_iterator_t atoms_;
 49 | };
 50 | 
 51 | /**
 52 |  * @brief Traits for Tile.
 53 |  *
 54 |  * @todo Implement an tile iterator, right now it is based on CSR only. Can be
 55 |  * abstracted very simply by allowing UDF iterators.
 56 |  *
 57 |  * @tparam tiles_type Type of the tiles.
 58 |  * @tparam tile_size_type Type of the tile size (default: std::size_t).
 59 |  */
 60 | template <typename tiles_type, typename tile_size_type>
 61 | class tile_traits<algorithms_t::thread_mapped, tiles_type, tile_size_type> {
 62 |  public:
 63 |   using tiles_t = tiles_type;
 64 |   using tiles_iterator_t = tiles_t*;
 65 |   using tile_size_t = tile_size_type;
 66 | 
 67 |   __host__ __device__ tile_traits() : size_(0), tiles_(nullptr) {}
 68 |   __host__ __device__ tile_traits(tile_size_t size, tiles_iterator_t tiles)
 69 |       : size_(size), tiles_(tiles) {}
 70 | 
 71 |   __host__ __device__ tile_size_t size() const { return size_; }
 72 |   __host__ __device__ tiles_iterator_t begin() { return tiles_; };
 73 |   __host__ __device__ tiles_iterator_t end() { return tiles_ + size_; };
 74 | 
 75 |  private:
 76 |   tile_size_t size_;
 77 |   tiles_iterator_t tiles_;
 78 | };
 79 | 
 80 | /**
 81 |  * @brief Thread-mapped schedule's setup interface.
 82 |  *
 83 |  * @tparam tiles_type Type of the tiles.
 84 |  * @tparam atoms_type Type of the atoms.
 85 |  * @tparam tile_size_type Type of the tile size.
 86 |  * @tparam atom_size_type Type of the atom size.
 87 |  */
 88 | template <typename tiles_type,
 89 |           typename atoms_type,
 90 |           typename tile_size_type,
 91 |           typename atom_size_type>
 92 | class setup<algorithms_t::thread_mapped,
 93 |             1,
 94 |             1,
 95 |             tiles_type,
 96 |             atoms_type,
 97 |             tile_size_type,
 98 |             atom_size_type> : public tile_traits<algorithms_t::thread_mapped,
 99 |                                                  tiles_type,
100 |                                                  tile_size_type>,
101 |                               public atom_traits<algorithms_t::thread_mapped,
102 |                                                  atoms_type,
103 |                                                  atom_size_type> {
104 |  public:
105 |   using tiles_t = tiles_type;          /// Tile Type
106 |   using atoms_t = atoms_type;          /// Atom Type
107 |   using tiles_iterator_t = tiles_t*;   /// Tile Iterator Type
108 |   using atoms_iterator_t = tiles_t*;   /// Atom Iterator Type
109 |   using tile_size_t = tile_size_type;  /// Tile Size Type
110 |   using atom_size_t = atom_size_type;  /// Atom Size Type
111 | 
112 |   using tile_traits_t =
113 |       tile_traits<algorithms_t::thread_mapped, tiles_type, tile_size_type>;
114 |   using atom_traits_t =
115 |       atom_traits<algorithms_t::thread_mapped, atoms_type, atom_size_type>;
116 | 
117 |   /**
118 |    * @brief Default constructor.
119 |    *
120 |    */
121 |   __host__ __device__ setup() : tile_traits_t(), atom_traits_t() {}
122 | 
123 |   /**
124 |    * @brief Construct a setup object for load balance schedule.
125 |    *
126 |    * @param tiles Tiles iterator.
127 |    * @param num_tiles Number of tiles.
128 |    * @param num_atoms Number of atoms.
129 |    */
130 |   __host__ __device__ setup(tiles_t* tiles,
131 |                             tile_size_t num_tiles,
132 |                             atom_size_t num_atoms)
133 |       : tile_traits_t(num_tiles, tiles), atom_traits_t(num_atoms) {}
134 | 
135 |   /**
136 |    * @brief Creates a range of tiles to process within a for loop.
137 |    *
138 |    * @example The following code snippet shows how to use this function.
139 |    * \code{.cpp}
140 |    * for (auto t : config.tiles()) {
141 |    *  // Process tile t.
142 |    * }
143 |    * \endcode
144 |    *
145 |    * @return grid_stride_range<tile_size_t> Range of tiles to process.
146 |    */
147 |   __device__ step_range_t<tile_size_t> tiles() const {
148 |     return grid_stride_range(tile_size_t(0), tile_traits_t::size());
149 |   }
150 | 
151 |   /**
152 |    * @brief Creates a range of atoms to process within a for loop.
153 |    *
154 |    * @param tid Tile ID for which to create the atom range for.
155 |    * @return Range of atoms to process.
156 |    */
157 |   __device__ auto atoms(const tile_size_t& tile) {
158 |     return loops::range(tile_traits_t::begin()[tile],
159 |                         tile_traits_t::begin()[tile + 1]);
160 |   }
161 | 
162 |   template <typename iterator_t>
163 |   __device__ auto atoms(const tile_size_t& tile, iterator_t count_entries) {
164 |     return loops::range(count_entries(tile), tile_traits_t::begin()[tile + 1]);
165 |   }
166 | };
167 | 
168 | }  // namespace schedule
169 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/schedule/work_oriented.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file work_oriented.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief Work-oriented scheduling algorithm (map even-share of work to
  5 |  * threads.)
  6 |  * @version 0.1
  7 |  * @date 2022-03-07
  8 |  *
  9 |  * @copyright Copyright (c) 2022
 10 |  *
 11 |  */
 12 | 
 13 | #pragma once
 14 | 
 15 | #include <loops/stride_ranges.hxx>
 16 | #include <loops/util/math.hxx>
 17 | 
 18 | #include <thrust/binary_search.h>
 19 | #include <thrust/distance.h>
 20 | #include <thrust/iterator/counting_iterator.h>
 21 | #include <thrust/pair.h>
 22 | #include <thrust/execution_policy.h>
 23 | 
 24 | namespace loops {
 25 | namespace schedule {
 26 | 
 27 | /**
 28 |  * @brief Traits for Atom.
 29 |  *
 30 |  * @todo Implement an atom iterator, right now it is based on CSR only. Can be
 31 |  * abstracted very simply by allowing UDF iterators.
 32 |  *
 33 |  * @tparam atoms_type Type of the atoms.
 34 |  * @tparam atom_size_type Type of the atom size.
 35 |  */
 36 | template <typename atoms_type, typename atom_size_type>
 37 | class atom_traits<algorithms_t::work_oriented, atoms_type, atom_size_type> {
 38 |  public:
 39 |   using atoms_t = atoms_type;
 40 |   using atoms_iterator_t = atoms_t*;
 41 |   using atom_size_t = atom_size_type;
 42 | 
 43 |   __host__ __device__ atom_traits() : size_(0), atoms_(nullptr) {}
 44 |   __host__ __device__ atom_traits(atom_size_t size)
 45 |       : size_(size), atoms_(nullptr) {}
 46 |   __host__ __device__ atom_traits(atom_size_t size, atoms_iterator_t atoms)
 47 |       : size_(size), atoms_(atoms) {}
 48 | 
 49 |   __host__ __device__ atom_size_t size() const { return size_; }
 50 |   __host__ __device__ atoms_iterator_t begin() { return atoms_; };
 51 |   __host__ __device__ atoms_iterator_t end() { return atoms_ + size_; };
 52 | 
 53 |  private:
 54 |   atom_size_t size_;
 55 |   atoms_iterator_t atoms_;
 56 | };
 57 | 
 58 | /**
 59 |  * @brief Traits for Tile.
 60 |  *
 61 |  * @todo Implement an tile iterator, right now it is based on CSR only. Can be
 62 |  * abstracted very simply by allowing UDF iterators.
 63 |  *
 64 |  * @tparam tiles_type Type of the tiles.
 65 |  * @tparam tile_size_type Type of the tile size (default: std::size_t).
 66 |  */
 67 | template <typename tiles_type, typename tile_size_type>
 68 | class tile_traits<algorithms_t::work_oriented, tiles_type, tile_size_type> {
 69 |  public:
 70 |   using tiles_t = tiles_type;
 71 |   using tiles_iterator_t = tiles_t*;
 72 |   using tile_size_t = tile_size_type;
 73 | 
 74 |   __host__ __device__ tile_traits() : size_(0), tiles_(nullptr) {}
 75 |   __host__ __device__ tile_traits(tile_size_t size, tiles_iterator_t tiles)
 76 |       : size_(size), tiles_(tiles) {}
 77 | 
 78 |   __host__ __device__ tile_size_t size() const { return size_; }
 79 |   __host__ __device__ tiles_iterator_t begin() { return tiles_; };
 80 |   __host__ __device__ tiles_iterator_t end() { return tiles_ + size_; };
 81 | 
 82 |  private:
 83 |   tile_size_t size_;
 84 |   tiles_iterator_t tiles_;
 85 | };
 86 | 
 87 | /**
 88 |  * @brief Work-oriented schedule's setup interface.
 89 |  *
 90 |  * @tparam THREADS_PER_BLOCK Threads per block.
 91 |  * @tparam ITEMS_PER_THREAD Number of Items per thread to process.
 92 |  * @tparam tiles_type Type of the tiles.
 93 |  * @tparam atoms_type Type of the atoms.
 94 |  * @tparam tile_size_type Type of the tile size.
 95 |  * @tparam atom_size_type Type of the atom size.
 96 |  */
 97 | template <std::size_t THREADS_PER_BLOCK,
 98 |           std::size_t ITEMS_PER_THREAD,
 99 |           typename tiles_type,
100 |           typename atoms_type,
101 |           typename tile_size_type,
102 |           typename atom_size_type>
103 | class setup<algorithms_t::work_oriented,
104 |             THREADS_PER_BLOCK,
105 |             ITEMS_PER_THREAD,
106 |             tiles_type,
107 |             atoms_type,
108 |             tile_size_type,
109 |             atom_size_type> : public tile_traits<algorithms_t::work_oriented,
110 |                                                  tiles_type,
111 |                                                  tile_size_type>,
112 |                               public atom_traits<algorithms_t::work_oriented,
113 |                                                  atoms_type,
114 |                                                  atom_size_type> {
115 |  public:
116 |   using tiles_t = tiles_type;          /// Tile Type
117 |   using atoms_t = atoms_type;          /// Atom Type
118 |   using tiles_iterator_t = tiles_t*;   /// Tile Iterator Type
119 |   using atoms_iterator_t = tiles_t*;   /// Atom Iterator Type
120 |   using tile_size_t = tile_size_type;  /// Tile Size Type
121 |   using atom_size_t = atom_size_type;  /// Atom Size Type
122 | 
123 |   using tile_traits_t =
124 |       tile_traits<algorithms_t::work_oriented, tiles_type, tile_size_type>;
125 |   using atom_traits_t =
126 |       atom_traits<algorithms_t::work_oriented, atoms_type, atom_size_type>;
127 | 
128 |   enum : unsigned int {
129 |     threads_per_block = THREADS_PER_BLOCK,
130 |     items_per_thread = ITEMS_PER_THREAD,
131 |     items_per_tile = threads_per_block * items_per_thread,
132 |   };
133 | 
134 |   /**
135 |    * @brief Construct a setup object for load balance schedule.
136 |    *
137 |    * @param tiles Tiles iterator.
138 |    * @param num_tiles Number of tiles.
139 |    * @param num_atoms Number of atoms.
140 |    */
141 |   __device__ __forceinline__ setup(tiles_iterator_t _tiles,
142 |                                    tile_size_t _num_tiles,
143 |                                    atom_size_t _num_atoms)
144 |       : tile_traits_t(_num_tiles, _tiles),
145 |         atom_traits_t(_num_atoms),
146 |         total_work(_num_tiles + _num_atoms),
147 |         num_threads(gridDim.x * threads_per_block),
148 |         work_per_thread(math::ceil_div(total_work, num_threads)) {}
149 | 
150 |   __device__ __forceinline__ auto init() {
151 |     thrust::counting_iterator<atoms_t> atoms_indices(0);
152 | 
153 |     /// Calculate the diagonals.
154 |     atom_size_t tid = threadIdx.x + blockIdx.x * blockDim.x;
155 |     atom_size_t upper =
156 |         min(atom_size_t(work_per_thread * tid), atom_size_t(total_work));
157 |     atom_size_t lower =
158 |         min(atom_size_t(upper + work_per_thread), atom_size_t(total_work));
159 | 
160 |     /// Search across the diagonals to find coordinates to process.
161 |     auto st = search(upper, tile_traits_t::begin() + 1, atoms_indices,
162 |                      tile_traits_t::size(), atom_traits_t::size());
163 |     auto en = search(lower, tile_traits_t::begin() + 1, atoms_indices,
164 |                      tile_traits_t::size(), atom_traits_t::size());
165 | 
166 |     return thrust::make_pair(st, en);
167 |   }
168 | 
169 |   template <typename map_t>
170 |   __device__ __forceinline__ step_range_t<tiles_t> tiles(map_t& m) const {
171 |     return custom_stride_range(tiles_t(m.first.first), tiles_t(m.second.first),
172 |                                tiles_t(1));
173 |   }
174 | 
175 |   template <typename map_t>
176 |   __device__ __forceinline__ step_range_t<atoms_t> atoms(tiles_t t, map_t& m) {
177 |     atoms_t nz_next_start = tile_traits_t::begin()[(t + 1)];
178 |     atoms_t nz_start = m.first.second;
179 |     m.first.second += (nz_next_start - nz_start);
180 |     return custom_stride_range(nz_start, nz_next_start, atoms_t(1));
181 |   }
182 | 
183 |   template <typename map_t>
184 |   __device__ __forceinline__ step_range_t<tiles_t> remainder_tiles(
185 |       map_t& m) const {
186 |     return custom_stride_range(tiles_t(m.second.first),
187 |                                tiles_t(m.second.first + 1), tiles_t(1));
188 |   }
189 | 
190 |   template <typename map_t>
191 |   __device__ __forceinline__ step_range_t<atoms_t> remainder_atoms(
192 |       map_t& m) const {
193 |     return custom_stride_range(atoms_t(m.first.second),
194 |                                (atoms_t)(m.second.second), atoms_t(1));
195 |   }
196 | 
197 |  private:
198 |   /**
199 |    * @brief Thrust based 2D binary-search for merge-path algorithm.
200 |    *
201 |    * @param diagonal Diagonal of the search.
202 |    * @param a First iterator.
203 |    * @param b Second iterator.
204 |    * @param a_len Length of the first iterator.
205 |    * @param b_len Length of the second iterator.
206 |    * @return A coordinate.
207 |    */
208 |   template <typename offset_t, typename xit_t, typename yit_t>
209 |   __device__ __forceinline__ auto search(const offset_t& diagonal,
210 |                                          const xit_t a,
211 |                                          const yit_t b,
212 |                                          const offset_t& a_len,
213 |                                          const offset_t& b_len) {
214 |     /// Diagonal search range (in x-coordinate space)
215 |     /// Note that the subtraction can result into a negative number, in which
216 |     /// case the max would result as 0. But if we use offset_t here, and it is
217 |     /// an unsigned type, we would get strange behavior, possible an unwanted
218 |     /// sign conversion that we do not want.
219 |     int x_min = max(int(diagonal) - int(b_len), int(0));
220 |     int x_max = min(int(diagonal), int(a_len));
221 | 
222 |     auto it = thrust::lower_bound(
223 |         thrust::seq,                                 // Sequential impl
224 |         thrust::counting_iterator<offset_t>(x_min),  // Start iterator @x_min
225 |         thrust::counting_iterator<offset_t>(x_max),  // End iterator @x_max
226 |         diagonal,                                    // ...
227 |         [=] __device__(const offset_t& idx, const offset_t& diagonal) {
228 |           return a[idx] <= b[diagonal - idx - 1];
229 |         });
230 | 
231 |     return thrust::make_pair(min(*it, a_len), (diagonal - *it));
232 |   }
233 | 
234 |   std::size_t total_work;
235 |   std::size_t num_threads;
236 |   std::size_t work_per_thread;
237 | };
238 | 
239 | }  // namespace schedule
240 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/stride_ranges.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file stride_ranges.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Stride ranges related functions.
 5 |  * @version 0.1
 6 |  * @date 2022-02-02
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | #include <loops/range.hxx>
14 | 
15 | namespace loops {
16 | template <typename T>
17 | using step_range_t = typename range_proxy<T>::step_range_proxy;
18 | 
19 | /**
20 |  * @brief Gride stride range defines a range that steps gridDim.x * blockDim.x
21 |  * per step.
22 |  *
23 |  * @tparam T The type of the range.
24 |  * @param begin The beginning of the range (0)
25 |  * @param end End of the range, can be num_elements.
26 |  * @return range_proxy<T>::step_range_proxy range returned.
27 |  */
28 | template <typename T>
29 | __device__ __forceinline__ step_range_t<T> grid_stride_range(T begin, T end) {
30 |   begin += blockDim.x * blockIdx.x + threadIdx.x;
31 |   return range(begin, end).step(gridDim.x * blockDim.x);
32 | }
33 | 
34 | /**
35 |  * @brief Block stride range defines as a range that steps blockDim.x per step.
36 |  *
37 |  * @tparam T The type of the range.
38 |  * @param begin The beginning of the range (e.g. 0)
39 |  * @param end End of the range, can be num_elements.
40 |  * @return step_range_t<T> range returned.
41 |  */
42 | template <typename T>
43 | __device__ __forceinline__ step_range_t<T> block_stride_range(T begin, T end) {
44 |   return range(begin, end).step(blockDim.x);
45 | }
46 | 
47 | /**
48 |  * @brief Custom stride range defines as a range that steps with a custom
49 |  * stride.
50 |  *
51 |  * @tparam T The type of the range.
52 |  * @param begin The beginning of the range (e.g. 0)
53 |  * @param end End of the range, can be num_elements.
54 |  * @param stride The stride of the range.
55 |  * @return step_range_t<T> range returned.
56 |  */
57 | template <typename T>
58 | __device__ __forceinline__ step_range_t<T> custom_stride_range(T begin,
59 |                                                                T end,
60 |                                                                T stride) {
61 |   return range(begin, end).step(stride);
62 | }
63 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/device.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file device.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Device related functions.
 5 |  * @version 0.1
 6 |  * @date 2022-07-10
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | namespace loops {
15 | namespace device {
16 | typedef int device_id_t;
17 | 
18 | /**
19 |  * @brief Set the device to use.
20 |  *
21 |  * @param ordinal device id.
22 |  */
23 | void set(device_id_t ordinal) {
24 |   cudaSetDevice(ordinal);
25 | }
26 | 
27 | /**
28 |  * @brief Get the device id.
29 |  *
30 |  * @return device_id_t device id.
31 |  */
32 | device_id_t get() {
33 |   device_id_t ordinal;
34 |   cudaGetDevice(&ordinal);
35 |   return ordinal;
36 | }
37 | 
38 | /**
39 |  * @brief Properties of the device.
40 |  *
41 |  */
42 | struct properties_t {
43 |   typedef cudaDeviceProp device_properties_t;
44 |   device_properties_t properties;
45 |   device_id_t ordinal;
46 | 
47 |   properties_t() : ordinal(device::get()) {
48 |     cudaGetDeviceProperties(&properties, ordinal);
49 |   }
50 | 
51 |   int multi_processor_count() { return properties.multiProcessorCount; }
52 | };
53 | 
54 | }  // namespace device
55 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/equal.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file compare.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief
 5 |  * @version 0.1
 6 |  * @date 2022-02-16
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <cstddef>
15 | #include <iostream>
16 | #include <iomanip>
17 | #include <limits>
18 | 
19 | namespace loops {
20 | namespace util {
21 | 
22 | namespace detail {
23 | auto default_comparator = [](auto& a, auto& b) -> bool { return a != b; };
24 | }
25 | 
26 | /**
27 |  * @brief Compares values between a device pointer and a host pointer of same
28 |  * length, and returns number of errors/mismatches found.
29 |  *
30 |  * @tparam type_t Type of the values to be compared.
31 |  * @tparam comp_t Type of the error comparator function.
32 |  * @param d_ptr device pointer
33 |  * @param h_ptr host pointer
34 |  * @param n number of elements to compare
35 |  * @param error_op lambda function to compare two value that result in a
36 |  * mismatch count to increment (default a != b).
37 |  * @param verbose if true, prints out the mismatches
38 |  * @return std::size_t number of mismatches found
39 |  */
40 | template <typename type_t,
41 |           typename comp_t = decltype(detail::default_comparator)>
42 | std::size_t equal(const type_t* d_ptr,
43 |                   const type_t* h_ptr,
44 |                   const std::size_t n,
45 |                   comp_t error_op = detail::default_comparator,
46 |                   const bool verbose = false) {
47 |   thrust::host_vector<type_t> d_vec(n);
48 |   cudaMemcpy(d_vec.data(), d_ptr, n * sizeof(type_t), cudaMemcpyDeviceToHost);
49 | 
50 |   std::size_t error_count = 0;
51 |   for (std::size_t i = 0; i < n; ++i) {
52 |     if (error_op(d_vec[i], h_ptr[i])) {
53 |       if (verbose)
54 |         std::cout << "Error[" << i << "]: " << std::setw(10) << std::fixed
55 |                   << std::setprecision(std::numeric_limits<type_t>::digits10)
56 |                   << d_vec[i] << " != " << std::setw(10) << h_ptr[i]
57 |                   << std::endl;
58 |       ++error_count;
59 |     }
60 |   }
61 |   return error_count;
62 | }
63 | 
64 | }  // namespace util
65 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/filepath.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file filepath.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Utilities for filepaths.
 5 |  * @version 0.1
 6 |  * @date 2022-02-03
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <string>
15 | 
16 | namespace loops {
17 | 
18 | std::string extract_filename(std::string path, std::string delim = "/") {
19 |   size_t lastSlashIndex = path.find_last_of("/");
20 |   return path.substr(lastSlashIndex + 1);
21 | }
22 | 
23 | std::string extract_dataset(std::string filename) {
24 |   size_t lastindex = filename.find_last_of(".");
25 |   return filename.substr(0, lastindex);
26 | }
27 | 
28 | bool is_market(std::string filename) {
29 |   return ((filename.substr(filename.size() - 4) == ".mtx") ||
30 |           (filename.substr(filename.size() - 5) == ".mmio"));
31 | }
32 | 
33 | bool is_binary_csr(std::string filename) {
34 |   return filename.substr(filename.size() - 4) == ".csr";
35 | }
36 | 
37 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/generate.hxx:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @file generate.hxx
  3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
  4 |  * @brief
  5 |  * @version 0.1
  6 |  * @date 2022-02-02
  7 |  *
  8 |  * @copyright Copyright (c) 2022
  9 |  *
 10 |  */
 11 | 
 12 | #pragma once
 13 | #include <chrono>
 14 | #include <thrust/random.h>
 15 | #include <thrust/distance.h>
 16 | #include <thrust/transform.h>
 17 | #include <thrust/execution_policy.h>
 18 | #include <thrust/iterator/counting_iterator.h>
 19 | 
 20 | #include <loops/memory.hxx>
 21 | #include <loops/container/formats.hxx>
 22 | 
 23 | namespace loops {
 24 | namespace generate {
 25 | namespace random {
 26 | 
 27 | /**
 28 |  * @brief Hash function for random numbers.
 29 |  *
 30 |  * @param a The number to hash.
 31 |  * @return unsigned int The hashed number.
 32 |  */
 33 | __forceinline__ __host__ __device__ unsigned int hash(unsigned int a) {
 34 |   a = (a + 0x7ed55d16) + (a << 12);
 35 |   a = (a ^ 0xc761c23c) ^ (a >> 19);
 36 |   a = (a + 0x165667b1) + (a << 5);
 37 |   a = (a + 0xd3a2646c) ^ (a << 9);
 38 |   a = (a + 0xfd7046c5) + (a << 3);
 39 |   a = (a ^ 0xb55a4f09) ^ (a >> 16);
 40 |   return a;
 41 | }
 42 | 
 43 | /**
 44 |  * @brief Generate a vector with uniform distribution of random numbers between
 45 |  * [begin, end].
 46 |  *
 47 |  * @tparam vector_t vector type.
 48 |  * @tparam vector_t::value_type value type.
 49 |  * @param input thrust host or device vector.
 50 |  * @param begin range begin value.
 51 |  * @param end range end value.
 52 |  * @param seed random seed.
 53 |  */
 54 | template <typename iterator_t, typename type_t>
 55 | void uniform_distribution(iterator_t begin_it,
 56 |                           iterator_t end_it,
 57 |                           type_t begin,
 58 |                           type_t end,
 59 |                           unsigned int useed = std::chrono::system_clock::now()
 60 |                                                    .time_since_epoch()
 61 |                                                    .count()) {
 62 |   int size = thrust::distance(begin_it, end_it);
 63 |   auto generate_random = [=] __host__ __device__(std::size_t i) -> type_t {
 64 |     unsigned int seed = hash(i) * useed;
 65 |     if (std::is_floating_point_v<type_t>) {
 66 |       thrust::default_random_engine rng(seed);
 67 |       thrust::uniform_real_distribution<type_t> uniform(begin, end);
 68 |       return uniform(rng);
 69 |     } else {
 70 |       thrust::default_random_engine rng(seed);
 71 |       thrust::uniform_int_distribution<type_t> uniform(begin, end);
 72 |       return uniform(rng);
 73 |     }
 74 |   };
 75 | 
 76 |   thrust::transform(thrust::make_counting_iterator<std::size_t>(0),
 77 |                     thrust::make_counting_iterator<std::size_t>(size), begin_it,
 78 |                     generate_random);
 79 | }
 80 | 
 81 | using namespace memory;
 82 | 
 83 | /**
 84 |  * @brief Generates a random Compressed Sparse Row (CSR) matrix.
 85 |  *
 86 |  * @tparam index_t index type.
 87 |  * @tparam offset_t offset type.
 88 |  * @tparam value_t value type.
 89 |  * @param rows number of rows.
 90 |  * @param columns number of columns.
 91 |  * @param sparsity sparsity ratio of the matrix.
 92 |  * @return csr_t<index_t, offset_t, value_t> random CSR matrix.
 93 |  */
 94 | template <typename index_t, typename offset_t, typename value_t>
 95 | void csr(std::size_t rows,
 96 |          std::size_t cols,
 97 |          float sparsity,
 98 |          csr_t<index_t, offset_t, value_t>& matrix) {
 99 |   std::size_t nnzs = sparsity * (rows * cols);
100 |   coo_t<index_t, value_t, memory_space_t::host> coo(rows, cols, nnzs);
101 | 
102 |   // Generate Random indices and values.
103 |   uniform_distribution(coo.row_indices.begin(), coo.row_indices.end(), 0,
104 |                        index_t(rows));
105 |   uniform_distribution(coo.col_indices.begin(), coo.col_indices.end(),
106 |                        index_t(0), index_t(cols));
107 |   uniform_distribution(coo.values.begin(), coo.values.end(), value_t(0.0),
108 |                        value_t(1.0));
109 | 
110 |   // Remove duplicates.
111 |   coo.remove_duplicates();
112 |   matrix = coo;
113 | }
114 | 
115 | }  // namespace random
116 | }  // namespace generate
117 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/launch.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file launch.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Kernel launch C++ related functions.
 5 |  * @version 0.1
 6 |  * @date 2022-07-10
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | #include <cstddef>
14 | #include <cuda.h>
15 | 
16 | namespace loops {
17 | namespace launch {
18 | namespace detail {
19 | inline void for_each_argument_address(void**) {}
20 | 
21 | template <typename arg_t, typename... args_t>
22 | inline void for_each_argument_address(void** collected_addresses,
23 |                                       arg_t&& arg,
24 |                                       args_t&&... args) {
25 |   collected_addresses[0] = const_cast<void*>(static_cast<const void*>(&arg));
26 |   for_each_argument_address(collected_addresses + 1,
27 |                             ::std::forward<args_t>(args)...);
28 | }
29 | }  // namespace detail
30 | 
31 | template <typename func_t, typename... args_t>
32 | void cooperative(cudaStream_t stream,
33 |                  const func_t& kernel,
34 |                  std::size_t number_of_blocks,
35 |                  std::size_t threads_per_block,
36 |                  args_t&&... args) {
37 |   constexpr const auto non_zero_num_params =
38 |       sizeof...(args_t) == 0 ? 1 : sizeof...(args_t);
39 |   void* argument_ptrs[non_zero_num_params];
40 |   detail::for_each_argument_address(argument_ptrs,
41 |                                     ::std::forward<args_t>(args)...);
42 |   cudaLaunchCooperativeKernel<func_t>(
43 |       &kernel, number_of_blocks, threads_per_block, argument_ptrs, 0, stream);
44 | }
45 | 
46 | /**
47 |  * @brief Launch a kernel.
48 |  *
49 |  * @par Overview
50 |  * This function is a reimplementation of `std::apply`, that allows
51 |  * for launching cuda kernels with launch param members of the class
52 |  * and a context argument. It follows the "possible implementation" of
53 |  * `std::apply` in the C++ reference:
54 |  * https://en.cppreference.com/w/cpp/utility/apply.
55 |  *
56 |  * @tparam func_t The type of the kernel function being passed in.
57 |  * @tparam args_tuple_t The type of the tuple of arguments being
58 |  * passed in.
59 |  * @param kernel Kernel function to call.
60 |  * @param args Tuple of arguments to be expanded as the arguments of
61 |  * the kernel function.
62 |  * @param context Reference to the context used to launch the kernel
63 |  * (used for the context's stream).
64 |  * \return void
65 |  */
66 | template <typename func_t, typename... args_t>
67 | void non_cooperative(cudaStream_t stream,
68 |                      const func_t& kernel,
69 |                      dim3 number_of_blocks,
70 |                      dim3 threads_per_block,
71 |                      args_t&&... args) {
72 |   kernel<<<number_of_blocks, threads_per_block, 0, stream>>>(
73 |       std::forward<args_t>(args)...);
74 | }
75 | }  // namespace launch
76 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/math.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file math.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Math related utilities.
 5 |  * @version 0.1
 6 |  * @date 2022-02-16
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | #pragma once
12 | 
13 | namespace loops {
14 | namespace math {
15 | 
16 | /**
17 |  * @brief Simple safe ceil division: (a + b - 1) / b. Handles overflow condition
18 |  * as well.
19 |  *
20 |  * @tparam numerator_t Type of the dividend.
21 |  * @tparam denominator_t Type of the divisor.
22 |  * @param n Dividend.
23 |  * @param d Divisor.
24 |  * @return The quotient.
25 |  */
26 | template <class numerator_t, class denominator_t>
27 | __host__ __device__ __forceinline__ constexpr numerator_t ceil_div(
28 |     numerator_t const& n,
29 |     denominator_t const& d) {
30 |   return static_cast<numerator_t>(n / d + (n % d != 0 ? 1 : 0));
31 | }
32 | 
33 | }  // namespace math
34 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/sample.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file sample.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief
 5 |  * @version 0.1
 6 |  * @date 2022-07-19
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <loops/memory.hxx>
15 | #include <loops/container/formats.hxx>
16 | 
17 | namespace loops {
18 | namespace sample {
19 | 
20 | using namespace memory;
21 | 
22 | /**
23 |  * @brief Returns a small sample CSR matrix of size 4x4x4.
24 |  *
25 |  * @par Overview
26 |  *
27 |  * **Logical Matrix Representation**
28 |  * \code
29 |  * r/c  0 1 2 3
30 |  * 0 [ 0 0 0 0 ]
31 |  * 1 [ 5 8 0 0 ]
32 |  * 2 [ 0 0 3 0 ]
33 |  * 3 [ 0 6 0 0 ]
34 |  * \endcode
35 |  *
36 |  * **Logical Graph Representation**
37 |  * \code
38 |  * (i, j) [w]
39 |  * (1, 0) [5]
40 |  * (1, 1) [8]
41 |  * (2, 2) [3]
42 |  * (3, 1) [6]
43 |  * \endcode
44 |  *
45 |  * **CSR Matrix Representation**
46 |  * \code
47 |  * VALUES       = [ 5 8 3 6 ]
48 |  * COLUMN_INDEX = [ 0 1 2 1 ]
49 |  * ROW_OFFSETS  = [ 0 0 2 3 4 ]
50 |  * \endcode
51 |  *
52 |  * @tparam space Memory space of the CSR matrix.
53 |  * @tparam index_t Type of vertex.
54 |  * @tparam offset_t Type of edge.
55 |  * @tparam value_t Type of weight.
56 |  * @return csr_t<index_t, offset_t, value_t> CSR matrix.
57 |  */
58 | template <memory_space_t space = memory_space_t::device,
59 |           typename index_t = int,
60 |           typename offset_t = int,
61 |           typename value_t = float>
62 | csr_t<index_t, offset_t, value_t, space> csr() {
63 |   csr_t<index_t, offset_t, value_t, memory_space_t::host> matrix(4, 4, 4);
64 | 
65 |   // Row Offsets
66 |   matrix.offsets[0] = 0;
67 |   matrix.offsets[1] = 0;
68 |   matrix.offsets[2] = 2;
69 |   matrix.offsets[3] = 3;
70 |   matrix.offsets[4] = 4;
71 | 
72 |   // Column Indices
73 |   matrix.indices[0] = 0;
74 |   matrix.indices[1] = 1;
75 |   matrix.indices[2] = 2;
76 |   matrix.indices[3] = 1;
77 | 
78 |   // Non-zero values
79 |   matrix.values[0] = 5;
80 |   matrix.values[1] = 8;
81 |   matrix.values[2] = 3;
82 |   matrix.values[3] = 6;
83 | 
84 |   if (space == memory_space_t::host) {
85 |     return matrix;
86 |   } else {
87 |     csr_t<index_t, offset_t, value_t, memory_space_t::device> d_matrix(matrix);
88 |     return d_matrix;
89 |   }
90 | }
91 | 
92 | }  // namespace sample
93 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/search.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file search.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Simple search functionality.
 5 |  * @version 0.1
 6 |  * @date 2022-11-12
 7 |  *
 8 |  * @copyright Copyright (c) 2022
 9 |  *
10 |  */
11 | 
12 | #include <thrust/binary_search.h>
13 | #include <thrust/functional.h>
14 | #include <thrust/execution_policy.h>
15 | #include <thrust/distance.h>
16 | #include <thrust/iterator/counting_iterator.h>
17 | 
18 | #include <loops/container/coordinate.hxx>
19 | 
20 | #pragma once
21 | 
22 | namespace loops {
23 | namespace search {
24 | /**
25 |  * @brief Thrust based 2D binary-search for merge-path algorithm.
26 |  *
27 |  * @param diagonal Diagonal of the search.
28 |  * @param a First iterator.
29 |  * @param b Second iterator.
30 |  * @param a_len Length of the first iterator.
31 |  * @param b_len Length of the second iterator.
32 |  * @return A coordinate.
33 |  */
34 | template <typename offset_t, typename xit_t, typename yit_t>
35 | __device__ __forceinline__ auto _binary_search(const offset_t& diagonal,
36 |                                                const xit_t a,
37 |                                                const yit_t b,
38 |                                                const offset_t& a_len,
39 |                                                const offset_t& b_len) {
40 |   using coord_idx_t = unsigned int;
41 |   /// Diagonal search range (in x-coordinate space)
42 |   /// Note that the subtraction can result into a negative number, in which
43 |   /// case the max would result as 0. But if we use offset_t here, and it is
44 |   /// an unsigned type, we would get strange behavior, possible an unwanted
45 |   /// sign conversion that we do not want.
46 |   int x_min = max(int(diagonal) - int(b_len), int(0));
47 |   int x_max = min(int(diagonal), int(a_len));
48 | 
49 |   auto it = thrust::lower_bound(
50 |       thrust::seq,                                 // Sequential impl
51 |       thrust::counting_iterator<offset_t>(x_min),  // Start iterator @x_min
52 |       thrust::counting_iterator<offset_t>(x_max),  // End iterator @x_max
53 |       diagonal,                                    // ...
54 |       [=] __device__(const offset_t& idx, const offset_t& diagonal) {
55 |         return a[idx] <= b[diagonal - idx - 1];
56 |       });
57 | 
58 |   return coordinate_t<coord_idx_t>{static_cast<coord_idx_t>(min(*it, a_len)),
59 |                                    static_cast<coord_idx_t>(diagonal - *it)};
60 | }
61 | }  // namespace search
62 | }  // namespace loops


--------------------------------------------------------------------------------
/include/loops/util/timer.hxx:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * @file timer.hxx
 3 |  * @author Muhammad Osama (mosama@ucdavis.edu)
 4 |  * @brief Simple timer utility for device side code.
 5 |  * @version 0.1
 6 |  * @date 2020-10-05
 7 |  *
 8 |  * @copyright Copyright (c) 2020
 9 |  *
10 |  */
11 | 
12 | #pragma once
13 | 
14 | #include <cuda_runtime.h>
15 | 
16 | namespace loops {
17 | namespace util {
18 | 
19 | struct timer_t {
20 |   float time;
21 | 
22 |   timer_t() {
23 |     cudaEventCreate(&start_);
24 |     cudaEventCreate(&stop_);
25 |     cudaEventRecord(start_);
26 |   }
27 | 
28 |   ~timer_t() {
29 |     cudaEventDestroy(start_);
30 |     cudaEventDestroy(stop_);
31 |   }
32 | 
33 |   // Alias of each other, start the timer.
34 |   void begin() { cudaEventRecord(start_); }
35 |   void start() { this->begin(); }
36 | 
37 |   // Alias of each other, stop the timer.
38 |   float end() {
39 |     cudaEventRecord(stop_);
40 |     cudaEventSynchronize(stop_);
41 |     cudaEventElapsedTime(&time, start_, stop_);
42 | 
43 |     return milliseconds();
44 |   }
45 |   float stop() { return this->end(); }
46 | 
47 |   float seconds() { return time * 1e-3; }
48 |   float milliseconds() { return time; }
49 | 
50 |  private:
51 |   cudaEvent_t start_, stop_;
52 | };
53 | 
54 | }  // namespace util
55 | }  // namespace loops


--------------------------------------------------------------------------------
/plots/performance_evaluation.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "markdown",
  5 |    "id": "4a4cbe1e",
  6 |    "metadata": {},
  7 |    "source": [
  8 |     "# Performance Evaluation (Visualization)\n",
  9 |     "\n",
 10 |     "This python notebook attempts to recreate all the graphs found in our paper submission. Along with the notebook, we provide a `requirements.txt` file to install all the required python/visualization tools. Simply run `pip install -r requirements.txt` to install them, and then run the following kernels."
 11 |    ]
 12 |   },
 13 |   {
 14 |    "cell_type": "code",
 15 |    "execution_count": null,
 16 |    "id": "77e1801e",
 17 |    "metadata": {},
 18 |    "outputs": [],
 19 |    "source": [
 20 |     "import altair as alt\n",
 21 |     "import numpy as np\n",
 22 |     "import pandas as pd\n",
 23 |     "from vega_datasets import data\n",
 24 |     "from altair_saver import save\n",
 25 |     "import os\n",
 26 |     "import sys"
 27 |    ]
 28 |   },
 29 |   {
 30 |    "attachments": {},
 31 |    "cell_type": "markdown",
 32 |    "id": "d348dd26",
 33 |    "metadata": {},
 34 |    "source": [
 35 |     "## Path to generated data\n",
 36 |     "Currently points to: `<repo>/plots/data` with all the generated results in csv files."
 37 |    ]
 38 |   },
 39 |   {
 40 |    "cell_type": "code",
 41 |    "execution_count": null,
 42 |    "id": "7d77185a",
 43 |    "metadata": {},
 44 |    "outputs": [],
 45 |    "source": [
 46 |     "data_dir = os.path.abspath(\"data\")"
 47 |    ]
 48 |   },
 49 |   {
 50 |    "cell_type": "markdown",
 51 |    "id": "bc9433f3",
 52 |    "metadata": {},
 53 |    "source": [
 54 |     "## Figure 2. SpMV runtime comparison versus CUB\n",
 55 |     "Recreates the overhead analysis of SpMV merge-path versus CUB's SpMV implementation."
 56 |    ]
 57 |   },
 58 |   {
 59 |    "cell_type": "code",
 60 |    "execution_count": null,
 61 |    "id": "673c482d",
 62 |    "metadata": {},
 63 |    "outputs": [],
 64 |    "source": [
 65 |     "versus_cub = os.path.join(data_dir, \"versus_cub.csv\")\n",
 66 |     "df_overhead = pd.read_csv(versus_cub)\n",
 67 |     "\n",
 68 |     "df_overhead[\"nnzs\"] = df_overhead[\"nnzs\"].astype(int)\n",
 69 |     "df_overhead[\"elapsed\"] = df_overhead[\"elapsed\"].astype(float)"
 70 |    ]
 71 |   },
 72 |   {
 73 |    "cell_type": "code",
 74 |    "execution_count": null,
 75 |    "id": "b990f3c5",
 76 |    "metadata": {},
 77 |    "outputs": [],
 78 |    "source": [
 79 |     "alt.data_transformers.disable_max_rows()\n",
 80 |     "\n",
 81 |     "overhead = alt.Chart(df_overhead).mark_point(\n",
 82 |     "    opacity=0.6\n",
 83 |     ").encode(\n",
 84 |     "    x=alt.X(\n",
 85 |     "        \"nnzs:Q\", \n",
 86 |     "        scale=alt.Scale(type=\"log\",domain=(1,1000000000)),\n",
 87 |     "#         axis=alt.Axis(tickCount=10),\n",
 88 |     "        title=\"Number of Nonzeros\"\n",
 89 |     "    ),\n",
 90 |     "    y=alt.Y(\n",
 91 |     "        \"elapsed:Q\", \n",
 92 |     "        scale=alt.Scale(type=\"log\",domain=(0.001,100)),\n",
 93 |     "        axis=alt.Axis(grid=True), \n",
 94 |     "        title=\"Runtime (ms)\"\n",
 95 |     "    ),\n",
 96 |     "    color=alt.Color(\n",
 97 |     "        \"kernel\",\n",
 98 |     "        scale=alt.Scale(range=['#76b900','#DC143C']), # scheme='tableau10'),\n",
 99 |     "        legend=alt.Legend(title=\"Kernel\",orient='top-left',fillColor='#FFFFFF', strokeColor='#E0DCDC', padding=10)\n",
100 |     "    ),\n",
101 |     "#     facet=alt.Facet('kernel:O', title=None, columns=2),\n",
102 |     "    shape=alt.Shape(\n",
103 |     "        \"kernel\",\n",
104 |     "        scale=alt.Scale(range=['cross', 'triangle-right']),\n",
105 |     "        legend=alt.Legend(title=\"Kernel\",orient='top-left')\n",
106 |     "    ),\n",
107 |     ").properties(\n",
108 |     "    width=700,\n",
109 |     "    height=500\n",
110 |     ").configure_point(\n",
111 |     "    size=6\n",
112 |     ").interactive()\n",
113 |     "\n",
114 |     "overhead\n",
115 |     "# overhead.save(os.path.join(data_dir, \"merge_vs_cub.pdf\"))"
116 |    ]
117 |   },
118 |   {
119 |    "cell_type": "markdown",
120 |    "id": "dd94d5bd",
121 |    "metadata": {},
122 |    "source": [
123 |     "## Figure 3. Complete performance landscape\n",
124 |     "Recreates the complete performance landscape of our SpMV implementation versus cuSparse."
125 |    ]
126 |   },
127 |   {
128 |    "cell_type": "code",
129 |    "execution_count": null,
130 |    "id": "d09e010d",
131 |    "metadata": {},
132 |    "outputs": [],
133 |    "source": [
134 |     "cusparse = pd.read_csv(os.path.join(data_dir, \"cusparse.csv\"))\n",
135 |     "merge_path = pd.read_csv(os.path.join(data_dir, \"merge_path.csv\"))\n",
136 |     "group_mapped = pd.read_csv(os.path.join(data_dir, \"group_mapped.csv\"))\n",
137 |     "thread_mapped = pd.read_csv(os.path.join(data_dir, \"thread_mapped.csv\"))\n",
138 |     "\n",
139 |     "df_landscape = cusparse.append(merge_path.append(thread_mapped.append(group_mapped)))\n",
140 |     "df_landscape[\"nnzs\"] = df_landscape[\"nnzs\"].astype(int)\n",
141 |     "df_landscape[\"elapsed\"] = df_landscape[\"elapsed\"].astype(float)"
142 |    ]
143 |   },
144 |   {
145 |    "cell_type": "code",
146 |    "execution_count": null,
147 |    "id": "804f1553",
148 |    "metadata": {},
149 |    "outputs": [],
150 |    "source": [
151 |     "alt.data_transformers.disable_max_rows()\n",
152 |     "\n",
153 |     "landscape = alt.Chart(df_landscape).mark_point(\n",
154 |     "    opacity=0.7\n",
155 |     ").encode(\n",
156 |     "    x=alt.X(\n",
157 |     "        \"nnzs:Q\", \n",
158 |     "        scale=alt.Scale(type=\"log\",domain=(1,1000000000)),\n",
159 |     "#         axis=alt.Axis(tickCount=10),\n",
160 |     "        title=\"Number of Nonzeros\"\n",
161 |     "    ),\n",
162 |     "    y=alt.Y(\n",
163 |     "        \"elapsed:Q\", \n",
164 |     "        scale=alt.Scale(type=\"log\",domain=(0.001,1000)),\n",
165 |     "        axis=alt.Axis(grid=True), \n",
166 |     "        title=\"Runtime (ms)\"\n",
167 |     "    ),\n",
168 |     "    color=alt.Color(\n",
169 |     "        \"kernel:O\",\n",
170 |     "        scale=alt.Scale(range=['#76b900','#0047AB','#DC143C','#B983FF']), #scheme='tableau10'),\n",
171 |     "        legend=alt.Legend(title=\"Kernel\") #,orient='top-left')\n",
172 |     "    ),\n",
173 |     "    shape=alt.Shape(\n",
174 |     "        \"kernel:O\",\n",
175 |     "        legend=alt.Legend(title=\"Kernel\",orient='top-left',fillColor='#FFFFFF', strokeColor='#E0DCDC', padding=10)\n",
176 |     "    ),\n",
177 |     "    facet=alt.Facet('kernel:N', title=None, spacing={'row':0}, header=alt.Header(labelExpr=\"''\", title=None), columns=2),\n",
178 |     ").properties(\n",
179 |     "    width=400,\n",
180 |     "    height=400\n",
181 |     ").configure_point(\n",
182 |     "    size=7\n",
183 |     ").interactive()\n",
184 |     "\n",
185 |     "landscape\n",
186 |     "# landscape.save(os.path.join(data_dir, \"performance_landscape.pdf\"))"
187 |    ]
188 |   },
189 |   {
190 |    "cell_type": "markdown",
191 |    "id": "cdd4cf5a",
192 |    "metadata": {},
193 |    "source": [
194 |     "## Figure 4. Heuristics Selection (Speedup)\n",
195 |     "Shows our heuristics (presented in the paper) result and also the oracle heuristics."
196 |    ]
197 |   },
198 |   {
199 |    "cell_type": "code",
200 |    "execution_count": null,
201 |    "id": "021bbb08",
202 |    "metadata": {},
203 |    "outputs": [],
204 |    "source": [
205 |     "df_speed_up = pd.read_csv(os.path.join(data_dir, \"heuristics.csv\"))"
206 |    ]
207 |   },
208 |   {
209 |    "cell_type": "code",
210 |    "execution_count": null,
211 |    "id": "de208c19",
212 |    "metadata": {},
213 |    "outputs": [],
214 |    "source": [
215 |     "speed_up = alt.Chart(df_speed_up).mark_point(\n",
216 |     "    opacity=0.7\n",
217 |     ").encode(\n",
218 |     "    x=alt.X(\n",
219 |     "        \"nnzs:Q\", \n",
220 |     "        scale=alt.Scale(type=\"log\",domain=(1,1000000000)),\n",
221 |     "#         axis=alt.Axis(tickCount=10),\n",
222 |     "        title=\"Number of Nonzeros\"\n",
223 |     "    ),\n",
224 |     "    y=alt.Y(\n",
225 |     "        \"speed-up:Q\", \n",
226 |     "        scale=alt.Scale(type=\"log\",domain=(0.1,100)),\n",
227 |     "        axis=alt.Axis(grid=True), \n",
228 |     "        title=\"Speedup w.r.t cuSparse\"\n",
229 |     "    ),\n",
230 |     "    color=alt.Color(\n",
231 |     "        \"kernel\",\n",
232 |     "        scale=alt.Scale(range=['#0047AB','#DC143C','#B983FF']),\n",
233 |     "        legend=alt.Legend(title=\"Kernel\",orient='top-left',fillColor='#FFFFFF', strokeColor='#E0DCDC', padding=10)\n",
234 |     "    ),\n",
235 |     "    shape=alt.Shape(\n",
236 |     "        \"kernel\",\n",
237 |     "        scale=alt.Scale(range=['square', 'triangle', 'cross']),\n",
238 |     "        legend=alt.Legend(title=\"Kernel\") #,orient='top-left')\n",
239 |     "    ),\n",
240 |     ").properties(\n",
241 |     "    width=600,\n",
242 |     "    height=400\n",
243 |     ")\n",
244 |     "\n",
245 |     "speed_up_oracle = alt.Chart(df_speed_up).mark_point(\n",
246 |     "    opacity=0.7\n",
247 |     ").encode(\n",
248 |     "    x=alt.X(\n",
249 |     "        \"nnzs:Q\", \n",
250 |     "        scale=alt.Scale(type=\"log\",domain=(1,1000000000)),\n",
251 |     "#         axis=alt.Axis(tickCount=10),\n",
252 |     "        title=\"Number of Nonzeros\"\n",
253 |     "    ),\n",
254 |     "    y=alt.Y(\n",
255 |     "        \"oracle-speed-up:Q\", \n",
256 |     "        scale=alt.Scale(type=\"log\",domain=(0.1,100)),\n",
257 |     "        axis=alt.Axis(grid=True), \n",
258 |     "        title=\"Oracle Speedup w.r.t cuSparse\"\n",
259 |     "    ),\n",
260 |     "    color=alt.Color(\n",
261 |     "        \"oracle-speed-up-kernel\",\n",
262 |     "        scale=alt.Scale(range=['#0047AB','#DC143C','#B983FF']),\n",
263 |     "        legend=alt.Legend(title=\"Kernel\",orient='top-left',fillColor='#FFFFFF', strokeColor='#E0DCDC', padding=10)\n",
264 |     "    ),\n",
265 |     "    shape=alt.Shape(\n",
266 |     "        \"oracle-speed-up-kernel\",\n",
267 |     "        scale=alt.Scale(range=['square', 'triangle', 'cross']),\n",
268 |     "        legend=alt.Legend(title=\"Kernel\") #,orient='top-left')\n",
269 |     "    ),\n",
270 |     ").properties(\n",
271 |     "    width=600,\n",
272 |     "    height=400\n",
273 |     ")\n",
274 |     "\n",
275 |     "\n",
276 |     "\n",
277 |     "# speed_up\n",
278 |     "speed_up | speed_up_oracle\n",
279 |     "# speed_up.save(os.path.join(data_dir, \"speed_up.pdf\"))\n",
280 |     "# speed_up_oracle.save(os.path.join(data_dir, \"speed_up_oracle.pdf\"))"
281 |    ]
282 |   }
283 |  ],
284 |  "metadata": {
285 |   "kernelspec": {
286 |    "display_name": "Python 3 (ipykernel)",
287 |    "language": "python",
288 |    "name": "python3"
289 |   },
290 |   "language_info": {
291 |    "codemirror_mode": {
292 |     "name": "ipython",
293 |     "version": 3
294 |    },
295 |    "file_extension": ".py",
296 |    "mimetype": "text/x-python",
297 |    "name": "python",
298 |    "nbconvert_exporter": "python",
299 |    "pygments_lexer": "ipython3",
300 |    "version": "3.7.15"
301 |   }
302 |  },
303 |  "nbformat": 4,
304 |  "nbformat_minor": 5
305 | }
306 | 


--------------------------------------------------------------------------------
/plots/requirements.txt:
--------------------------------------------------------------------------------
1 | altair
2 | numpy
3 | pandas
4 | vega_datasets
5 | altair_saver
6 | 


--------------------------------------------------------------------------------
/scripts/README.md:
--------------------------------------------------------------------------------
1 | # Script for producing results.
2 | 
3 | >⚡Warning! A complete run can take multiple **days** and requires an NVIDIA GPU capable system.
4 | 
5 | Once the dataset is downloaded, modify the `run.sh` script to point to correct paths and simply execute it using `./run.sh`. Pre-collected results for algorithms within this library are included under `docs` directory. Note, `cusparse` is *not* part of our framework.


--------------------------------------------------------------------------------
/scripts/format.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | # By Gabriel Staples
 4 | # For documentation on this file, see my answer here:
 5 | # [my answer] How to call clang-format over a cpp project folder?:
 6 | # https://stackoverflow.com/a/65988393/4561887
 7 | 
 8 | # See my ans: https://stackoverflow.com/a/60157372/4561887
 9 | INPUT_PATH_TO_FILES=${1:-"./"}
10 | FULL_PATH_TO_SCRIPT="$(realpath "$INPUT_PATH_TO_FILES")"
11 | SCRIPT_DIRECTORY="$(dirname "$FULL_PATH_TO_SCRIPT")"
12 | 
13 | RETURN_CODE_SUCCESS=0
14 | RETURN_CODE_ERROR=1
15 | 
16 | # Find all files in SCRIPT_DIRECTORY with one of these extensions
17 | FILE_LIST="$(find "$SCRIPT_DIRECTORY" | grep -E ".*\.(hxx|cxx|cuh|cu|ino|cpp|c|h|hpp|hh)$")"
18 | # echo "\"$FILE_LIST\"" # debugging
19 | # split into an array; see my ans: https://stackoverflow.com/a/71575442/4561887
20 | # mapfile -t FILE_LIST_ARRAY <<< "$FILE_LIST"
21 | IFS=$'\n' read -r -d '' -a FILE_LIST_ARRAY <<< "$FILE_LIST"
22 | 
23 | num_files="${#FILE_LIST_ARRAY[@]}"
24 | echo -e "$num_files files found to format:"
25 | if [ "$num_files" -eq 0 ]; then
26 |     echo "Nothing to do."
27 |     exit $RETURN_CODE_SUCCESS
28 | fi
29 | 
30 | # print the list of all files
31 | for i in "${!FILE_LIST_ARRAY[@]}"; do
32 |     file="${FILE_LIST_ARRAY["$i"]}"
33 |     printf "  %2i: %s\n" $((i + 1)) "$file"
34 | done
35 | echo ""
36 | 
37 | format_files="false"
38 | # See: https://stackoverflow.com/a/226724/4561887
39 | case "$2" in 
40 |     [Yy]* ) format_files="true"
41 |     ;;
42 |     *)
43 | read -p "Do you wish to auto-format all of these files [y/N] " user_response
44 | case "$user_response" in
45 |     [Yy]* ) format_files="true"
46 | esac
47 | esac
48 | 
49 | if [ "$format_files" = "false" ]; then
50 |     echo "Aborting."
51 |     exit $RETURN_CODE_SUCCESS
52 | fi
53 | 
54 | # Format each file.
55 | clang-format --verbose -i --style=file "${FILE_LIST_ARRAY[@]}"


--------------------------------------------------------------------------------
/scripts/run.sh:
--------------------------------------------------------------------------------
 1 | #! /bin/bash
 2 | # Warning, a complete run can take days.
 3 | DATASET_DIR="$HOME/suitesparse"
 4 | DATASET_FILES_NAMES="../datasets/suitesparse.txt"
 5 | BINARY="../build/bin"
 6 | 
 7 | SPMV[0]="work_oriented"
 8 | SPMV[1]="group_mapped"
 9 | SPMV[2]="thread_mapped"
10 | SPMV[3]="original"
11 | SPMV[4]="merge_path"
12 | 
13 | EXE_PREFIX="loops.spmv"
14 | 
15 | for i in {0..4} 
16 | do
17 |   echo "kernel,dataset,rows,cols,nnzs,elapsed" >> spmv_log.${SPMV[$i]}.csv 
18 |   i=0
19 |   while read -r DATA
20 |   do
21 |     # Run only 10 datasets (EDIT this to run more.)
22 |     if [[ $i -eq 10 ]]
23 |     then
24 |         break
25 |     fi
26 |     ((i++))
27 |     echo $BINARY/$EXE_PREFIX.${SPMV[$i]} -m $DATASET_DIR/$DATA
28 |     timeout 60 $BINARY/$EXE_PREFIX.${SPMV[$i]} -m $DATASET_DIR/$DATA >> spmv_log.${SPMV[$i]}.csv 
29 |     done < "$DATASET_FILES_NAMES"
30 | done
31 | 


--------------------------------------------------------------------------------
/unittests/CMakeLists.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gunrock/loops/6169cf64d06e17b24b7a687fe0baf7ba2347002b/unittests/CMakeLists.txt


--------------------------------------------------------------------------------