├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── benchmarks ├── cuda │ ├── CMakeLists.txt │ ├── buffer.cu │ ├── buffer.h │ ├── ctrl.cc │ ├── ctrl.h │ ├── event.h │ ├── main.cu │ ├── queue.cu │ ├── queue.h │ ├── settings.cu │ └── settings.h └── latency │ ├── CMakeLists.txt │ ├── include │ ├── barrier.h │ ├── benchmark.h │ ├── buffer.h │ ├── ctrl.h │ ├── gpu.h │ ├── queue.h │ ├── settings.h │ ├── stats.h │ ├── transfer.h │ └── utility.h │ ├── smartio │ ├── buffer.cc │ ├── ctrl.cc │ ├── gpu.cc │ └── queue.cc │ └── src │ ├── barrier.cc │ ├── benchmark.cc │ ├── buffer.cc │ ├── ctrl.cc │ ├── gpu.cu │ ├── host.cu │ ├── main.cc │ ├── queue.cc │ ├── settings.cc │ └── stats.cc ├── examples ├── README.md ├── identify │ ├── CMakeLists.txt │ ├── common.c │ ├── common.h │ ├── module.c │ ├── smartio.c │ └── userspace.c ├── integrity │ ├── CMakeLists.txt │ ├── integrity.c │ ├── integrity.h │ ├── transfer.c │ └── util.c ├── read-blocks │ ├── CMakeLists.txt │ ├── args.c │ ├── args.h │ ├── module.c │ ├── read.c │ ├── read.h │ └── smartio.c └── rpc │ ├── CMakeLists.txt │ ├── rpc_dd.c │ ├── rpc_identify.c │ ├── rpc_server.c │ ├── segment.c │ ├── segment.h │ ├── util.c │ └── util.h ├── include ├── nvm_admin.h ├── nvm_aq.h ├── nvm_cmd.h ├── nvm_ctrl.h ├── nvm_dma.h ├── nvm_error.h ├── nvm_queue.h ├── nvm_rpc.h ├── nvm_types.h └── nvm_util.h ├── module ├── Makefile.in ├── ctrl.c ├── ctrl.h ├── list.c ├── list.h ├── map.c ├── map.h └── pci.c └── src ├── admin.c ├── admin.h ├── ctrl.c ├── ctrl.h ├── dis ├── device.c ├── device.h ├── dma.c ├── interrupt.c ├── interrupt.h ├── local.c ├── local.h ├── map.c ├── map.h └── rpc.c ├── dma.c ├── dma.h ├── dprintf.h ├── error.c ├── ioctl.h ├── queue.c ├── regs.h ├── rpc.c ├── rpc.h └── util.h /.gitignore: -------------------------------------------------------------------------------- 1 | # vim backup files 2 | .*.swp 3 | 4 | # Object files 5 | *.o 6 | *.ko 7 | 8 | # Library builds 9 | *.so 10 | 11 | # Intermediate files 12 | *.mod 13 | *.mod.c 14 | .*.cmd 15 | 16 | # Kernel symbols 17 | Module.symvers 18 | modules.order 19 | .tmp_versions/* 20 | 21 | # CUDA stuff 22 | *.i 23 | *.ii 24 | *.gpu 25 | *.ptx 26 | *.cubin 27 | *.fatbin 28 | 29 | # Profiling stuff 30 | *.prof 31 | *.nvprof 32 | 33 | # Ignore build directory 34 | build/* 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017, Jonas Markussen 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /benchmarks/cuda/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | project (libnvm-benchmarks) 3 | 4 | set (THREADS_PREFER_PTHREAD_FLAG ON) 5 | find_package (Threads REQUIRED) 6 | 7 | find_package (CUDA 8.0 REQUIRED) 8 | 9 | make_sisci_benchmark (cuda-benchmark-smartio cuda-bench "main.cu;buffer.cu;ctrl.cc;settings.cu;queue.cu") 10 | 11 | make_benchmark (cuda-benchmark-module cuda-bench "main.cu;buffer.cu;ctrl.cc;settings.cu;queue.cu") 12 | 13 | make_benchmark_choice (cuda-benchmark cuda-benchmark-smartio cuda-benchmark-module) 14 | -------------------------------------------------------------------------------- /benchmarks/cuda/buffer.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "buffer.h" 14 | 15 | using error = std::runtime_error; 16 | using std::string; 17 | 18 | 19 | 20 | static void getDeviceMemory(int device, void*& bufferPtr, void*& devicePtr, size_t size) 21 | { 22 | bufferPtr = nullptr; 23 | devicePtr = nullptr; 24 | 25 | cudaError_t err = cudaSetDevice(device); 26 | if (err != cudaSuccess) 27 | { 28 | throw error(string("Failed to set CUDA device: ") + cudaGetErrorString(err)); 29 | } 30 | 31 | err = cudaMalloc(&bufferPtr, size); 32 | if (err != cudaSuccess) 33 | { 34 | throw error(string("Failed to allocate device memory: ") + cudaGetErrorString(err)); 35 | } 36 | 37 | err = cudaMemset(bufferPtr, 0, size); 38 | if (err != cudaSuccess) 39 | { 40 | cudaFree(bufferPtr); 41 | throw error(string("Failed to clear device memory: ") + cudaGetErrorString(err)); 42 | } 43 | 44 | cudaPointerAttributes attrs; 45 | err = cudaPointerGetAttributes(&attrs, bufferPtr); 46 | if (err != cudaSuccess) 47 | { 48 | cudaFree(bufferPtr); 49 | throw error(string("Failed to get pointer attributes: ") + cudaGetErrorString(err)); 50 | } 51 | 52 | devicePtr = attrs.devicePointer; 53 | } 54 | 55 | 56 | 57 | static void getDeviceMemory(int device, void*& bufferPtr, size_t size) 58 | { 59 | void* notUsed = nullptr; 60 | getDeviceMemory(device, bufferPtr, notUsed, size); 61 | } 62 | 63 | 64 | 65 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size) 66 | { 67 | nvm_dma_t* dma = nullptr; 68 | void* buffer = nullptr; 69 | 70 | cudaError_t err = cudaHostAlloc(&buffer, size, cudaHostAllocDefault); 71 | if (err != cudaSuccess) 72 | { 73 | throw error(string("Failed to allocate host memory: ") + cudaGetErrorString(err)); 74 | } 75 | 76 | int status = nvm_dma_map_host(&dma, ctrl, buffer, size); 77 | if (!nvm_ok(status)) 78 | { 79 | cudaFreeHost(buffer); 80 | throw error(string("Failed to map host memory: ") + nvm_strerror(status)); 81 | } 82 | 83 | return DmaPtr(dma, [buffer](nvm_dma_t* dma) { 84 | nvm_dma_unmap(dma); 85 | cudaFreeHost(buffer); 86 | }); 87 | } 88 | 89 | 90 | 91 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, int cudaDevice) 92 | { 93 | if (cudaDevice < 0) 94 | { 95 | return createDma(ctrl, size); 96 | } 97 | 98 | nvm_dma_t* dma = nullptr; 99 | void* bufferPtr = nullptr; 100 | void* devicePtr = nullptr; 101 | 102 | getDeviceMemory(cudaDevice, bufferPtr, devicePtr, size); 103 | 104 | int status = nvm_dma_map_device(&dma, ctrl, (void *)NVM_PAGE_ALIGN((uintptr_t)devicePtr, 1UL << 16), NVM_ADDR_MASK(size, 1UL << 16)); 105 | if (!nvm_ok(status)) 106 | { 107 | cudaFree(bufferPtr); 108 | throw error(string("Failed to map device memory: ") + nvm_strerror(status)); 109 | } 110 | 111 | //dma->vaddr = bufferPtr; 112 | 113 | return DmaPtr(dma, [bufferPtr](nvm_dma_t* dma) { 114 | nvm_dma_unmap(dma); 115 | cudaFree(bufferPtr); 116 | }); 117 | } 118 | 119 | 120 | 121 | BufferPtr createBuffer(size_t size) 122 | { 123 | void* buffer = nullptr; 124 | 125 | cudaError_t err = cudaHostAlloc(&buffer, size, cudaHostAllocDefault); 126 | if (err != cudaSuccess) 127 | { 128 | throw error(string("Failed to allocate host memory: ") + cudaGetErrorString(err)); 129 | } 130 | 131 | return BufferPtr(buffer, [](void* ptr) { cudaFreeHost(ptr); }); 132 | } 133 | 134 | 135 | 136 | BufferPtr createBuffer(size_t size, int cudaDevice) 137 | { 138 | if (cudaDevice < 0) 139 | { 140 | return createBuffer(size); 141 | } 142 | 143 | void* bufferPtr = nullptr; 144 | 145 | getDeviceMemory(cudaDevice, bufferPtr, size); 146 | 147 | return BufferPtr(bufferPtr, [](void* ptr) { cudaFree(ptr); }); 148 | } 149 | 150 | 151 | 152 | #ifdef __DIS_CLUSTER__ 153 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, uint32_t adapter, uint32_t id) 154 | { 155 | nvm_dma_t* dma = nullptr; 156 | 157 | int status = nvm_dis_dma_create(&dma, ctrl, adapter, id, size); 158 | if (!nvm_ok(status)) 159 | { 160 | throw error(string("Failed to create local segment: ") + nvm_strerror(status)); 161 | } 162 | 163 | return DmaPtr(dma, nvm_dma_unmap); 164 | } 165 | #else 166 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, uint32_t, uint32_t) 167 | { 168 | return createDma(ctrl, size); 169 | } 170 | #endif 171 | 172 | 173 | #ifdef __DIS_CLUSTER__ 174 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, int cudaDevice, uint32_t adapter, uint32_t id) 175 | { 176 | if (cudaDevice < 0) 177 | { 178 | return createDma(ctrl, size, adapter, id); 179 | } 180 | 181 | nvm_dma_t* dma = nullptr; 182 | void* bufferPtr = nullptr; 183 | void* devicePtr = nullptr; 184 | 185 | getDeviceMemory(cudaDevice, bufferPtr, devicePtr, size); 186 | 187 | int status = nvm_dis_dma_map_device(&dma, ctrl, adapter, id, devicePtr, size); 188 | if (!nvm_ok(status)) 189 | { 190 | throw error(string("Failed to create local segment: ") + nvm_strerror(status)); 191 | } 192 | 193 | dma->vaddr = devicePtr; 194 | 195 | return DmaPtr(dma, nvm_dma_unmap); 196 | } 197 | #else 198 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, int cudaDevice, uint32_t, uint32_t) 199 | { 200 | return createDma(ctrl, size, cudaDevice); 201 | } 202 | #endif 203 | 204 | 205 | #ifdef __DIS_CLUSTER__ 206 | DmaPtr createRemoteDma(const nvm_ctrl_t* ctrl, size_t size, uint32_t adapter, uint32_t segno) 207 | { 208 | nvm_dma_t* dma = nullptr; 209 | 210 | int status = nvm_dis_dma_connect(&dma, ctrl, adapter, segno, size, true); 211 | if (!nvm_ok(status)) 212 | { 213 | throw error(string("Failed to connect to segment: ") + nvm_strerror(status)); 214 | } 215 | 216 | return DmaPtr(dma, nvm_dma_unmap); 217 | } 218 | #endif 219 | 220 | -------------------------------------------------------------------------------- /benchmarks/cuda/buffer.h: -------------------------------------------------------------------------------- 1 | #ifndef __BENCHMARK_BUFFER_H__ 2 | #define __BENCHMARK_BUFFER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | 11 | typedef std::shared_ptr DmaPtr; 12 | 13 | typedef std::shared_ptr BufferPtr; 14 | 15 | 16 | 17 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size); 18 | 19 | 20 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, int cudaDevice); 21 | 22 | 23 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, uint32_t adapter, uint32_t id); 24 | 25 | 26 | DmaPtr createDma(const nvm_ctrl_t* ctrl, size_t size, int cudaDevice, uint32_t adapter, uint32_t id); 27 | 28 | 29 | BufferPtr createBuffer(size_t size); 30 | 31 | 32 | BufferPtr createBuffer(size_t size, int cudaDevice); 33 | 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /benchmarks/cuda/ctrl.cc: -------------------------------------------------------------------------------- 1 | #include "ctrl.h" 2 | #include "buffer.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | using error = std::runtime_error; 23 | using std::string; 24 | 25 | 26 | static void initializeController(struct Controller& ctrl, uint32_t ns_id) 27 | { 28 | // Create admin queue reference 29 | int status = nvm_aq_create(&ctrl.aq_ref, ctrl.ctrl, ctrl.aq_mem.get()); 30 | if (!nvm_ok(status)) 31 | { 32 | throw error(string("Failed to reset controller: ") + nvm_strerror(status)); 33 | } 34 | 35 | // Identify controller 36 | status = nvm_admin_ctrl_info(ctrl.aq_ref, &ctrl.info, NVM_DMA_OFFSET(ctrl.aq_mem, 2), ctrl.aq_mem->ioaddrs[2]); 37 | if (!nvm_ok(status)) 38 | { 39 | throw error(nvm_strerror(status)); 40 | } 41 | 42 | // Identify namespace 43 | status = nvm_admin_ns_info(ctrl.aq_ref, &ctrl.ns, ns_id, NVM_DMA_OFFSET(ctrl.aq_mem, 2), ctrl.aq_mem->ioaddrs[2]); 44 | if (!nvm_ok(status)) 45 | { 46 | throw error(nvm_strerror(status)); 47 | } 48 | 49 | // Get number of queues 50 | status = nvm_admin_get_num_queues(ctrl.aq_ref, &ctrl.n_cqs, &ctrl.n_sqs); 51 | if (!nvm_ok(status)) 52 | { 53 | throw error(nvm_strerror(status)); 54 | } 55 | } 56 | 57 | 58 | 59 | #ifdef __DIS_CLUSTER__ 60 | Controller::Controller(uint64_t ctrl_id, uint32_t ns_id, uint32_t adapter, uint32_t segment_id) 61 | : ctrl(nullptr) 62 | , aq_ref(nullptr) 63 | { 64 | // Get controller reference 65 | int status = nvm_dis_ctrl_init(&ctrl, ctrl_id, adapter); 66 | if (!nvm_ok(status)) 67 | { 68 | throw error(string("Failed to get controller reference: ") + nvm_strerror(status)); 69 | } 70 | 71 | // Create admin queue memory 72 | aq_mem = createDma(ctrl, ctrl->page_size * 3, adapter, segment_id); 73 | 74 | initializeController(*this, ns_id); 75 | } 76 | #endif 77 | 78 | 79 | 80 | Controller::Controller(const char* path, uint32_t ns_id) 81 | : ctrl(nullptr) 82 | , aq_ref(nullptr) 83 | { 84 | int fd = open(path, O_RDWR | O_NONBLOCK); 85 | if (fd < 0) 86 | { 87 | throw error(string("Failed to open descriptor: ") + strerror(errno)); 88 | } 89 | 90 | // Get controller reference 91 | int status = nvm_ctrl_init(&ctrl, fd); 92 | if (!nvm_ok(status)) 93 | { 94 | throw error(string("Failed to get controller reference: ") + nvm_strerror(status)); 95 | } 96 | 97 | // Create admin queue memory 98 | aq_mem = createDma(ctrl, ctrl->page_size * 3); 99 | 100 | initializeController(*this, ns_id); 101 | 102 | close(fd); 103 | } 104 | 105 | 106 | 107 | Controller::~Controller() 108 | { 109 | nvm_aq_destroy(aq_ref); 110 | nvm_ctrl_free(ctrl); 111 | } 112 | 113 | 114 | 115 | void Controller::reserveQueues() 116 | { 117 | reserveQueues(n_sqs, n_cqs); 118 | } 119 | 120 | 121 | 122 | void Controller::reserveQueues(uint16_t numSubmissionQueues) 123 | { 124 | reserveQueues(numSubmissionQueues, n_cqs); 125 | } 126 | 127 | 128 | 129 | void Controller::reserveQueues(uint16_t numSubs, uint16_t numCpls) 130 | { 131 | int status = nvm_admin_request_num_queues(aq_ref, &numSubs, &numCpls); 132 | if (!nvm_ok(status)) 133 | { 134 | throw error(string("Failed to reserve queues: ") + nvm_strerror(status)); 135 | } 136 | 137 | n_sqs = numSubs; 138 | n_cqs = numCpls; 139 | } 140 | 141 | -------------------------------------------------------------------------------- /benchmarks/cuda/ctrl.h: -------------------------------------------------------------------------------- 1 | #ifndef __BENCHMARK_CTRL_H__ 2 | #define __BENCHMARK_CTRL_H__ 3 | 4 | #include 5 | #include 6 | #include "buffer.h" 7 | 8 | 9 | 10 | struct Controller 11 | { 12 | nvm_ctrl_t* ctrl; 13 | nvm_aq_ref aq_ref; 14 | DmaPtr aq_mem; 15 | struct nvm_ctrl_info info; 16 | struct nvm_ns_info ns; 17 | uint16_t n_sqs; 18 | uint16_t n_cqs; 19 | 20 | #ifdef __DIS_CLUSTER__ 21 | Controller(uint64_t controllerId, uint32_t nvmNamespace, uint32_t adapter, uint32_t segmentId); 22 | #endif 23 | 24 | Controller(const char* path, uint32_t nvmNamespace); 25 | 26 | void reserveQueues(); 27 | 28 | void reserveQueues(uint16_t numSubmissionQueues); 29 | 30 | void reserveQueues(uint16_t numSubmissionQueues, uint16_t numCompletionQueues); 31 | 32 | ~Controller(); 33 | }; 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /benchmarks/cuda/event.h: -------------------------------------------------------------------------------- 1 | #ifndef __BENCHMARK_EVENT_H__ 2 | #define __BENCHMARK_EVENT_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | struct Event 10 | { 11 | cudaEvent_t event; 12 | 13 | Event(cudaStream_t stream = 0) 14 | { 15 | auto err = cudaEventCreateWithFlags(&event, cudaEventDefault); 16 | if (err != cudaSuccess) 17 | { 18 | throw std::runtime_error(std::string("Failed to create event: ") + cudaGetErrorString(err)); 19 | } 20 | 21 | err = cudaEventRecord(event, stream); 22 | if (err != cudaSuccess) 23 | { 24 | throw std::runtime_error(std::string("Failed to record event on stream: ") + cudaGetErrorString(err)); 25 | } 26 | 27 | } 28 | 29 | 30 | ~Event() 31 | { 32 | cudaEventDestroy(event); 33 | } 34 | 35 | 36 | double operator-(const Event& other) const 37 | { 38 | float msecs = 0; 39 | auto err = cudaEventElapsedTime(&msecs, other.event, event); 40 | if (err != cudaSuccess) 41 | { 42 | throw std::runtime_error(std::string("Could not calculate elapsed time: ") + cudaGetErrorString(err)); 43 | } 44 | 45 | return ((double) msecs) * 1e3; 46 | } 47 | }; 48 | 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /benchmarks/cuda/queue.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include "settings.h" 3 | #include "ctrl.h" 4 | #include "buffer.h" 5 | #include "queue.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | using error = std::runtime_error; 15 | using std::string; 16 | 17 | 18 | 19 | __host__ DmaPtr prepareQueuePair(QueuePair& qp, const Controller& ctrl, const Settings& settings) 20 | { 21 | size_t queueMemSize = ctrl.info.page_size * 2; 22 | size_t prpListSize = ctrl.info.page_size * settings.numThreads * (settings.doubleBuffered + 1); 23 | 24 | // qmem->vaddr will be already a device pointer after the following call 25 | auto qmem = createDma(ctrl.ctrl, queueMemSize + prpListSize + (1UL << 16), settings.cudaDevice, settings.adapter, settings.segmentId); 26 | 27 | // Set members 28 | qp.pageSize = ctrl.info.page_size; 29 | qp.blockSize = ctrl.ns.lba_data_size; 30 | qp.nvmNamespace = ctrl.ns.ns_id; 31 | qp.pagesPerChunk = settings.numPages; 32 | qp.doubleBuffered = settings.doubleBuffered; 33 | 34 | qp.prpList = NVM_DMA_OFFSET(qmem, 2); 35 | qp.prpListIoAddr = qmem->ioaddrs[2]; 36 | 37 | // Create completion queue 38 | int status = nvm_admin_cq_create(ctrl.aq_ref, &qp.cq, 1, qmem->vaddr, qmem->ioaddrs[0]); 39 | if (!nvm_ok(status)) 40 | { 41 | throw error(string("Failed to create completion queue: ") + nvm_strerror(status)); 42 | } 43 | 44 | // Get a valid device pointer for CQ doorbell 45 | void* devicePtr = nullptr; 46 | cudaError_t err = cudaHostGetDevicePointer(&devicePtr, (void*) qp.cq.db, 0); 47 | if (err != cudaSuccess) 48 | { 49 | throw error(string("Failed to get device pointer") + cudaGetErrorString(err)); 50 | } 51 | qp.cq.db = (volatile uint32_t*) devicePtr; 52 | 53 | // Create submission queue 54 | status = nvm_admin_sq_create(ctrl.aq_ref, &qp.sq, &qp.cq, 1, NVM_DMA_OFFSET(qmem, 1), qmem->ioaddrs[1]); 55 | if (!nvm_ok(status)) 56 | { 57 | throw error(string("Failed to create submission queue: ") + nvm_strerror(status)); 58 | } 59 | 60 | // Get a valid device pointer for SQ doorbell 61 | err = cudaHostGetDevicePointer(&devicePtr, (void*) qp.sq.db, 0); 62 | if (err != cudaSuccess) 63 | { 64 | throw error(string("Failed to get device pointer") + cudaGetErrorString(err)); 65 | } 66 | qp.sq.db = (volatile uint32_t*) devicePtr; 67 | 68 | return qmem; 69 | } 70 | 71 | -------------------------------------------------------------------------------- /benchmarks/cuda/queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __BENCHMARK_QUEUEPAIR_H__ 2 | #define __BENCHMARK_QUEUEPAIR_H__ 3 | 4 | #include 5 | #include 6 | #include "buffer.h" 7 | #include "settings.h" 8 | #include "ctrl.h" 9 | 10 | 11 | struct __align__(64) QueuePair 12 | { 13 | uint32_t pageSize; 14 | uint32_t blockSize; 15 | uint32_t nvmNamespace; 16 | uint32_t pagesPerChunk; 17 | bool doubleBuffered; 18 | void* prpList; 19 | uint64_t prpListIoAddr; 20 | nvm_queue_t sq; 21 | nvm_queue_t cq; 22 | }; 23 | 24 | 25 | __host__ DmaPtr prepareQueuePair(QueuePair& qp, const Controller& ctrl, const Settings& settings); 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /benchmarks/cuda/settings.h: -------------------------------------------------------------------------------- 1 | #ifndef __SETTINGS_H__ 2 | #define __SETTINGS_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | struct Settings 10 | { 11 | uint32_t cudaDevice; 12 | uint64_t cudaDeviceId; 13 | const char* blockDevicePath; 14 | const char* controllerPath; 15 | uint64_t controllerId; 16 | uint32_t adapter; 17 | uint32_t segmentId; 18 | uint32_t nvmNamespace; 19 | bool doubleBuffered; 20 | size_t numChunks; 21 | size_t numPages; 22 | size_t startBlock; 23 | bool stats; 24 | const char* output; 25 | size_t numThreads; 26 | uint32_t domain; 27 | uint32_t bus; 28 | uint32_t devfn; 29 | 30 | Settings(); 31 | void parseArguments(int argc, char** argv); 32 | 33 | static std::string usageString(const std::string& name); 34 | 35 | std::string getDeviceBDF() const; 36 | }; 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /benchmarks/latency/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | project (libnvm-benchmarks) 3 | 4 | set (THREADS_PREFER_PTHREAD_FLAG ON) 5 | find_package (Threads REQUIRED) 6 | 7 | find_package (CUDA 8.0 REQUIRED) 8 | 9 | # CUDA targets doesn't support target_include_directories 10 | #include_directories ("${CMAKE_CURRENT_SOURCE_DIR}/include" ) 11 | include_directories ("include") 12 | 13 | file (GLOB module_files "src/*.cu" "src/*.cc") 14 | file (GLOB smartio_files "src/*.cu" "src/*.cc" "smartio/*.cu" "smartio/*.cc") 15 | 16 | make_benchmark (latency-benchmark-module latency-bench "${module_files}") 17 | 18 | make_sisci_benchmark (latency-benchmark-smartio latency-bench "${smartio_files}") 19 | 20 | make_benchmark_choice (latency-benchmark latency-benchmark-smartio latency-benchmark-module) 21 | -------------------------------------------------------------------------------- /benchmarks/latency/include/barrier.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_BARRIER_H__ 2 | #define __LATENCY_BENCHMARK_BARRIER_H__ 3 | 4 | 5 | #include 6 | 7 | 8 | class Barrier 9 | { 10 | public: 11 | Barrier(int numThreads); 12 | 13 | void wait(); 14 | 15 | private: 16 | const int reset; 17 | int arrived; 18 | int flag; 19 | std::mutex mtx; 20 | }; 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /benchmarks/latency/include/benchmark.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_BENCHMARK_H__ 2 | #define __LATENCY_BENCHMARK_BENCHMARK_H__ 3 | 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "transfer.h" 10 | #include "settings.h" 11 | 12 | 13 | /* 14 | * Create a convenience type for representing microseconds. 15 | */ 16 | typedef std::chrono::duration usec_t; 17 | 18 | 19 | 20 | /* 21 | * Record time for a number of commands, that is, submitting a bunch of 22 | * commands and waiting for their completions. On a per-command basis, 23 | * this would be the equivalent of number of IO operations per second. 24 | */ 25 | struct Event 26 | { 27 | size_t commands; // Number of commands 28 | size_t blocks; // Number of blocks 29 | usec_t time; // Number of microseconds 30 | 31 | Event(size_t ncmds, size_t nblks, usec_t usecs) 32 | : commands(ncmds), blocks(nblks), time(usecs) 33 | { } 34 | 35 | 36 | /* 37 | * Calcluate number of IO operations per second (IOPS). 38 | * This will only be an estimate, unless commands == 1. 39 | */ 40 | double estimateIops() const 41 | { 42 | return 1e6 / averageLatencyPerCommand(); 43 | } 44 | 45 | 46 | /* 47 | * Estimate number of IO operations per second (IOPS) 48 | * adjusted for increased transfer sizes. 49 | */ 50 | double adjustedIops(size_t blocksPerPrp = 1) const 51 | { 52 | return 1e6 / (averageLatencyPerBlock() * blocksPerPrp); 53 | } 54 | 55 | 56 | /* 57 | * Calculate average number of microseconds per block. 58 | */ 59 | double averageLatencyPerBlock() const 60 | { 61 | return time.count() / blocks; 62 | } 63 | 64 | 65 | /* 66 | * Calculate average number of microseconds per command. 67 | */ 68 | double averageLatencyPerCommand() const 69 | { 70 | return (time.count() / commands); 71 | } 72 | 73 | 74 | size_t transferSize(size_t blockSize) const 75 | { 76 | return blocks * blockSize; 77 | } 78 | 79 | 80 | double bandwidth(size_t blockSize) const 81 | { 82 | return (blocks * blockSize) / time.count(); 83 | } 84 | }; 85 | 86 | 87 | 88 | /* 89 | * Convenience types for a series of recorded events. 90 | */ 91 | typedef std::vector EventList; 92 | typedef std::shared_ptr EventListPtr; 93 | typedef std::map EventMap; 94 | 95 | 96 | 97 | void benchmark(EventMap& times, const TransferMap& transfers, const Settings& settings, bool write); 98 | 99 | #endif /* __LATENCY_BENCHMARK_BENCHMARK_H__ */ 100 | -------------------------------------------------------------------------------- /benchmarks/latency/include/buffer.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_BUFFER_H__ 2 | #define __LATENCY_BENCHMARK_BUFFER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | /* Forward declaration */ 10 | struct Ctrl; 11 | 12 | 13 | 14 | /* 15 | * Convenience type for DMA handles. 16 | * Will automaticall clean itself up once not used anymore. 17 | */ 18 | typedef std::shared_ptr DmaPtr; 19 | 20 | 21 | 22 | /* 23 | * Convenience type for a (non-mapped) memory buffer. 24 | */ 25 | typedef std::shared_ptr MemoryPtr; 26 | 27 | 28 | 29 | /* 30 | * Memory buffer type. 31 | */ 32 | struct MemoryBuffer 33 | { 34 | public: 35 | const size_t size; 36 | const DmaPtr buffer; 37 | 38 | 39 | MemoryBuffer(const Ctrl& controller, size_t size); 40 | 41 | 42 | MemoryBuffer(const Ctrl& controller, size_t size, uint32_t segmentId); 43 | 44 | 45 | virtual ~MemoryBuffer() {} 46 | 47 | 48 | /* 49 | * Copy memory from ptr and store in buffer. 50 | */ 51 | virtual size_t load(const void* ptr, size_t size, size_t offset = 0); 52 | 53 | 54 | 55 | /* 56 | * Copy memory from buffer and store in ptr. 57 | */ 58 | virtual size_t save(void* ptr, size_t size, size_t offset = 0) const; 59 | 60 | 61 | /* 62 | * Set memory to zero. 63 | */ 64 | virtual void clear(); 65 | 66 | protected: 67 | MemoryBuffer(const DmaPtr& buffer, size_t size) 68 | : size(size), buffer(buffer) {} 69 | }; 70 | 71 | 72 | 73 | /* Convenience type */ 74 | typedef std::shared_ptr MemoryBufferPtr; 75 | 76 | 77 | 78 | /* 79 | * Allocate memory in local RAM mapped for the NVMe controller. 80 | */ 81 | DmaPtr allocateBuffer(const nvm_ctrl_t* controller, size_t size); 82 | 83 | 84 | 85 | /* 86 | * Allocate memory in local RAM mapped for the NVMe controller. 87 | */ 88 | DmaPtr allocateBuffer(const Ctrl& controller, size_t size); 89 | 90 | 91 | 92 | /* 93 | * Allocate memory in local RAM. 94 | * This memory is not mapped for the NVMe controller. 95 | */ 96 | MemoryPtr allocateHostMemory(size_t size); 97 | 98 | 99 | 100 | /* 101 | * Allocate memory in local RAM mapped for the NVMe controller. 102 | */ 103 | DmaPtr allocateBuffer(const nvm_ctrl_t* controller, size_t size, uint32_t adapter, uint32_t segmentId); 104 | 105 | 106 | 107 | /* 108 | * Allocate memory in local RAM mapped for the NVMe controller. 109 | */ 110 | DmaPtr allocateBuffer(const Ctrl& controller, size_t size, uint32_t adapter, uint32_t segmentId); 111 | 112 | 113 | /* 114 | * Allocate memory in local RAM mapped for the NVMe controller. 115 | */ 116 | DmaPtr allocateBuffer(const Ctrl& controller, size_t size, uint32_t segmentId); 117 | 118 | 119 | 120 | #ifdef __DIS_CLUSTER__ 121 | DmaPtr connectBuffer(const nvm_ctrl_t* controller, size_t size, uint32_t adapter, uint32_t remoteSegmentId); 122 | DmaPtr connectBuffer(const Ctrl& controller, size_t size, uint32_t adapter, uint32_t remoteSegmentId); 123 | #endif 124 | 125 | 126 | #endif /* __LATENCY_BENCHMARK_BUFFER_H__ */ 127 | -------------------------------------------------------------------------------- /benchmarks/latency/include/ctrl.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_CTRL_H__ 2 | #define __LATENCY_BENCHMARK_CTRL_H__ 3 | 4 | #include 5 | #include "ctrl.h" 6 | #include "buffer.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | /* 14 | * NVMe controller reference wrapper. 15 | */ 16 | struct Ctrl final 17 | { 18 | private: 19 | mutable nvm_ctrl_t* writeHandle; 20 | mutable DmaPtr adminQueueMemory; 21 | mutable nvm_dma_t* adminQueues; 22 | mutable nvm_aq_ref adminRef; 23 | 24 | public: 25 | const nvm_ctrl_t* handle; /* Read only-handle */ 26 | const uint64_t fdid; /* SmartIO fabric device identifier */ 27 | const uint32_t adapter; /* SmartIO NTB adapter */ 28 | uint32_t namespaceId; /* NVM namespace identifier */ 29 | uint64_t namespaceSize; /* Size of namespace in blocks */ 30 | size_t pageSize; /* Controller page size */ 31 | size_t blockSize; /* Block size */ 32 | size_t chunkSize; /* Maximum data transfer size */ 33 | uint16_t numQueues; /* Maximum number of queue pairs */ 34 | size_t maxEntries; 35 | 36 | ~Ctrl() noexcept; 37 | 38 | nvm_aq_ref adminReference() const 39 | { 40 | return adminRef; 41 | } 42 | 43 | private: 44 | friend struct CtrlManager; 45 | Ctrl(const Ctrl&) = delete; 46 | Ctrl(Ctrl&&) noexcept = delete; 47 | Ctrl& operator=(const Ctrl&) = delete; 48 | Ctrl& operator=(const Ctrl&&) noexcept = delete; 49 | Ctrl(uint64_t fdid, uint32_t adapter, nvm_ctrl_t* controller, DmaPtr memory, nvm_dma_t* dma, nvm_aq_ref ref, uint32_t ns); 50 | }; 51 | 52 | 53 | 54 | /* 55 | * Declare controller reference pointer type. 56 | */ 57 | typedef std::shared_ptr CtrlPtr; 58 | 59 | 60 | 61 | /* 62 | * NVMe controller reference manager. 63 | */ 64 | struct CtrlManager final 65 | { 66 | public: 67 | CtrlManager(const std::string& devicePath, uint32_t nvmNamespace); 68 | 69 | 70 | CtrlManager(uint64_t fdid, uint32_t adapter, uint32_t segmentId, bool adminManager, uint32_t nvmNamespace); 71 | 72 | 73 | ~CtrlManager(); 74 | 75 | 76 | const Ctrl& getController() const 77 | { 78 | return *controller; 79 | } 80 | 81 | const CtrlPtr& getControllerPtr() const 82 | { 83 | return controller; 84 | } 85 | 86 | private: 87 | CtrlPtr controller; 88 | int fileDescriptor; 89 | }; 90 | 91 | 92 | /* Convenience type */ 93 | typedef std::shared_ptr CtrlManagerPtr; 94 | 95 | #endif /* __LATENCY_BENCHMARK_CTRL_H__ */ 96 | -------------------------------------------------------------------------------- /benchmarks/latency/include/gpu.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_GPU_H__ 2 | #define __LATENCY_BENCHMARK_GPU_H__ 3 | 4 | #include "buffer.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | /* Forward declaration */ 11 | struct Device; 12 | struct Ctrl; 13 | 14 | 15 | /* 16 | * Wrapper for GPU functionality, such as allocating and mapping 17 | * memory buffers for an NVMe controller. 18 | */ 19 | struct Gpu final 20 | { 21 | private: 22 | std::shared_ptr device; /* SmartIO device wrapper */ 23 | 24 | public: 25 | const int cudaDevice; /* CUDA device ID */ 26 | const uint64_t fdid; /* Fabric device ID (used by SmartIO) */ 27 | const uint32_t adapter; /* NTB adapter number where GPU can be reached */ 28 | 29 | 30 | /* Get a string representation of the BDF */ 31 | std::string deviceBdf() const; 32 | 33 | 34 | /* Get canonical GPU name */ 35 | std::string deviceName() const; 36 | 37 | 38 | /* 39 | * Set current GPU for CUDA context. 40 | */ 41 | void setDevice() const; 42 | 43 | 44 | /* 45 | * Get number of CUDA devices 46 | */ 47 | static int deviceCount(); 48 | 49 | 50 | /* 51 | * Initialize instance using specified CUDA device. 52 | */ 53 | Gpu(int cudaDevice); 54 | 55 | 56 | /* 57 | * Initialize instance using specified fabric device ID. 58 | */ 59 | Gpu(uint64_t fdid, uint32_t adapter); 60 | 61 | 62 | #ifdef __DIS_CLUSTER__ 63 | /* 64 | * Allocate buffer in GPU memory and map it for the NVMe controller, 65 | * as well as the local CPU. 66 | * This is only possible for remote GPUs using SmartIO. 67 | */ 68 | DmaPtr allocateBufferAndMap(const Ctrl& controller, size_t size, uint32_t segmentId) const; 69 | #endif 70 | 71 | 72 | /* 73 | * Allocate buffer in GPU memory. 74 | * This memory is not mapped for the NVMe controller. 75 | */ 76 | MemoryPtr allocateMemory(size_t size) const; 77 | 78 | 79 | /* 80 | * Allocate buffer in GPU memory and map it for the NVMe controller. 81 | */ 82 | DmaPtr allocateBuffer(const Ctrl& controller, size_t size) const; 83 | 84 | 85 | /* 86 | * Allocate buffer in GPU memory and map it for the NVMe controller using SamrtIO. 87 | */ 88 | DmaPtr allocateBuffer(const Ctrl& controller, size_t size, uint32_t segmentId) const; 89 | 90 | 91 | 92 | /* 93 | * Helper function to look up a device from its BDF. 94 | */ 95 | static int findDevice(int domain, int bus, int devfn); 96 | 97 | 98 | private: 99 | /* 100 | * Helper function to look up a device pointer from a CUDA pointer. 101 | */ 102 | static void* getDevicePointer(const MemoryPtr&); 103 | }; 104 | 105 | 106 | 107 | 108 | /* 109 | * Convenience type. 110 | */ 111 | typedef std::shared_ptr GpuPtr; 112 | 113 | 114 | 115 | 116 | struct GpuBuffer : public MemoryBuffer 117 | { 118 | public: 119 | const GpuPtr gpu; 120 | 121 | GpuBuffer(const GpuPtr& gpu, const Ctrl& controller, size_t size); 122 | 123 | 124 | GpuBuffer(const GpuPtr& gpu, const Ctrl& controller, size_t size, uint32_t segmentId); 125 | 126 | 127 | size_t load(const void* ptr, size_t size, size_t offset = 0) override; 128 | 129 | 130 | size_t save(void* ptr, size_t size, size_t offset = 0) const override; 131 | 132 | 133 | void clear() override; 134 | }; 135 | 136 | 137 | #endif /* __LATENCY_BENCHMARK_GPU_H__ */ 138 | -------------------------------------------------------------------------------- /benchmarks/latency/include/queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_QUEUE_H__ 2 | #define __LATENCY_BENCHMARK_QUEUE_H__ 3 | 4 | #include 5 | #include "ctrl.h" 6 | #include "buffer.h" 7 | #include "gpu.h" 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | 15 | enum QueueLocation : int 16 | { 17 | REMOTE, LOCAL, GPU 18 | }; 19 | 20 | 21 | /* 22 | * Container for a queue pair (CQ + SQ). 23 | */ 24 | struct QueuePair 25 | { 26 | public: 27 | const uint16_t no; // Queue number 28 | const size_t depth; // Queue depth (number of commands) 29 | const size_t pages; // Maximum data transfer size (in pages) 30 | 31 | mutable nvm_queue_t sq; 32 | mutable nvm_queue_t cq; 33 | 34 | 35 | QueuePair(const CtrlPtr& controller, uint16_t no, size_t depth, size_t pages, DmaPtr queueMemory); 36 | 37 | 38 | QueuePair(const CtrlPtr& controller, uint16_t no, size_t depth, size_t pages, DmaPtr cqMemory, DmaPtr sqMemory); 39 | 40 | virtual ~QueuePair(); 41 | 42 | virtual std::string type() const = 0; 43 | 44 | virtual QueueLocation location() const = 0; 45 | 46 | const Ctrl& getController() const 47 | { 48 | return *controller; 49 | } 50 | 51 | const DmaPtr& getQueueMemory() const 52 | { 53 | return sqMemory; 54 | } 55 | 56 | protected: 57 | QueuePair(); 58 | 59 | private: 60 | CtrlPtr controller; 61 | DmaPtr cqMemory; 62 | DmaPtr sqMemory; 63 | }; 64 | 65 | 66 | typedef std::shared_ptr QueuePtr; 67 | typedef std::map QueueMap; 68 | 69 | 70 | 71 | struct GpuQueue : public QueuePair 72 | { 73 | public: 74 | GpuQueue(const CtrlPtr& controller, 75 | uint16_t no, 76 | size_t depth, 77 | size_t pages, 78 | const GpuPtr& gpu, 79 | uint32_t adapter, 80 | uint32_t cqSegmentId, 81 | uint32_t sqSegmentId); 82 | 83 | std::string type() const override; 84 | 85 | QueueLocation location() const override 86 | { 87 | return QueueLocation::GPU; 88 | } 89 | 90 | const Gpu& getGpu() const 91 | { 92 | return *gpu; 93 | } 94 | 95 | private: 96 | GpuPtr gpu; 97 | }; 98 | 99 | 100 | 101 | struct LocalQueue : public QueuePair 102 | { 103 | public: 104 | std::string type() const override; 105 | 106 | LocalQueue(const CtrlPtr& controller, uint16_t no, size_t depth, size_t pages); 107 | 108 | LocalQueue(const CtrlPtr& controller, uint16_t no, size_t depth, size_t pages, uint32_t adapter, uint32_t segmentId); 109 | 110 | QueueLocation location() const override 111 | { 112 | return QueueLocation::LOCAL; 113 | } 114 | }; 115 | 116 | 117 | 118 | struct RemoteQueue : public QueuePair 119 | { 120 | public: 121 | std::string type() const override; 122 | 123 | RemoteQueue(const CtrlPtr& controller, uint16_t no, size_t depth, size_t pages, uint32_t adapter, uint32_t segmentId); 124 | 125 | QueueLocation location() const override 126 | { 127 | return QueueLocation::REMOTE; 128 | } 129 | }; 130 | 131 | 132 | #endif /* __LATENCY_BENCHMARK_QUEUE_H__ */ 133 | -------------------------------------------------------------------------------- /benchmarks/latency/include/settings.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_SETTINGS_H__ 2 | #define __LATENCY_BENCHMARK_SETTINGS_H__ 3 | 4 | #include "gpu.h" 5 | #include "ctrl.h" 6 | #include "queue.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | struct CtrlParam 14 | { 15 | std::string path; 16 | uint64_t fdid; 17 | uint32_t adapter; 18 | }; 19 | 20 | 21 | struct GpuParam 22 | { 23 | int device; 24 | uint64_t fdid; 25 | uint32_t adapter; 26 | }; 27 | 28 | 29 | struct QueueParam 30 | { 31 | uint16_t no; 32 | uint16_t depth; 33 | size_t pages; 34 | QueueLocation location; 35 | uint64_t fdid; 36 | uint32_t adapter; 37 | QueueParam(); 38 | QueueParam(const std::string& arg); 39 | }; 40 | 41 | 42 | enum AccessUnit : int 43 | { 44 | BLOCKS, PAGES, BYTES, COMMANDS 45 | }; 46 | 47 | 48 | struct Settings 49 | { 50 | CtrlParam ctrl; 51 | std::vector queues; 52 | GpuParam gpu; 53 | bool latency; 54 | uint32_t nvmNamespace; 55 | size_t outerIterations; 56 | size_t innerIterations; 57 | size_t count; 58 | size_t offset; 59 | AccessUnit unit; 60 | bool sequential; 61 | bool parallel; 62 | bool shared; 63 | bool random; 64 | std::string outputFilename; 65 | bool stats; 66 | std::string statsFilename; 67 | std::string inputFilename; 68 | bool manager; 69 | uint32_t segmentOffset; 70 | bool write; 71 | bool verify; 72 | bool transferInfo; 73 | 74 | Settings(); 75 | 76 | uint32_t id() 77 | { 78 | return segmentOffset++; 79 | } 80 | 81 | void parseArguments(int argc, char** argv); 82 | }; 83 | 84 | 85 | #endif /* __LATENCY_BENCHMARK_SETTINGS_H__ */ 86 | -------------------------------------------------------------------------------- /benchmarks/latency/include/stats.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_STATISTICS_H__ 2 | #define __LATENCY_BENCHMARK_STATISTICS_H__ 3 | 4 | #include 5 | #include "ctrl.h" 6 | #include "queue.h" 7 | #include "benchmark.h" 8 | #include "transfer.h" 9 | 10 | 11 | void calculatePercentiles(const Ctrl& controller, const EventMap& events, bool write); 12 | 13 | 14 | void printStatistics(const Ctrl& controller, const EventMap& readEvents, const EventMap& writeEvents, const TransferMap& transfers, const Settings& settings); 15 | 16 | 17 | #endif /* __LATENCY_BENCHMARK_STATISTICS_H__ */ 18 | -------------------------------------------------------------------------------- /benchmarks/latency/include/transfer.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_TRANSFER_H__ 2 | #define __LATENCY_BENCHMARK_TRANSFER_H__ 3 | 4 | #include "ctrl.h" 5 | #include "queue.h" 6 | #include "buffer.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | 14 | /* 15 | * Describe a transfer of "chunk" -- the maximum transfer of a single command. 16 | */ 17 | struct Chunk 18 | { 19 | size_t startBlock; 20 | size_t numBlocks; 21 | size_t startPage; 22 | size_t numPages; 23 | }; 24 | 25 | 26 | 27 | /* Convenience types */ 28 | typedef std::vector ChunkList; 29 | typedef ChunkList::const_iterator ChunkPtr; 30 | 31 | 32 | 33 | /* 34 | * Describes a total transfer. 35 | */ 36 | struct Transfer 37 | { 38 | QueuePtr queue; 39 | MemoryBufferPtr buffer; 40 | size_t count; 41 | size_t offset; 42 | size_t pageOffset; 43 | ChunkList chunks; 44 | }; 45 | 46 | 47 | typedef std::shared_ptr TransferPtr; 48 | typedef std::map TransferMap; 49 | 50 | 51 | 52 | #endif /* __LATENCY_BENCHMARK_SETTINGS_H__ */ 53 | -------------------------------------------------------------------------------- /benchmarks/latency/include/utility.h: -------------------------------------------------------------------------------- 1 | #ifndef __LATENCY_BENCHMARK_UTILITY_H__ 2 | #define __LATENCY_BENCHMARK_UTILITY_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "ctrl.h" 8 | #include "settings.h" 9 | 10 | 11 | 12 | static size_t calculateTransferBlocks(const Ctrl& ctrl, const Settings& settings) 13 | { 14 | size_t size = 0; 15 | 16 | switch (settings.unit) 17 | { 18 | case AccessUnit::BLOCKS: 19 | size = settings.count; 20 | break; 21 | 22 | case AccessUnit::PAGES: 23 | size = NVM_PAGE_ALIGN(settings.count * ctrl.pageSize, ctrl.blockSize) / ctrl.blockSize; 24 | break; 25 | 26 | case AccessUnit::BYTES: 27 | size = NVM_PAGE_ALIGN(settings.count, ctrl.blockSize) / ctrl.blockSize; 28 | break; 29 | 30 | default: 31 | throw std::logic_error("Unknown transfer unit"); 32 | } 33 | 34 | return size; 35 | } 36 | 37 | 38 | 39 | static size_t calculateOffsetBlocks(const Ctrl& ctrl, const Settings& settings) 40 | { 41 | size_t offset = 0; 42 | 43 | switch (settings.unit) 44 | { 45 | case AccessUnit::BLOCKS: 46 | offset = settings.offset; 47 | break; 48 | 49 | case AccessUnit::PAGES: 50 | offset = NVM_PAGE_TO_BLOCK(ctrl.pageSize, ctrl.blockSize, settings.offset); 51 | break; 52 | 53 | case AccessUnit::BYTES: 54 | offset = NVM_PAGE_ALIGN(settings.offset, ctrl.blockSize) / ctrl.blockSize; 55 | break; 56 | 57 | default: 58 | throw std::logic_error("Unknown transfer unit"); 59 | } 60 | 61 | return offset; 62 | } 63 | 64 | 65 | 66 | static size_t calculateBufferSize(const Ctrl& ctrl, const Settings& settings) 67 | { 68 | size_t size = 0; 69 | 70 | switch (settings.unit) 71 | { 72 | case AccessUnit::BLOCKS: 73 | size = NVM_PAGE_ALIGN(settings.count * ctrl.blockSize, ctrl.pageSize); 74 | break; 75 | 76 | case AccessUnit::PAGES: 77 | size = settings.count * ctrl.pageSize; 78 | break; 79 | 80 | case AccessUnit::BYTES: 81 | size = NVM_PAGE_ALIGN(settings.count, ctrl.pageSize); 82 | break; 83 | 84 | default: 85 | throw std::logic_error("Unknown transfer unit"); 86 | } 87 | 88 | return size; 89 | } 90 | 91 | #endif /* __LATENCY_BENCHMARK_UTILITY_H__ */ 92 | -------------------------------------------------------------------------------- /benchmarks/latency/smartio/buffer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "ctrl.h" 11 | #include "buffer.h" 12 | 13 | using std::runtime_error; 14 | using std::logic_error; 15 | using std::string; 16 | 17 | 18 | 19 | DmaPtr allocateBuffer(const nvm_ctrl_t* ctrl, size_t size, uint32_t adapter, uint32_t segmentId) 20 | { 21 | nvm_dma_t* dma = nullptr; 22 | 23 | auto err = nvm_dis_dma_create(&dma, ctrl, adapter, segmentId, size); 24 | if (!nvm_ok(err)) 25 | { 26 | throw runtime_error(string("Failed to create local segment mapped for controller: ") + nvm_strerror(err)); 27 | } 28 | 29 | return DmaPtr(dma, nvm_dma_unmap); 30 | } 31 | 32 | 33 | 34 | DmaPtr allocateBuffer(const Ctrl& ctrl, size_t size, uint32_t adapter, uint32_t segmentId) 35 | { 36 | if (ctrl.fdid == 0) 37 | { 38 | return allocateBuffer(ctrl, size); 39 | } 40 | 41 | return allocateBuffer(ctrl.handle, size, adapter, segmentId); 42 | } 43 | 44 | 45 | 46 | DmaPtr allocateBuffer(const Ctrl& ctrl, size_t size, uint32_t segmentId) 47 | { 48 | return allocateBuffer(ctrl, size, ctrl.adapter, segmentId); 49 | } 50 | 51 | 52 | 53 | DmaPtr connectBuffer(const nvm_ctrl_t* ctrl, size_t size, uint32_t adapter, uint32_t number) 54 | { 55 | nvm_dma_t* dma = nullptr; 56 | 57 | auto err = nvm_dis_dma_connect(&dma, ctrl, adapter, number, size, true); 58 | if (!nvm_ok(err)) 59 | { 60 | throw runtime_error(string("Failed to connect to remote segment: ") + nvm_strerror(err)); 61 | } 62 | 63 | return DmaPtr(dma, nvm_dma_unmap); 64 | } 65 | 66 | 67 | 68 | DmaPtr connectBuffer(const Ctrl& ctrl, size_t size, uint32_t adapter, uint32_t number) 69 | { 70 | if (ctrl.fdid == 0) 71 | { 72 | throw logic_error("Controller is not a SmartIO device"); 73 | } 74 | 75 | return connectBuffer(ctrl.handle, size, adapter, number); 76 | } 77 | 78 | -------------------------------------------------------------------------------- /benchmarks/latency/smartio/ctrl.cc: -------------------------------------------------------------------------------- 1 | #include "ctrl.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | using error = std::runtime_error; 19 | using std::string; 20 | 21 | 22 | 23 | CtrlManager::CtrlManager(uint64_t fdid, uint32_t adapter, uint32_t segmentId, bool admin, uint32_t ns) 24 | : controller(nullptr) 25 | , fileDescriptor(-1) 26 | { 27 | nvm_ctrl_t* ctrl = nullptr; 28 | nvm_dma_t* dma = nullptr; 29 | nvm_aq_ref ref = nullptr; 30 | 31 | try 32 | { 33 | // Get controller reference 34 | int status = nvm_dis_ctrl_init(&ctrl, fdid, adapter); 35 | if (!nvm_ok(status)) 36 | { 37 | throw error("Failed to get controller reference: " + string(nvm_strerror(status))); 38 | } 39 | 40 | // Create and map queue memory for controller 41 | status = nvm_dis_dma_create(&dma, ctrl, adapter, segmentId, 3 * ctrl->page_size); 42 | if (!nvm_ok(status)) 43 | { 44 | throw error("Failed to create admin queue memory: " + string(nvm_strerror(status))); 45 | } 46 | 47 | // Create or retrieve admin queue reference 48 | if (admin) 49 | { 50 | status = nvm_aq_create(&ref, ctrl, dma); 51 | } 52 | else 53 | { 54 | status = nvm_dis_rpc_bind(&ref, ctrl, adapter); 55 | } 56 | 57 | if (!nvm_ok(status)) 58 | { 59 | throw error("Failed to initialize admin queues: " + string(nvm_strerror(status))); 60 | } 61 | 62 | // Create controller reference wrapper 63 | auto* ptr = new (std::nothrow) Ctrl(fdid, adapter, ctrl, nullptr, dma, ref, ns); 64 | if (ptr == nullptr) 65 | { 66 | throw error("Failed to allocate shared controller reference"); 67 | } 68 | 69 | controller.reset(ptr); 70 | } 71 | catch (const error& e) 72 | { 73 | nvm_aq_destroy(ref); 74 | nvm_dma_unmap(dma); 75 | nvm_ctrl_free(ctrl); 76 | throw e; 77 | } 78 | } 79 | 80 | -------------------------------------------------------------------------------- /benchmarks/latency/smartio/queue.cc: -------------------------------------------------------------------------------- 1 | #include "queue.h" 2 | #include "buffer.h" 3 | #include "ctrl.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | 11 | LocalQueue::LocalQueue(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages, uint32_t adapter, uint32_t segmentId) 12 | : QueuePair(ctrl, no, depth, pages, 13 | allocateBuffer(*ctrl, (2 + depth) * ctrl->pageSize, adapter, segmentId)) 14 | { 15 | } 16 | 17 | 18 | 19 | GpuQueue::GpuQueue(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages, const GpuPtr& gpu, uint32_t adapter, uint32_t cqSegId, uint32_t sqSegId) 20 | : QueuePair(ctrl, no, depth, pages, 21 | allocateBuffer(*ctrl, ctrl->pageSize, adapter, cqSegId), 22 | gpu->allocateBufferAndMap(*ctrl, (depth + 1) * ctrl->pageSize, sqSegId)) 23 | , gpu(gpu) 24 | { 25 | } 26 | 27 | 28 | 29 | RemoteQueue::RemoteQueue(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages, uint32_t adapter, uint32_t segmentId) 30 | : QueuePair(ctrl, no, depth, pages, 31 | allocateBuffer(*ctrl, ctrl->pageSize, adapter, segmentId), 32 | connectBuffer(*ctrl, (depth + 1) * ctrl->pageSize, adapter, no)) 33 | { 34 | } 35 | 36 | -------------------------------------------------------------------------------- /benchmarks/latency/src/barrier.cc: -------------------------------------------------------------------------------- 1 | #include "barrier.h" 2 | #include 3 | #include 4 | 5 | 6 | 7 | Barrier::Barrier(int numThreads) 8 | : reset(numThreads) 9 | , arrived(0) 10 | , flag(0) 11 | { 12 | } 13 | 14 | 15 | void Barrier::wait() 16 | { 17 | mtx.lock(); 18 | 19 | int localFlag = !flag; 20 | 21 | if (++arrived == reset) 22 | { 23 | mtx.unlock(); 24 | arrived = 0; 25 | flag = localFlag; 26 | } 27 | else 28 | { 29 | mtx.unlock(); 30 | 31 | while (flag != localFlag) 32 | { 33 | std::this_thread::yield(); 34 | } 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /benchmarks/latency/src/buffer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "ctrl.h" 12 | #include "buffer.h" 13 | 14 | using error = std::runtime_error; 15 | using std::string; 16 | 17 | 18 | 19 | DmaPtr allocateBuffer(const nvm_ctrl_t* ctrl, size_t size) 20 | { 21 | auto memory = allocateHostMemory(size); 22 | 23 | nvm_dma_t* dma = nullptr; 24 | int err = nvm_dma_map_host(&dma, ctrl, memory.get(), size); 25 | if (!nvm_ok(err)) 26 | { 27 | throw error("Failed to map local RAM for controller: " + string(nvm_strerror(err))); 28 | } 29 | 30 | return DmaPtr(dma, [memory](nvm_dma_t* dma) mutable { 31 | nvm_dma_unmap(dma); 32 | memory.reset(); 33 | }); 34 | } 35 | 36 | 37 | 38 | #ifndef __DIS_CLUSTER__ 39 | DmaPtr allocateBuffer(const nvm_ctrl_t* ctrl, size_t size, uint32_t, uint32_t) 40 | { 41 | return allocateBuffer(ctrl, size); 42 | } 43 | #endif 44 | 45 | 46 | 47 | DmaPtr allocateBuffer(const Ctrl& ctrl, size_t size) 48 | { 49 | return allocateBuffer(ctrl.handle, size); 50 | } 51 | 52 | 53 | 54 | #ifndef __DIS_CLUSTER__ 55 | DmaPtr allocateBuffer(const Ctrl& ctrl, size_t size, uint32_t, uint32_t) 56 | { 57 | return allocateBuffer(ctrl.handle, size); 58 | } 59 | #endif 60 | 61 | 62 | 63 | #ifndef __DIS_CLUSTER__ 64 | DmaPtr allocateBuffer(const Ctrl& ctrl, size_t size, uint32_t) 65 | { 66 | return allocateBuffer(ctrl.handle, size); 67 | } 68 | #endif 69 | 70 | 71 | 72 | MemoryBuffer::MemoryBuffer(const Ctrl& ctrl, size_t size) 73 | : size(size) 74 | , buffer(allocateBuffer(ctrl, size)) 75 | { 76 | } 77 | 78 | 79 | 80 | MemoryBuffer::MemoryBuffer(const Ctrl& ctrl, size_t size, uint32_t segmentId) 81 | : size(size) 82 | , buffer(allocateBuffer(ctrl, size, segmentId)) 83 | { 84 | } 85 | 86 | 87 | 88 | size_t MemoryBuffer::load(const void* src, size_t size, size_t offset) 89 | { 90 | if (offset + size > this->size) 91 | { 92 | throw error("Specified offset and size is too large"); 93 | } 94 | 95 | void* dst = (void*) (((unsigned char*) buffer->vaddr) + offset); 96 | memcpy(dst, src, size); 97 | return size; 98 | } 99 | 100 | 101 | 102 | size_t MemoryBuffer::save(void* dst, size_t size, size_t offset) const 103 | { 104 | if (offset + size > this->size) 105 | { 106 | throw error("Specified offset and size is too large"); 107 | } 108 | 109 | void* src = (void*) (((unsigned char*) buffer->vaddr) + offset); 110 | memcpy(dst, src, size); 111 | return size; 112 | } 113 | 114 | 115 | 116 | void MemoryBuffer::clear() 117 | { 118 | memset(buffer->vaddr, 0, size); 119 | } 120 | 121 | -------------------------------------------------------------------------------- /benchmarks/latency/src/ctrl.cc: -------------------------------------------------------------------------------- 1 | #include "ctrl.h" 2 | #include "buffer.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | using error = std::runtime_error; 26 | using std::string; 27 | 28 | 29 | 30 | Ctrl::Ctrl(uint64_t fdid, uint32_t adapter, nvm_ctrl_t* controller, DmaPtr memory, nvm_dma_t* dma, nvm_aq_ref ref, uint32_t ns) 31 | : writeHandle(controller) 32 | , adminQueueMemory(memory) 33 | , adminQueues(dma) 34 | , adminRef(ref) 35 | , handle(controller) 36 | , fdid(fdid) 37 | , adapter(adapter) 38 | , namespaceId(ns) 39 | { 40 | // Identify cntroller 41 | nvm_ctrl_info ctrlInfo; 42 | int status = nvm_admin_ctrl_info(adminRef, &ctrlInfo, NVM_DMA_OFFSET(adminQueues, 2), adminQueues->ioaddrs[2]); 43 | if (!nvm_ok(status)) 44 | { 45 | throw error("Failed to identify controller: " + string(nvm_strerror(status))); 46 | } 47 | 48 | // Identify namespace 49 | nvm_ns_info nsInfo; 50 | status = nvm_admin_ns_info(adminRef, &nsInfo, ns, NVM_DMA_OFFSET(adminQueues, 2), adminQueues->ioaddrs[2]); 51 | if (!nvm_ok(status)) 52 | { 53 | throw error("Failed to identify namespace: " + string(nvm_strerror(status))); 54 | } 55 | 56 | // Figure out the maximum number of queues 57 | uint16_t numSQs = 0; 58 | uint16_t numCQs = 0; 59 | status = nvm_admin_get_num_queues(adminRef, &numCQs, &numSQs); 60 | if (!nvm_ok(status)) 61 | { 62 | throw error("Failed to get number of queues: " + string(nvm_strerror(status))); 63 | } 64 | 65 | // Set members 66 | namespaceSize = nsInfo.size; 67 | pageSize = ctrlInfo.page_size; 68 | blockSize = nsInfo.lba_data_size; 69 | chunkSize = ctrlInfo.max_data_size; 70 | numQueues = std::min(numSQs, numCQs); 71 | maxEntries = std::min((size_t) ctrlInfo.max_entries, ctrlInfo.page_size / ctrlInfo.sq_entry_size); 72 | } 73 | 74 | 75 | 76 | Ctrl::~Ctrl() 77 | { 78 | nvm_aq_destroy(adminRef); 79 | if (adminQueueMemory.get() == nullptr) 80 | { 81 | nvm_dma_unmap(adminQueues); 82 | } 83 | adminQueueMemory.reset(); 84 | nvm_ctrl_free(writeHandle); 85 | } 86 | 87 | 88 | 89 | CtrlManager::CtrlManager(const string& path, uint32_t ns) 90 | : controller(nullptr) 91 | , fileDescriptor(-1) 92 | { 93 | // Open file descriptor 94 | fileDescriptor = open(path.c_str(), O_RDWR | O_NONBLOCK); 95 | if (fileDescriptor < 0) 96 | { 97 | throw error("Failed to open file descriptor: " + string(strerror(errno))); 98 | } 99 | 100 | DmaPtr queueMemory; 101 | nvm_ctrl_t* ctrl = nullptr; 102 | nvm_aq_ref ref = nullptr; 103 | 104 | try 105 | { 106 | // Get controller reference 107 | int status = nvm_ctrl_init(&ctrl, fileDescriptor); 108 | if (!nvm_ok(status)) 109 | { 110 | throw error("Failed to get controller reference: " + string(nvm_strerror(status))); 111 | } 112 | 113 | // Map queue memory for controller 114 | queueMemory = allocateBuffer(ctrl, ctrl->page_size * 3); 115 | 116 | // Create admin queue reference 117 | status = nvm_aq_create(&ref, ctrl, queueMemory.get()); 118 | if (!nvm_ok(status)) 119 | { 120 | throw error("Failed to reset controller: " + string(nvm_strerror(status))); 121 | } 122 | 123 | // Create controller reference wrapper 124 | auto* ptr = new (std::nothrow) Ctrl(0, 0, ctrl, queueMemory, queueMemory.get(), ref, ns); 125 | if (ptr == nullptr) 126 | { 127 | throw error("Failed to allocate shared controller reference"); 128 | } 129 | 130 | controller.reset(ptr); 131 | } 132 | catch (const error& e) 133 | { 134 | nvm_aq_destroy(ref); 135 | queueMemory.reset(); 136 | nvm_ctrl_free(ctrl); 137 | close(fileDescriptor); 138 | throw e; 139 | } 140 | } 141 | 142 | 143 | CtrlManager::~CtrlManager() 144 | { 145 | controller.reset(); 146 | if (fileDescriptor >= 0) 147 | { 148 | close(fileDescriptor); 149 | } 150 | } 151 | 152 | 153 | 154 | #ifndef __DIS_CLUSTER__ 155 | CtrlManager::CtrlManager(uint64_t, uint32_t, uint32_t, bool, uint32_t) 156 | : controller(nullptr) 157 | , fileDescriptor(-1) 158 | { 159 | throw std::logic_error("SmartIO unsupported"); 160 | } 161 | #endif 162 | 163 | -------------------------------------------------------------------------------- /benchmarks/latency/src/host.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "buffer.h" 9 | 10 | 11 | /* 12 | * We need to use cudaHostAlloc, since posix_memalign does not work well with CUDA. 13 | */ 14 | MemoryPtr allocateHostMemory(size_t size) 15 | { 16 | void* pointer = nullptr; 17 | 18 | auto err = cudaHostAlloc(&pointer, size, cudaHostAllocDefault); 19 | if (err != cudaSuccess) 20 | { 21 | throw std::runtime_error(std::string("Failed to allocate memory in local RAM: ") + cudaGetErrorString(err)); 22 | } 23 | 24 | memset(pointer, 0, size); 25 | return MemoryPtr(pointer, cudaFreeHost); 26 | } 27 | 28 | -------------------------------------------------------------------------------- /benchmarks/latency/src/queue.cc: -------------------------------------------------------------------------------- 1 | #include "queue.h" 2 | #include "buffer.h" 3 | #include "ctrl.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using std::string; 16 | using error = std::runtime_error; 17 | 18 | 19 | 20 | QueuePair::QueuePair() 21 | : no(0) 22 | , depth(0) 23 | , pages(0) 24 | { 25 | throw std::logic_error("SmartIO not supported"); 26 | } 27 | 28 | 29 | 30 | QueuePair::QueuePair(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages, DmaPtr cqPtr, DmaPtr sqPtr) 31 | : no(no) 32 | , depth(std::max(std::min(depth, ctrl->maxEntries - 1), (size_t) 1)) 33 | , pages(std::max(pages, (size_t) 1)) 34 | , controller(ctrl) 35 | , cqMemory(cqPtr) 36 | , sqMemory(sqPtr) 37 | { 38 | if (depth > this->depth) 39 | { 40 | throw error("Queue depth is greater than maximum queue size"); 41 | } 42 | 43 | if (this->pages * ctrl->pageSize > ctrl->chunkSize) 44 | { 45 | throw error("Chunk size is greater than maximum data transfer size"); 46 | } 47 | 48 | if (sqPtr->n_ioaddrs < 1 + this->depth) 49 | { 50 | throw error("Invalid DMA mapping for submission queue memory"); 51 | } 52 | 53 | nvm_aq_ref ref = controller->adminReference(); 54 | 55 | memset(cqMemory->vaddr, 0, controller->pageSize); 56 | int status = nvm_admin_cq_create(ref, &cq, no, cqMemory->vaddr, cqMemory->ioaddrs[0]); 57 | if (!nvm_ok(status)) 58 | { 59 | throw error("Failed to create completion queue: " + string(nvm_strerror(status))); 60 | } 61 | 62 | memset(sqMemory->vaddr, 0, controller->pageSize); 63 | status = nvm_admin_sq_create(ref, &sq, &cq, no, sqMemory->vaddr, sqMemory->ioaddrs[0]); 64 | if (!nvm_ok(status)) 65 | { 66 | throw error("Failed to create submission queue: " + string(nvm_strerror(status))); 67 | } 68 | } 69 | 70 | 71 | 72 | QueuePair::QueuePair(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages, DmaPtr queueMemory) 73 | : no(no) 74 | , depth(std::max((size_t) 1, std::min(depth, ctrl->maxEntries - 1))) 75 | , pages(std::max(pages, (size_t) 1)) 76 | , controller(ctrl) 77 | , cqMemory(nullptr) 78 | , sqMemory(queueMemory) 79 | { 80 | if (depth > this->depth) 81 | { 82 | throw error("Queue depth is greater than maximum queue size"); 83 | } 84 | 85 | if (this->pages * ctrl->pageSize > ctrl->chunkSize) 86 | { 87 | throw error("Chunk size is greater than maximum data transfer size"); 88 | } 89 | 90 | if (queueMemory->n_ioaddrs < 2 + this->depth) 91 | { 92 | throw error("Invalid DMA mapping for queue memory"); 93 | } 94 | 95 | nvm_aq_ref ref = controller->adminReference(); 96 | 97 | void* cqPtr = NVM_DMA_OFFSET(queueMemory, 1 + this->depth); 98 | memset(cqPtr, 0, ctrl->pageSize); 99 | int status = nvm_admin_cq_create(ref, &cq, no, cqPtr, queueMemory->ioaddrs[1 + this->depth]); 100 | if (!nvm_ok(status)) 101 | { 102 | throw error("Failed to create completion queue: " + string(nvm_strerror(status))); 103 | } 104 | 105 | memset(queueMemory->vaddr, 0, controller->pageSize); 106 | status = nvm_admin_sq_create(ref, &sq, &cq, no, queueMemory->vaddr, queueMemory->ioaddrs[0]); 107 | if (!nvm_ok(status)) 108 | { 109 | throw error("Failed to create submission queue: " + string(nvm_strerror(status))); 110 | } 111 | } 112 | 113 | 114 | 115 | QueuePair::~QueuePair() 116 | { 117 | nvm_aq_ref ref = controller->adminReference(); 118 | nvm_admin_sq_delete(ref, &sq, &cq); 119 | nvm_admin_cq_delete(ref, &cq); 120 | } 121 | 122 | 123 | 124 | LocalQueue::LocalQueue(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages) 125 | : QueuePair(ctrl, no, depth, pages, allocateBuffer(*ctrl, (2 + depth) * ctrl->pageSize)) 126 | { 127 | } 128 | 129 | 130 | 131 | #ifndef __DIS_CLUSTER__ 132 | LocalQueue::LocalQueue(const CtrlPtr& ctrl, uint16_t no, size_t depth, size_t pages, uint32_t, uint32_t) 133 | : QueuePair(ctrl, no, depth, pages, allocateBuffer(*ctrl, (2 + depth) * ctrl->pageSize)) 134 | { 135 | } 136 | #endif 137 | 138 | 139 | string LocalQueue::type() const 140 | { 141 | return "host"; 142 | } 143 | 144 | 145 | 146 | string RemoteQueue::type() const 147 | { 148 | return "remote"; 149 | } 150 | 151 | 152 | 153 | string GpuQueue::type() const 154 | { 155 | return "gpu"; 156 | } 157 | 158 | 159 | 160 | #ifndef __DIS_CLUSTER__ 161 | GpuQueue::GpuQueue(const CtrlPtr&, uint16_t, size_t, size_t, const GpuPtr&, uint32_t, uint32_t, uint32_t) 162 | { 163 | throw std::logic_error("SmartIO not supported"); 164 | } 165 | #endif 166 | 167 | 168 | 169 | #ifndef __DIS_CLUSTER__ 170 | RemoteQueue::RemoteQueue(const CtrlPtr&, uint16_t, size_t, size_t, uint32_t, uint32_t) 171 | { 172 | throw std::logic_error("SmartIO not supported"); 173 | } 174 | #endif 175 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | Collection of example programs using the library 2 | -------------------------------------------------------------------------------- /examples/identify/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | project (libnvm-samples) 3 | 4 | make_sample (identify-userspace identify-userspace "userspace.c;common.c") 5 | 6 | make_sample (identify-module identify "module.c;common.c") 7 | 8 | make_sample (identify-smartio identify "smartio.c;common.c") 9 | set_sisci (identify-smartio) 10 | 11 | make_samples_choice(identify identify-smartio identify-module) 12 | -------------------------------------------------------------------------------- /examples/identify/common.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "common.h" 8 | 9 | 10 | 11 | /* 12 | * Print controller information. 13 | */ 14 | static void print_ctrl_info(FILE* fp, const struct nvm_ctrl_info* info, uint16_t n_cqs, uint16_t n_sqs) 15 | { 16 | unsigned char vendor[4]; 17 | memcpy(vendor, &info->pci_vendor, sizeof(vendor)); 18 | 19 | char serial[21]; 20 | memset(serial, 0, 21); 21 | memcpy(serial, info->serial_no, 20); 22 | 23 | char model[41]; 24 | memset(model, 0, 41); 25 | memcpy(model, info->model_no, 40); 26 | 27 | char revision[9]; 28 | memset(revision, 0, 9); 29 | memcpy(revision, info->firmware, 8); 30 | 31 | fprintf(fp, "------------- Controller information -------------\n"); 32 | fprintf(fp, "PCI Vendor ID : %x %x\n", vendor[0], vendor[1]); 33 | fprintf(fp, "PCI Subsystem Vendor ID : %x %x\n", vendor[2], vendor[3]); 34 | fprintf(fp, "NVM Express version : %u.%u.%u\n", 35 | info->nvme_version >> 16, (info->nvme_version >> 8) & 0xff, info->nvme_version & 0xff); 36 | fprintf(fp, "Controller page size : %zu\n", info->page_size); 37 | fprintf(fp, "Max queue entries : %u\n", info->max_entries); 38 | fprintf(fp, "Serial Number : %s\n", serial); 39 | fprintf(fp, "Model Number : %s\n", model); 40 | fprintf(fp, "Firmware revision : %s\n", revision); 41 | fprintf(fp, "Max data transfer size : %zu\n", info->max_data_size); 42 | fprintf(fp, "Max outstanding commands: %zu\n", info->max_out_cmds); 43 | fprintf(fp, "Max number of namespaces: %zu\n", info->max_n_ns); 44 | fprintf(fp, "Current number of CQs : %u\n", n_cqs); 45 | fprintf(fp, "Current number of SQs : %u\n", n_sqs); 46 | fprintf(fp, "--------------------------------------------------\n"); 47 | } 48 | 49 | 50 | /* 51 | * Print namespace information. 52 | */ 53 | static void print_ns_info(FILE* fp, const struct nvm_ns_info* info) 54 | { 55 | fprintf(fp, "------------- Namespace information -------------\n"); 56 | fprintf(fp, "Namespace identifier : %x\n", info->ns_id); 57 | fprintf(fp, "Logical block size : %zu bytes\n", info->lba_data_size); 58 | fprintf(fp, "Namespace size : %zu blocks\n", info->size); 59 | fprintf(fp, "Namespace capacity : %zu blocks\n", info->capacity); 60 | fprintf(fp, "--------------------------------------------------\n"); 61 | } 62 | 63 | 64 | 65 | nvm_aq_ref reset_ctrl(const nvm_ctrl_t* ctrl, const nvm_dma_t* dma_window) 66 | { 67 | int status; 68 | nvm_aq_ref admin; 69 | 70 | if (dma_window->n_ioaddrs < 2) 71 | { 72 | return NULL; 73 | } 74 | memset(dma_window->vaddr, 0, dma_window->page_size * 2); 75 | 76 | fprintf(stderr, "Resetting controller and setting up admin queues...\n"); 77 | status = nvm_aq_create(&admin, ctrl, dma_window); 78 | if (status != 0) 79 | { 80 | fprintf(stderr, "Failed to reset controller: %s\n", strerror(status)); 81 | return NULL; 82 | } 83 | 84 | return admin; 85 | } 86 | 87 | 88 | 89 | int identify_ctrl(nvm_aq_ref admin, void* ptr, uint64_t ioaddr) 90 | { 91 | int status; 92 | uint16_t n_cqs = 0; 93 | uint16_t n_sqs = 0; 94 | struct nvm_ctrl_info info; 95 | 96 | status = nvm_admin_get_num_queues(admin, &n_cqs, &n_sqs); 97 | if (status != 0) 98 | { 99 | fprintf(stderr, "Failed to get number of queues\n"); 100 | return status; 101 | } 102 | 103 | status = nvm_admin_ctrl_info(admin, &info, ptr, ioaddr); 104 | if (status != 0) 105 | { 106 | fprintf(stderr, "Failed to identify controller: %s\n", strerror(status)); 107 | return status; 108 | } 109 | 110 | print_ctrl_info(stdout, &info, n_cqs, n_sqs); 111 | return 0; 112 | } 113 | 114 | 115 | 116 | int identify_ns(nvm_aq_ref admin, uint32_t nvm_namespace, void* ptr, uint64_t ioaddr) 117 | { 118 | int status; 119 | struct nvm_ns_info info; 120 | 121 | status = nvm_admin_ns_info(admin, &info, nvm_namespace, ptr, ioaddr); 122 | if (status != 0) 123 | { 124 | fprintf(stderr, "Failed to identify namespace: %s\n", strerror(status)); 125 | return status; 126 | } 127 | 128 | print_ns_info(stdout, &info); 129 | return 0; 130 | } 131 | 132 | -------------------------------------------------------------------------------- /examples/identify/common.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_SAMPLES_IDENTIFY_H__ 2 | #define __LIBNVM_SAMPLES_IDENTIFY_H__ 3 | 4 | #include 5 | #include 6 | 7 | 8 | /* 9 | * Reset controller and take ownership of admin queues. 10 | * DMA window must be at least 2 pages. 11 | * Caller must manually destroy the admin reference. 12 | */ 13 | nvm_aq_ref reset_ctrl(const nvm_ctrl_t* ctrl, const nvm_dma_t* dma_window); 14 | 15 | 16 | /* 17 | * Identify controller and print information. 18 | */ 19 | int identify_ctrl(nvm_aq_ref admin, void* ptr, uint64_t ioaddr); 20 | 21 | 22 | /* 23 | * Identify namespace and print information. 24 | */ 25 | int identify_ns(nvm_aq_ref admin, uint32_t nvm_namespace, void* ptr, uint64_t ioaddr); 26 | 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /examples/identify/module.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "common.h" 19 | 20 | 21 | 22 | static void parse_args(int argc, char** argv, char** device, uint32_t* ns_id); 23 | 24 | 25 | 26 | static int open_fd(const char* path) 27 | { 28 | int fd; 29 | 30 | fd = open(path, O_RDWR|O_NONBLOCK); 31 | if (fd < 0) 32 | { 33 | fprintf(stderr, "Failed to open descriptor: %s\n", strerror(errno)); 34 | return -1; 35 | } 36 | 37 | return fd; 38 | } 39 | 40 | 41 | int main(int argc, char** argv) 42 | { 43 | int status; 44 | nvm_ctrl_t* ctrl; 45 | nvm_dma_t* window = NULL; 46 | nvm_aq_ref admin = NULL; 47 | uint32_t nvm_namespace; 48 | void* memory; 49 | 50 | long page_size = sysconf(_SC_PAGESIZE); 51 | 52 | char* path = NULL; 53 | parse_args(argc, argv, &path, &nvm_namespace); 54 | 55 | int fd = open_fd(path); 56 | if (fd < 0) 57 | { 58 | exit(1); 59 | } 60 | 61 | status = nvm_ctrl_init(&ctrl, fd); 62 | if (status != 0) 63 | { 64 | close(fd); 65 | fprintf(stderr, "Failed to get controller reference: %s\n", strerror(status)); 66 | exit(1); 67 | } 68 | 69 | close(fd); 70 | 71 | status = posix_memalign(&memory, ctrl->page_size, 3 * page_size); 72 | if (status != 0) 73 | { 74 | fprintf(stderr, "Failed to allocate page-aligned memory: %s\n", strerror(status)); 75 | nvm_ctrl_free(ctrl); 76 | exit(2); 77 | } 78 | 79 | status = nvm_dma_map_host(&window, ctrl, memory, 3 * page_size); 80 | if (status != 0) 81 | { 82 | free(memory); 83 | nvm_ctrl_free(ctrl); 84 | exit(1); 85 | } 86 | 87 | admin = reset_ctrl(ctrl, window); 88 | if (admin == NULL) 89 | { 90 | status = 1; 91 | goto leave; 92 | } 93 | 94 | status = identify_ctrl(admin, NVM_DMA_OFFSET(window, 2), window->ioaddrs[2]); 95 | if (status != 0) 96 | { 97 | goto leave; 98 | } 99 | 100 | if (nvm_namespace != 0) 101 | { 102 | status = identify_ns(admin, nvm_namespace, NVM_DMA_OFFSET(window, 2), window->ioaddrs[2]); 103 | } 104 | 105 | leave: 106 | nvm_aq_destroy(admin); 107 | nvm_dma_unmap(window); 108 | free(memory); 109 | nvm_ctrl_free(ctrl); 110 | 111 | fprintf(stderr, "Goodbye!\n"); 112 | exit(status); 113 | } 114 | 115 | 116 | static void give_usage(const char* name) 117 | { 118 | fprintf(stderr, "Usage: %s --ctrl=\n", name); 119 | } 120 | 121 | 122 | static void show_help(const char* name) 123 | { 124 | give_usage(name); 125 | fprintf(stderr, "\nCreate a manager and run an IDENTIFY CONTROLLER NVM admin command.\n\n" 126 | " --ctrl Path to controller device (/dev/libnvmXXX).\n" 127 | " --ns Show information about NVM namespace.\n" 128 | " --help Show this information.\n\n"); 129 | } 130 | 131 | 132 | static void parse_args(int argc, char** argv, char** dev, uint32_t* ns_id) 133 | { 134 | static struct option opts[] = { 135 | { "help", no_argument, NULL, 'h' }, 136 | { "ctrl", required_argument, NULL, 'c' }, 137 | { "ns", required_argument, NULL, 'n' }, 138 | { NULL, 0, NULL, 0 } 139 | }; 140 | 141 | int opt; 142 | int idx; 143 | char* endptr = NULL; 144 | 145 | *dev = NULL; 146 | *ns_id = 0; 147 | 148 | while ((opt = getopt_long(argc, argv, ":hc:n:", opts, &idx)) != -1) 149 | { 150 | switch (opt) 151 | { 152 | case '?': // unknown option 153 | fprintf(stderr, "Unknown option: `%s'\n", argv[optind - 1]); 154 | give_usage(argv[0]); 155 | exit('?'); 156 | 157 | case ':': // missing option argument 158 | fprintf(stderr, "Missing argument for option: `%s'\n", argv[optind - 1]); 159 | give_usage(argv[0]); 160 | exit(':'); 161 | 162 | case 'c': // device identifier 163 | *dev = optarg; 164 | break; 165 | 166 | case 'n': // namespace identifier 167 | *ns_id = strtoul(optarg, &endptr, 0); 168 | if (endptr == NULL || *endptr != '\0') 169 | { 170 | fprintf(stderr, "Invalid NVM namespace"); 171 | give_usage(argv[0]); 172 | exit('n'); 173 | } 174 | break; 175 | 176 | case 'h': 177 | show_help(argv[0]); 178 | exit(0); 179 | } 180 | } 181 | 182 | if (*dev == NULL) 183 | { 184 | fprintf(stderr, "Controller is not set!\n"); 185 | give_usage(argv[0]); 186 | exit('c'); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /examples/identify/smartio.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "common.h" 19 | 20 | 21 | /* 22 | * Command line arguments. 23 | */ 24 | struct cl_args 25 | { 26 | uint64_t dev_id; 27 | uint32_t dis_adapter; 28 | uint32_t segment_id; 29 | uint32_t namespace_id; 30 | }; 31 | 32 | 33 | static void parse_args(int argc, char** argv, struct cl_args* args); 34 | 35 | 36 | int main(int argc, char** argv) 37 | { 38 | sci_error_t err; 39 | struct cl_args args; 40 | 41 | parse_args(argc, argv, &args); 42 | 43 | SCIInitialize(0, &err); 44 | 45 | nvm_ctrl_t* ctrl; 46 | int status = nvm_dis_ctrl_init(&ctrl, args.dev_id, args.dis_adapter); 47 | if (status != 0) 48 | { 49 | fprintf(stderr, "Failed to initialize controller reference: %s\n", strerror(status)); 50 | exit(status); 51 | } 52 | 53 | nvm_dma_t* window; 54 | status = nvm_dis_dma_create(&window, ctrl, args.dis_adapter, args.segment_id, 3 * 0x1000); 55 | if (status != 0) 56 | { 57 | nvm_ctrl_free(ctrl); 58 | fprintf(stderr, "Failed to create local segment: %s\n", strerror(status)); 59 | exit(status); 60 | } 61 | memset(window->vaddr, 0, 3 * 0x1000); 62 | 63 | nvm_aq_ref aq; 64 | aq = reset_ctrl(ctrl, window); 65 | if (aq == NULL) 66 | { 67 | status = 1; 68 | goto leave; 69 | } 70 | 71 | status = identify_ctrl(aq, NVM_DMA_OFFSET(window, 2), window->ioaddrs[2]); 72 | if (status != 0) 73 | { 74 | goto leave; 75 | } 76 | 77 | if (args.namespace_id != 0) 78 | { 79 | status = identify_ns(aq, args.namespace_id, NVM_DMA_OFFSET(window, 2), window->ioaddrs[2]); 80 | } 81 | 82 | leave: 83 | nvm_aq_destroy(aq); 84 | nvm_dma_unmap(window); 85 | nvm_ctrl_free(ctrl); 86 | SCITerminate(); 87 | 88 | fprintf(stderr, "Goodbye!\n"); 89 | exit(status); 90 | } 91 | 92 | 93 | static int parse_u64(const char* str, uint64_t* num, int base) 94 | { 95 | char* endptr = NULL; 96 | uint64_t ul = strtoul(str, &endptr, base); 97 | 98 | if (endptr == NULL || *endptr != '\0') 99 | { 100 | return EINVAL; 101 | } 102 | 103 | *num = ul; 104 | return 0; 105 | } 106 | 107 | 108 | static int parse_u32(const char* str, uint32_t* num, int base) 109 | { 110 | int status; 111 | uint64_t ul; 112 | 113 | status = parse_u64(str, &ul, base); 114 | 115 | if (status != 0 || ul > UINT_MAX) 116 | { 117 | return EINVAL; 118 | } 119 | 120 | *num = (uint32_t) ul; 121 | return status; 122 | } 123 | 124 | 125 | static void give_usage(const char* name) 126 | { 127 | fprintf(stderr, "Usage: %s --ctrl= [--adapter=] [--id=]\n", name); 128 | } 129 | 130 | 131 | static void show_help(const char* name) 132 | { 133 | give_usage(name); 134 | fprintf(stderr, "\nCreate a manager and run an IDENTIFY CONTROLLER NVM admin command.\n\n" 135 | " --ctrl SmartIO device identifier (fabric device id).\n" 136 | " --ns Show information about NVM namespace.\n" 137 | " --adapter DIS adapter number (defaults to 0).\n" 138 | " --segment SISCI segment identifier (defaults to 0).\n" 139 | " --help Show this information.\n\n"); 140 | } 141 | 142 | 143 | 144 | 145 | static void parse_args(int argc, char** argv, struct cl_args* args) 146 | { 147 | static struct option opts[] = { 148 | { "help", no_argument, NULL, 'h' }, 149 | { "ctrl", required_argument, NULL, 'c' }, 150 | { "adapter", required_argument, NULL, 'a' }, 151 | { "ns", required_argument, NULL, 'n' }, 152 | { "segment", required_argument, NULL, 's' }, 153 | { NULL, 0, NULL, 0 } 154 | }; 155 | 156 | int opt; 157 | int idx; 158 | 159 | bool dev_set = false; 160 | args->dev_id = 0; 161 | args->dis_adapter = 0; 162 | args->segment_id = 0; 163 | args->namespace_id = 0; 164 | 165 | while ((opt = getopt_long(argc, argv, ":hc:a:s:n:", opts, &idx)) != -1) 166 | { 167 | switch (opt) 168 | { 169 | case '?': // unknown option 170 | fprintf(stderr, "Unknown option: `%s'\n", argv[optind - 1]); 171 | give_usage(argv[0]); 172 | exit('?'); 173 | 174 | case ':': // missing option argument 175 | fprintf(stderr, "Missing argument for option: `%s'\n", argv[optind - 1]); 176 | give_usage(argv[0]); 177 | exit(':'); 178 | 179 | case 'c': // device identifier 180 | dev_set = true; 181 | if (parse_u64(optarg, &args->dev_id, 16) != 0) 182 | { 183 | give_usage(argv[0]); 184 | exit('c'); 185 | } 186 | break; 187 | 188 | case 'a': 189 | if (parse_u32(optarg, &args->dis_adapter, 10) != 0) 190 | { 191 | give_usage(argv[0]); 192 | exit('a'); 193 | } 194 | break; 195 | 196 | case 's': 197 | if (parse_u32(optarg, &args->segment_id, 0) != 0) 198 | { 199 | give_usage(argv[0]); 200 | exit('s'); 201 | } 202 | break; 203 | 204 | case 'n': 205 | if (parse_u32(optarg, &args->namespace_id, 0) != 0) 206 | { 207 | give_usage(argv[0]); 208 | exit('n'); 209 | } 210 | break; 211 | 212 | case 'h': 213 | show_help(argv[0]); 214 | exit(0); 215 | } 216 | } 217 | 218 | if (!dev_set) 219 | { 220 | fprintf(stderr, "Device ID is not set!\n"); 221 | give_usage(argv[0]); 222 | exit('c'); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /examples/integrity/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | project (libnvm-samples) 3 | 4 | set (THREADS_PREFER_PTHREAD_FLAG ON) 5 | find_package (Threads REQUIRED) 6 | 7 | make_sample (integrity-smartio integrity-util "integrity.c;transfer.c;util.c") 8 | set_multithread (integrity-smartio) 9 | set_sisci (integrity-smartio) 10 | 11 | 12 | make_sample (integrity-module integrity-util "integrity.c;transfer.c;util.c") 13 | set_multithread (integrity-module) 14 | 15 | 16 | make_samples_choice (integrity-util integrity-smartio integrity-module) 17 | add_custom_target (integrity DEPENDS integrity-util) 18 | -------------------------------------------------------------------------------- /examples/integrity/integrity.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_SAMPLES_INTEGRITY_H__ 2 | #define __LIBNVM_SAMPLES_INTEGRITY_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | /* Memory descriptor */ 10 | struct buffer 11 | { 12 | uint32_t id; 13 | uint32_t adapter; 14 | void* buffer; 15 | nvm_dma_t* dma; 16 | }; 17 | 18 | 19 | /* Queue descriptor */ 20 | struct queue 21 | { 22 | struct buffer qmem; 23 | nvm_queue_t queue; 24 | size_t counter; 25 | }; 26 | 27 | 28 | /* Disk descriptor */ 29 | struct disk 30 | { 31 | size_t page_size; 32 | size_t max_data_size; 33 | uint32_t ns_id; 34 | size_t block_size; 35 | }; 36 | 37 | 38 | int create_buffer(struct buffer* b, nvm_aq_ref, size_t size, uint32_t adapter, uint32_t id); 39 | 40 | 41 | void remove_buffer(struct buffer* b); 42 | 43 | 44 | 45 | int create_queue(struct queue* q, nvm_aq_ref ref, const struct queue* cq, uint16_t qno, uint32_t adapter, uint32_t id); 46 | 47 | 48 | void remove_queue(struct queue* q); 49 | 50 | 51 | 52 | int disk_write(const struct disk* disk, struct buffer* buffer, struct queue* queues, uint16_t n_queues, FILE* fp, off_t size); 53 | 54 | int disk_read(const struct disk* disk, struct buffer* buffer, struct queue* queues, uint16_t n_queues, FILE* fp, off_t size); 55 | 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /examples/integrity/transfer.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "integrity.h" 18 | 19 | 20 | struct consumer 21 | { 22 | pthread_t thread; 23 | struct queue* queues; 24 | uint16_t n_queues; 25 | bool cancel; 26 | }; 27 | 28 | 29 | struct producer 30 | { 31 | pthread_t thread; 32 | bool write; 33 | struct buffer* buffer; 34 | uint16_t queue_no; 35 | struct queue* queues; 36 | const struct disk* disk; 37 | size_t start_block; 38 | size_t n_blocks; 39 | }; 40 | 41 | 42 | static struct consumer* consume_completions(struct consumer* c) 43 | { 44 | nvm_cpl_t* cpl; 45 | nvm_queue_t* cq = &c->queues[0].queue; 46 | nvm_queue_t* sq = NULL; 47 | 48 | while (!c->cancel) 49 | { 50 | cpl = nvm_cq_dequeue_block(cq, 10); 51 | 52 | if (cpl == NULL) 53 | { 54 | sched_yield(); 55 | continue; 56 | } 57 | 58 | sq = &c->queues[*NVM_CPL_SQID(cpl)].queue; 59 | nvm_sq_update(sq); 60 | 61 | if (!NVM_ERR_OK(cpl)) 62 | { 63 | fprintf(stderr, "%s\n", nvm_strerror(NVM_ERR_STATUS(cpl))); 64 | } 65 | 66 | nvm_cq_update(cq); 67 | c->queues[0].counter++; 68 | } 69 | 70 | return c; 71 | } 72 | 73 | 74 | static struct producer* produce_commands(struct producer* p) 75 | { 76 | nvm_cmd_t* cmd; 77 | size_t block_size = p->disk->block_size; 78 | size_t page_size = p->buffer->dma->page_size; 79 | 80 | size_t n_pages = NVM_PAGE_ALIGN(p->n_blocks * block_size, page_size) / page_size; // FIXME: block to page? 81 | 82 | size_t transfer_pages = p->disk->max_data_size / page_size; 83 | 84 | size_t page_base = NVM_BLOCK_TO_PAGE(page_size, block_size, p->start_block); 85 | size_t page_offset = 0; 86 | 87 | uint32_t ns_id = p->disk->ns_id; 88 | 89 | nvm_dma_t* dma = p->buffer->dma; 90 | struct queue* queue = &p->queues[p->queue_no]; 91 | nvm_dma_t* prp = queue->qmem.dma; 92 | 93 | nvm_queue_t* sq = &queue->queue; 94 | 95 | while (page_offset < n_pages) 96 | { 97 | if (n_pages - page_offset < transfer_pages) 98 | { 99 | transfer_pages = n_pages - page_offset; 100 | } 101 | 102 | while ((cmd = nvm_sq_enqueue(sq)) == NULL) 103 | { 104 | nvm_sq_submit(sq); 105 | sched_yield(); 106 | } 107 | 108 | nvm_cmd_header(cmd, NVM_DEFAULT_CID(sq), p->write ? NVM_IO_WRITE : NVM_IO_READ, ns_id); 109 | 110 | size_t n_blocks = NVM_PAGE_TO_BLOCK(page_size, block_size, transfer_pages); 111 | size_t start_block = p->start_block + NVM_PAGE_TO_BLOCK(page_size, block_size, page_offset); 112 | nvm_cmd_rw_blks(cmd, start_block, n_blocks); 113 | 114 | uint16_t prp_no = (*NVM_CMD_CID(cmd) % sq->max_entries) + 1; 115 | 116 | nvm_cmd_data(cmd, page_size, transfer_pages, NVM_DMA_OFFSET(prp, prp_no), 117 | prp->ioaddrs[prp_no], &dma->ioaddrs[page_base+page_offset]); 118 | 119 | page_offset += transfer_pages; 120 | queue->counter++; 121 | } 122 | 123 | if (p->write) 124 | { 125 | while ((cmd = nvm_sq_enqueue(sq)) == NULL); 126 | nvm_cmd_header(cmd, NVM_DEFAULT_CID(sq), NVM_IO_FLUSH, ns_id); 127 | nvm_cmd_data_ptr(cmd, 0, 0); 128 | nvm_cmd_rw_blks(cmd, 0, 0); 129 | queue->counter++; 130 | } 131 | 132 | nvm_sq_submit(sq); 133 | 134 | return p; 135 | } 136 | 137 | 138 | 139 | static int transfer(const struct disk* d, struct buffer* buffer, struct queue* queues, uint16_t n_queues, off_t size, bool write) 140 | { 141 | size_t n_blocks = NVM_PAGE_ALIGN(size, d->block_size) / d->block_size; 142 | size_t n_pages = NVM_PAGE_ALIGN(size, d->page_size) / d->page_size; 143 | 144 | size_t pages_per_queue = n_pages / n_queues; 145 | 146 | struct producer* producers = calloc(n_queues, sizeof(struct producer)); 147 | if (producers == NULL) 148 | { 149 | fprintf(stderr, "Failed to allocate thread descriptors\n"); 150 | return -1; 151 | } 152 | 153 | struct consumer consumer; 154 | consumer.queues = queues; 155 | consumer.n_queues = n_queues; 156 | consumer.cancel = false; 157 | 158 | pthread_create(&consumer.thread, NULL, (void *(*)(void*)) consume_completions, &consumer); 159 | 160 | for (uint16_t i = 0; i < n_queues; ++i) 161 | { 162 | producers[i].write = write; 163 | producers[i].buffer = buffer; 164 | producers[i].queue_no = i + 1; 165 | producers[i].queues = queues; 166 | producers[i].disk = d; 167 | producers[i].start_block = NVM_PAGE_TO_BLOCK(d->page_size, d->block_size, pages_per_queue * i); 168 | producers[i].n_blocks = NVM_PAGE_TO_BLOCK(d->page_size, d->block_size, pages_per_queue); 169 | 170 | if (i == n_queues - 1) 171 | { 172 | producers[i].n_blocks = n_blocks - producers[i].start_block; 173 | } 174 | 175 | pthread_create(&producers[i].thread, NULL, (void *(*)(void*)) produce_commands, &producers[i]); 176 | fprintf(stderr, "\tQueue #%u: block %zu to block %zu (page %zu + %zu)\n", 177 | i, producers[i].start_block, producers[i].start_block + producers[i].n_blocks, 178 | NVM_BLOCK_TO_PAGE(d->page_size, d->block_size, producers[i].start_block), 179 | NVM_PAGE_ALIGN(producers[i].n_blocks * d->block_size, d->page_size) / d->page_size); 180 | 181 | } 182 | 183 | size_t commands = 0; 184 | for (uint16_t i = 0; i < n_queues; ++i) 185 | { 186 | struct producer* p; 187 | pthread_join(producers[i].thread, (void**) &p); 188 | commands += queues[p->queue_no].counter; 189 | } 190 | 191 | while (queues[0].counter < commands); 192 | 193 | consumer.cancel = true; 194 | pthread_join(consumer.thread, NULL); 195 | fprintf(stderr, "Total blocks: %zu\n", n_blocks); 196 | 197 | 198 | free(producers); 199 | return 0; 200 | } 201 | 202 | 203 | 204 | int disk_write(const struct disk* d, struct buffer* buffer, struct queue* queues, uint16_t n_queues, FILE* fp, off_t size) 205 | { 206 | fread(buffer->dma->vaddr, 1, size, fp); 207 | return transfer(d, buffer, queues, n_queues, size, true); 208 | } 209 | 210 | 211 | int disk_read(const struct disk* d, struct buffer* buffer, struct queue* queues, uint16_t n_queues, FILE* fp, off_t size) 212 | { 213 | int status = transfer(d, buffer, queues, n_queues, size, false); 214 | if (status == 0) 215 | { 216 | fwrite(buffer->dma->vaddr, 1, size, fp); 217 | fflush(fp); 218 | } 219 | return status; 220 | } 221 | 222 | -------------------------------------------------------------------------------- /examples/integrity/util.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "integrity.h" 12 | 13 | 14 | int create_buffer(struct buffer* b, nvm_aq_ref ref, size_t size, uint32_t adapter, uint32_t id) 15 | { 16 | int status; 17 | 18 | const nvm_ctrl_t* ctrl = nvm_ctrl_from_aq_ref(ref); 19 | 20 | #ifdef __DIS_CLUSTER__ 21 | b->buffer = NULL; 22 | status = nvm_dis_dma_create(&b->dma, ctrl, adapter, id, size); 23 | #else 24 | status = posix_memalign(&b->buffer, ctrl->page_size, size); 25 | if (status != 0) 26 | { 27 | fprintf(stderr, "Failed to allocate memory: %s\n", strerror(status)); 28 | return status; 29 | } 30 | 31 | status = nvm_dma_map_host(&b->dma, ctrl, b->buffer, size); 32 | #endif 33 | if (!nvm_ok(status)) 34 | { 35 | free(b->buffer); 36 | fprintf(stderr, "Failed to create local segment: %s\n", nvm_strerror(status)); 37 | return status; 38 | } 39 | 40 | memset(b->dma->vaddr, 0, b->dma->page_size * b->dma->n_ioaddrs); 41 | 42 | b->id = id; 43 | b->adapter = adapter; 44 | 45 | return 0; 46 | } 47 | 48 | 49 | void remove_buffer(struct buffer* b) 50 | { 51 | nvm_dma_unmap(b->dma); 52 | free(b->buffer); 53 | } 54 | 55 | 56 | int create_queue(struct queue* q, nvm_aq_ref ref, const struct queue* cq, uint16_t qno, uint32_t adapter, uint32_t id) 57 | { 58 | int status; 59 | 60 | const nvm_ctrl_t* ctrl = nvm_ctrl_from_aq_ref(ref); 61 | 62 | size_t prp_lists = 0; 63 | if (cq != NULL) 64 | { 65 | size_t n_entries = ctrl->page_size / sizeof(nvm_cmd_t); 66 | prp_lists = n_entries <= ctrl->max_entries ? n_entries : ctrl->max_entries; 67 | } 68 | 69 | status = create_buffer(&q->qmem, ref, prp_lists * ctrl->page_size + ctrl->page_size, adapter, id); 70 | if (!nvm_ok(status)) 71 | { 72 | return status; 73 | } 74 | 75 | if (cq == NULL) 76 | { 77 | status = nvm_admin_cq_create(ref, &q->queue, qno, NVM_DMA_OFFSET(q->qmem.dma, 0), q->qmem.dma->ioaddrs[0]); 78 | } 79 | else 80 | { 81 | status = nvm_admin_sq_create(ref, &q->queue, &cq->queue, qno, NVM_DMA_OFFSET(q->qmem.dma, 0), q->qmem.dma->ioaddrs[0]); 82 | } 83 | 84 | if (!nvm_ok(status)) 85 | { 86 | remove_buffer(&q->qmem); 87 | fprintf(stderr, "Failed to create queue: %s\n", nvm_strerror(status)); 88 | return status; 89 | } 90 | 91 | q->counter = 0; 92 | return 0; 93 | } 94 | 95 | 96 | void remove_queue(struct queue* q) 97 | { 98 | remove_buffer(&q->qmem); 99 | } 100 | 101 | -------------------------------------------------------------------------------- /examples/read-blocks/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | project (libnvm-samples) 3 | 4 | set (THREADS_PREFER_PTHREAD_FLAG ON) 5 | find_package (Threads REQUIRED) 6 | 7 | make_sample (read-blocks-module read-blocks "module.c;args.c;read.c") 8 | set_multithread (read-blocks-module) 9 | 10 | make_sample (read-blocks-smartio read-blocks "smartio.c;args.c;read.c") 11 | set_multithread (read-blocks-smartio) 12 | set_sisci (read-blocks-smartio) 13 | 14 | make_samples_choice (read-blocks read-blocks-smartio read-blocks-module) 15 | -------------------------------------------------------------------------------- /examples/read-blocks/args.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_SAMPLES_READ_BLOCKS_OPTIONS_H__ 2 | #define __LIBNVM_SAMPLES_READ_BLOCKS_OPTIONS_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | struct options 11 | { 12 | #ifdef __DIS_CLUSTER__ 13 | uint64_t controller_id; 14 | uint32_t adapter; 15 | uint32_t segment_id; // TODO: FIXME: This needs to be memory region for sq_mem instead 16 | #else 17 | const char* controller_path; 18 | #endif 19 | size_t chunk_size; 20 | uint32_t namespace_id; 21 | size_t num_blocks; 22 | size_t offset; 23 | FILE* output; 24 | FILE* input; 25 | bool ascii; 26 | bool identify; 27 | }; 28 | 29 | 30 | void parse_options(int argc, char** argv, struct options* options); 31 | 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /examples/read-blocks/module.c: -------------------------------------------------------------------------------- 1 | #include "args.h" 2 | #include "read.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | 19 | static int prepare_and_read(nvm_aq_ref ref, const struct disk_info* disk, const struct options* args) 20 | { 21 | int status = 0; 22 | 23 | void* buffer_ptr = NULL; 24 | nvm_dma_t* buffer = NULL; 25 | void* queue_ptr = NULL; 26 | nvm_dma_t* sq_mem = NULL; 27 | nvm_dma_t* cq_mem = NULL; 28 | size_t n_prp_lists = disk->page_size / sizeof(nvm_cmd_t); 29 | struct queue_pair queues; 30 | 31 | const nvm_ctrl_t* ctrl = nvm_ctrl_from_aq_ref(ref); 32 | 33 | status = posix_memalign(&buffer_ptr, disk->page_size, NVM_CTRL_ALIGN(ctrl, args->num_blocks * disk->block_size)); 34 | if (status != 0) 35 | { 36 | fprintf(stderr, "Failed to allocate memory buffer: %s\n", strerror(status)); 37 | goto leave; 38 | } 39 | 40 | status = posix_memalign(&queue_ptr, disk->page_size, disk->page_size * (n_prp_lists + 2)); 41 | if (status != 0) 42 | { 43 | fprintf(stderr, "Failed to allocate queue memory: %s\n", strerror(status)); 44 | goto leave; 45 | } 46 | 47 | status = nvm_dma_map_host(&sq_mem, ctrl, NVM_PTR_OFFSET(queue_ptr, disk->page_size, 1), disk->page_size * (n_prp_lists + 1)); 48 | if (!nvm_ok(status)) 49 | { 50 | fprintf(stderr, "Failed to map memory for controller: %s\n", nvm_strerror(status)); 51 | goto leave; 52 | } 53 | 54 | status = nvm_dma_map_host(&cq_mem, ctrl, queue_ptr, disk->page_size); 55 | if (!nvm_ok(status)) 56 | { 57 | fprintf(stderr, "Failed to map memory for controller: %s\n", nvm_strerror(status)); 58 | goto leave; 59 | } 60 | 61 | status = nvm_dma_map_host(&buffer, ctrl, buffer_ptr, args->num_blocks * disk->block_size); 62 | if (!nvm_ok(status)) 63 | { 64 | fprintf(stderr, "Failed to map memory for controller: %s\n", nvm_strerror(status)); 65 | goto leave; 66 | } 67 | 68 | status = create_queue_pair(ref, &queues, cq_mem, sq_mem); 69 | if (status != 0) 70 | { 71 | goto leave; 72 | } 73 | 74 | if (args->input != NULL) 75 | { 76 | status = write_blocks(disk, &queues, buffer, args); 77 | if (status != 0) 78 | { 79 | goto leave; 80 | } 81 | } 82 | 83 | status = read_and_dump(disk, &queues, buffer, args); 84 | 85 | leave: 86 | nvm_dma_unmap(buffer); 87 | nvm_dma_unmap(sq_mem); 88 | nvm_dma_unmap(cq_mem); 89 | free(buffer_ptr); 90 | free(queue_ptr); 91 | return status; 92 | } 93 | 94 | 95 | 96 | int main(int argc, char** argv) 97 | { 98 | int status; 99 | int fd; 100 | 101 | struct disk_info disk; 102 | 103 | nvm_ctrl_t* ctrl = NULL; 104 | void* aq_ptr = NULL; 105 | nvm_dma_t* aq_mem = NULL; 106 | nvm_aq_ref aq_ref = NULL; 107 | 108 | struct options args; 109 | 110 | // Parse arguments from command line 111 | parse_options(argc, argv, &args); 112 | 113 | // Get controller reference 114 | fd = open(args.controller_path, O_RDWR | O_NONBLOCK); 115 | if (fd < 0) 116 | { 117 | fprintf(stderr, "Failed to open file descriptor: %s\n", strerror(errno)); 118 | exit(1); 119 | } 120 | 121 | status = nvm_ctrl_init(&ctrl, fd); 122 | if (!nvm_ok(status)) 123 | { 124 | fprintf(stderr, "Failed to initialize controller reference: %s\n", nvm_strerror(status)); 125 | goto leave; 126 | } 127 | 128 | // Create admin queue pair + page for identify commands 129 | status = posix_memalign(&aq_ptr, ctrl->page_size, ctrl->page_size * 3); 130 | if (status != 0) 131 | { 132 | fprintf(stderr, "Failed to allocate queue memory: %s\n", strerror(status)); 133 | goto leave; 134 | } 135 | 136 | status = nvm_dma_map_host(&aq_mem, ctrl, aq_ptr, ctrl->page_size * 3); 137 | if (!nvm_ok(status)) 138 | { 139 | fprintf(stderr, "Failed to map memory for controller: %s\n", nvm_strerror(status)); 140 | goto leave; 141 | } 142 | 143 | // Reset controller and set admin queues 144 | status = nvm_aq_create(&aq_ref, ctrl, aq_mem); 145 | if (!nvm_ok(status)) 146 | { 147 | fprintf(stderr, "Failed to reset controller: %s\n", nvm_strerror(status)); 148 | goto leave; 149 | } 150 | 151 | // Identify controller and namespace 152 | status = get_disk_info(aq_ref, &disk, args.namespace_id, NVM_DMA_OFFSET(aq_mem, 2), aq_mem->ioaddrs[2], args.identify); 153 | if (status != 0) 154 | { 155 | goto leave; 156 | } 157 | 158 | status = prepare_and_read(aq_ref, &disk, &args); 159 | 160 | leave: 161 | if (args.input != NULL) 162 | { 163 | fclose(args.input); 164 | } 165 | 166 | if (args.output != NULL) 167 | { 168 | fprintf(stderr, "Flushing output file...\n"); 169 | fclose(args.output); 170 | } 171 | 172 | fprintf(stderr, "Done\n"); 173 | 174 | nvm_aq_destroy(aq_ref); 175 | nvm_dma_unmap(aq_mem); 176 | free(aq_ptr); 177 | nvm_ctrl_free(ctrl); 178 | close(fd); 179 | exit(status); 180 | } 181 | -------------------------------------------------------------------------------- /examples/read-blocks/read.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_SAMPLES_READ_BLOCKS_READ_H__ 2 | #define __LIBNVM_SAMPLES_READ_BLOCKS_READ_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "args.h" 8 | 9 | 10 | /* 11 | * Information about controller and namespace. 12 | */ 13 | struct disk_info 14 | { 15 | uint32_t ns_id; 16 | size_t max_data_size; 17 | size_t page_size; 18 | size_t block_size; 19 | }; 20 | 21 | 22 | 23 | struct queue_pair 24 | { 25 | nvm_dma_t* sq_mem; 26 | nvm_dma_t* cq_mem; 27 | nvm_queue_t sq; 28 | nvm_queue_t cq; 29 | bool stop; 30 | size_t num_cpls; 31 | }; 32 | 33 | 34 | 35 | int get_disk_info(nvm_aq_ref ref, struct disk_info* info, uint32_t ns_id, void* ptr, uint64_t ioaddr, bool show); 36 | 37 | 38 | int create_queue_pair(nvm_aq_ref ref, struct queue_pair* qp, nvm_dma_t* cq_mem, nvm_dma_t* sq_mem); 39 | 40 | 41 | int read_and_dump(const struct disk_info* disk, struct queue_pair* qp, const nvm_dma_t* buffer, const struct options* args); 42 | 43 | 44 | int write_blocks(const struct disk_info* disk, struct queue_pair* qp, const nvm_dma_t* buffer, const struct options* args); 45 | 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /examples/read-blocks/smartio.c: -------------------------------------------------------------------------------- 1 | #include "args.h" 2 | #include "read.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | 18 | int main(int argc, char** argv) 19 | { 20 | int status; 21 | sci_error_t err; 22 | 23 | struct disk_info info; 24 | struct queue_pair queues; 25 | 26 | nvm_ctrl_t* ctrl = NULL; 27 | nvm_dma_t* aq_mem = NULL; 28 | nvm_aq_ref aq_ref = NULL; 29 | nvm_dma_t* buffer = NULL; 30 | nvm_dma_t* cq_mem = NULL; 31 | nvm_dma_t* sq_mem = NULL; 32 | 33 | struct options args; 34 | 35 | parse_options(argc, argv, &args); 36 | 37 | SCIInitialize(0, &err); 38 | if (err != SCI_ERR_OK) 39 | { 40 | fprintf(stderr, "Something went wrong: %s\n", SCIGetErrorString(err)); 41 | exit(1); 42 | } 43 | 44 | // Get controller reference 45 | status = nvm_dis_ctrl_init(&ctrl, args.controller_id, args.adapter); 46 | if (!nvm_ok(status)) 47 | { 48 | fprintf(stderr, "Failed to get controller reference: %s\n", nvm_strerror(status)); 49 | goto leave; 50 | } 51 | 52 | // Create admin queue pair + page for identify commands 53 | status = nvm_dis_dma_create(&aq_mem, ctrl, args.adapter, args.segment_id++, ctrl->page_size * 3); 54 | if (!nvm_ok(status)) 55 | { 56 | fprintf(stderr, "Failed to create queue memory: %s\n", nvm_strerror(status)); 57 | goto leave; 58 | } 59 | 60 | // Reset controller and set admin queue pair 61 | status = nvm_aq_create(&aq_ref, ctrl, aq_mem); 62 | if (!nvm_ok(status)) 63 | { 64 | fprintf(stderr, "Failed to reset controller: %s\n", nvm_strerror(status)); 65 | goto leave; 66 | } 67 | 68 | // Identify controller and namespace 69 | status = get_disk_info(aq_ref, &info, args.namespace_id, NVM_DMA_OFFSET(aq_mem, 2), aq_mem->ioaddrs[2], args.identify); 70 | if (status != 0) 71 | { 72 | goto leave; 73 | } 74 | 75 | // Create data buffer 76 | size_t buffer_size = (args.chunk_size <= args.num_blocks ? args.chunk_size : args.num_blocks) * info.block_size; 77 | status = nvm_dis_dma_create(&buffer, ctrl, args.adapter, args.segment_id++, buffer_size); 78 | if (!nvm_ok(status)) 79 | { 80 | fprintf(stderr, "Failed to create data buffer: %s\n", nvm_strerror(status)); 81 | goto leave; 82 | } 83 | 84 | // Create memory for completion queue 85 | status = nvm_dis_dma_create(&cq_mem, ctrl, args.adapter, args.segment_id++, ctrl->page_size); 86 | if (!nvm_ok(status)) 87 | { 88 | fprintf(stderr, "Failed to create completion queue memory: %s\n", nvm_strerror(status)); 89 | goto leave; 90 | } 91 | 92 | // Create memory for submission queue and PRP lists 93 | size_t n_prp_lists = ctrl->page_size / sizeof(nvm_cmd_t); 94 | if (ctrl->max_entries < n_prp_lists) 95 | { 96 | n_prp_lists = ctrl->max_entries; 97 | } 98 | 99 | // FIXME: make this false when supported as well as using args.segment_id 100 | status = nvm_dis_dma_connect(&sq_mem, ctrl, args.adapter, 1, ctrl->page_size * (n_prp_lists + 1), true); 101 | if (!nvm_ok(status)) 102 | { 103 | goto leave; 104 | } 105 | 106 | // Create queues 107 | status = create_queue_pair(aq_ref, &queues, cq_mem, sq_mem); 108 | if (!nvm_ok(status)) 109 | { 110 | goto leave; 111 | } 112 | 113 | if (args.input != NULL) 114 | { 115 | status = write_blocks(&info, &queues, buffer, &args); 116 | if (status != 0) 117 | { 118 | goto leave; 119 | } 120 | } 121 | 122 | status = read_and_dump(&info, &queues, buffer, &args); 123 | 124 | 125 | leave: 126 | if (args.input != NULL) 127 | { 128 | fclose(args.input); 129 | } 130 | 131 | if (args.output != NULL) 132 | { 133 | fprintf(stderr, "Flushing output file...\n"); 134 | fclose(args.output); 135 | } 136 | fprintf(stderr, "Done\n"); 137 | nvm_dma_unmap(cq_mem); 138 | nvm_dma_unmap(sq_mem); 139 | nvm_dma_unmap(buffer); 140 | nvm_aq_destroy(aq_ref); 141 | nvm_dma_unmap(aq_mem); 142 | nvm_ctrl_free(ctrl); 143 | SCITerminate(); 144 | exit(status); 145 | } 146 | -------------------------------------------------------------------------------- /examples/rpc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.1) 2 | project (libnvm-samples) 3 | 4 | set (THREADS_PREFER_PTHREAD_FLAG ON) 5 | find_package (Threads REQUIRED) 6 | 7 | make_sample (rpc-server rpc-server "rpc_server.c;util.c;segment.c") 8 | set_sisci (rpc-server) 9 | 10 | make_sample (rpc-identify rpc-identify "rpc_identify.c;util.c;segment.c") 11 | set_sisci (rpc-identify) 12 | 13 | make_sample (rpc-dd rpc-dd "rpc_dd.c;util.c;segment.c") 14 | set_sisci (rpc-dd) 15 | 16 | add_custom_target(rpc DEPENDS rpc-server rpc-identify rpc-dd) 17 | add_custom_target(rpc-clients DEPENDS rpc-identify rpc-dd) 18 | -------------------------------------------------------------------------------- /examples/rpc/segment.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "segment.h" 11 | 12 | 13 | int segment_create(struct segment* segment, uint32_t segment_id, size_t size) 14 | { 15 | sci_error_t err; 16 | sci_error_t status; 17 | 18 | SCIOpen(&segment->sd, 0, &err); 19 | if (err != SCI_ERR_OK) 20 | { 21 | return EIO; 22 | } 23 | 24 | SCICreateSegment(segment->sd, &segment->segment, segment_id, size, NULL, NULL, 0, &err); 25 | if (err != SCI_ERR_OK) 26 | { 27 | SCIClose(segment->sd, 0, &status); 28 | 29 | if (err == SCI_ERR_SEGMENTID_USED) 30 | { 31 | return EEXIST; 32 | } 33 | 34 | return ENOSPC; 35 | } 36 | 37 | segment->id = segment_id; 38 | segment->size = size; 39 | return 0; 40 | } 41 | 42 | 43 | void segment_remove(struct segment* segment) 44 | { 45 | sci_error_t err; 46 | 47 | do 48 | { 49 | SCIRemoveSegment(segment->segment, 0, &err); 50 | } 51 | while (err == SCI_ERR_BUSY); 52 | 53 | SCIClose(segment->sd, 0, &err); 54 | } 55 | 56 | 57 | int dma_create(nvm_dma_t** window, const nvm_ctrl_t* ctrl, struct segment* segment, uint32_t adapter) 58 | { 59 | sci_error_t err; 60 | 61 | SCIPrepareSegment(segment->segment, adapter, 0, &err); 62 | if (err != SCI_ERR_OK) 63 | { 64 | return ENOSPC; 65 | } 66 | 67 | SCISetSegmentAvailable(segment->segment, adapter, 0, &err); 68 | if (err != SCI_ERR_OK) 69 | { 70 | return EIO; 71 | } 72 | 73 | int status = nvm_dis_dma_map_local(window, ctrl, adapter, segment->segment, true); 74 | if (status != 0) 75 | { 76 | do 77 | { 78 | SCISetSegmentUnavailable(segment->segment, adapter, 0, &err); 79 | } 80 | while (err == SCI_ERR_BUSY); 81 | 82 | return status; 83 | } 84 | 85 | return 0; 86 | } 87 | 88 | 89 | void dma_remove(nvm_dma_t* window, struct segment* segment, uint32_t adapter) 90 | { 91 | sci_error_t err; 92 | 93 | nvm_dma_unmap(window); 94 | 95 | do 96 | { 97 | SCISetSegmentUnavailable(segment->segment, adapter, 0, &err); 98 | } 99 | while (err == SCI_ERR_BUSY); 100 | } 101 | 102 | -------------------------------------------------------------------------------- /examples/rpc/segment.h: -------------------------------------------------------------------------------- 1 | #ifndef __DIS_NVM_EXAMPLES_SEGMENT_H__ 2 | #define __DIS_NVM_EXAMPLES_SEGMENT_H__ 3 | #ifdef __DIS_CLUSTER__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | /* 12 | * Wrapper for SISCI segment descriptors. 13 | */ 14 | struct segment 15 | { 16 | uint32_t id; // Segment ID 17 | sci_desc_t sd; // SISCI virtual device 18 | sci_local_segment_t segment; // Local segment descriptor 19 | size_t size; // Size of segment 20 | }; 21 | 22 | 23 | 24 | /* 25 | * Create a local segment and initialize the wrapper struct. 26 | */ 27 | int segment_create(struct segment* segment, uint32_t segment_id, size_t size); 28 | 29 | 30 | /* 31 | * Remove the local segment. 32 | */ 33 | void segment_remove(struct segment* segment); 34 | 35 | 36 | 37 | /* 38 | * Create a DMA window for the segment. 39 | */ 40 | int dma_create(nvm_dma_t** dma_window, const nvm_ctrl_t* ctrl, struct segment* segment, uint32_t dis_adapter); 41 | 42 | 43 | 44 | /* 45 | * Destroy the DMA window. 46 | */ 47 | void dma_remove(nvm_dma_t* dma_window, struct segment* segment, uint32_t dis_adapter); 48 | 49 | 50 | #endif // __DIS_CLUSTER__ 51 | #endif // __DIS_NVM_EXAMPLES_SEGMENT_H__ 52 | -------------------------------------------------------------------------------- /examples/rpc/util.c: -------------------------------------------------------------------------------- 1 | #include "util.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | int parse_u64(const char* str, uint64_t* num, int base) 14 | { 15 | char* endptr = NULL; 16 | uint64_t ul = strtoul(str, &endptr, base); 17 | 18 | if (endptr == NULL || *endptr != '\0') 19 | { 20 | return EINVAL; 21 | } 22 | 23 | *num = ul; 24 | return 0; 25 | } 26 | 27 | 28 | int parse_u32(const char* str, uint32_t* num, int base) 29 | { 30 | int status; 31 | uint64_t ul; 32 | 33 | status = parse_u64(str, &ul, base); 34 | 35 | if (status != 0 || ul > UINT_MAX) 36 | { 37 | return EINVAL; 38 | } 39 | 40 | *num = (uint32_t) ul; 41 | return status; 42 | } 43 | 44 | 45 | int parse_u16(const char* str, uint16_t* num, int base) 46 | { 47 | int status; 48 | uint64_t ul; 49 | 50 | status = parse_u64(str, &ul, base); 51 | 52 | if (status != 0 || ul > 0xffff) 53 | { 54 | return EINVAL; 55 | } 56 | 57 | *num = (uint16_t) ul; 58 | return status; 59 | } 60 | 61 | 62 | 63 | 64 | uint16_t random_id() 65 | { 66 | static uint16_t unique_counter = 0; 67 | struct timeval tv; 68 | 69 | srand(time(NULL)); 70 | 71 | if (gettimeofday(&tv, NULL) == 0) 72 | { 73 | srand(tv.tv_usec); 74 | } 75 | 76 | return (rand() + unique_counter++) & 0xffff; 77 | } 78 | 79 | 80 | void print_ctrl_info(FILE* fp, const struct nvm_ctrl_info* info) 81 | { 82 | unsigned char vendor[4]; 83 | memcpy(vendor, &info->pci_vendor, sizeof(vendor)); 84 | 85 | char serial[21]; 86 | memset(serial, 0, 21); 87 | memcpy(serial, info->serial_no, 20); 88 | 89 | char model[41]; 90 | memset(model, 0, 41); 91 | memcpy(model, info->model_no, 40); 92 | 93 | char revision[9]; 94 | memset(revision, 0, 9); 95 | memcpy(revision, info->firmware, 8); 96 | 97 | fprintf(fp, "------------- Controller information -------------\n"); 98 | fprintf(fp, "PCI Vendor ID : %x %x\n", vendor[0], vendor[1]); 99 | fprintf(fp, "PCI Subsystem Vendor ID : %x %x\n", vendor[2], vendor[3]); 100 | fprintf(fp, "NVM Express version : %u.%u.%u\n", 101 | info->nvme_version >> 16, (info->nvme_version >> 8) & 0xff, info->nvme_version & 0xff); 102 | fprintf(fp, "Controller page size : %zu\n", info->page_size); 103 | fprintf(fp, "Max queue entries : %u\n", info->max_entries); 104 | fprintf(fp, "Serial Number : %s\n", serial); 105 | fprintf(fp, "Model Number : %s\n", model); 106 | fprintf(fp, "Firmware revision : %s\n", revision); 107 | fprintf(fp, "Max data transfer size : %zu\n", info->max_data_size); 108 | fprintf(fp, "Max outstanding commands: %zu\n", info->max_out_cmds); 109 | fprintf(fp, "Max number of namespaces: %zu\n", info->max_n_ns); 110 | fprintf(fp, "--------------------------------------------------\n"); 111 | } 112 | 113 | 114 | void print_ns_info(FILE* fp, const struct nvm_ns_info* info) 115 | { 116 | fprintf(fp, "------------- Namespace information -------------\n"); 117 | fprintf(fp, "Namespace identifier : %x\n", info->ns_id); 118 | fprintf(fp, "Logical block size : %zu bytes\n", info->lba_data_size); 119 | fprintf(fp, "Namespace size : %zu blocks\n", info->size); 120 | fprintf(fp, "Namespace capacity : %zu blocks\n", info->capacity); 121 | fprintf(fp, "--------------------------------------------------\n"); 122 | } 123 | 124 | -------------------------------------------------------------------------------- /examples/rpc/util.h: -------------------------------------------------------------------------------- 1 | #ifndef __DISNVM_EXAMPLES_UTIL_H__ 2 | #define __DISNVM_EXAMPLES_UTIL_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | /* 11 | * Generate a random identifier. 12 | */ 13 | uint16_t random_id(); 14 | 15 | 16 | /* 17 | * Parse an uint64_t from a string. 18 | */ 19 | int parse_u64(const char* str, uint64_t* number, int base); 20 | 21 | 22 | /* 23 | * Parse an uint32_t from a string. 24 | */ 25 | int parse_u32(const char* str, uint32_t* number, int base); 26 | 27 | 28 | /* 29 | * Parse an uint16_t from a string. 30 | */ 31 | int parse_u16(const char* str, uint16_t* number, int base); 32 | 33 | 34 | /* 35 | * Pretty print controller information. 36 | */ 37 | void print_ctrl_info(FILE* fp, const struct nvm_ctrl_info* info); 38 | 39 | 40 | /* 41 | * Pretty print namespace information. 42 | */ 43 | void print_ns_info(FILE* fp, const struct nvm_ns_info* info); 44 | 45 | #endif // __DISNVM_EXAMPLES_UTIL_H__ 46 | -------------------------------------------------------------------------------- /include/nvm_admin.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_ADMIN_H__ 2 | #define __NVM_ADMIN_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | 14 | /* 15 | * Get controller information. 16 | */ 17 | int nvm_admin_ctrl_info(nvm_aq_ref ref, // AQ pair reference 18 | struct nvm_ctrl_info* info, // Controller information structure 19 | void* buffer, // Temporary buffer (must be at least 4 KB) 20 | uint64_t ioaddr); // Bus address of buffer as seen by the controller 21 | 22 | 23 | 24 | /* 25 | * Get namespace information. 26 | */ 27 | int nvm_admin_ns_info(nvm_aq_ref ref, // AQ pair reference 28 | struct nvm_ns_info* info, // NVM namespace information 29 | uint32_t ns_id, // Namespace identifier 30 | void* buffer, // Temporary buffer (must be at least 4 KB) 31 | uint64_t ioaddr); // Bus address of buffer as seen by controller 32 | 33 | 34 | 35 | /* 36 | * Make controller allocate and reserve queues. 37 | */ 38 | int nvm_admin_set_num_queues(nvm_aq_ref ref, uint16_t n_cqs, uint16_t n_sqs); 39 | 40 | 41 | /* 42 | * Retrieve the number of allocated queues. 43 | */ 44 | int nvm_admin_get_num_queues(nvm_aq_ref ref, uint16_t* n_cqs, uint16_t* n_sqs); 45 | 46 | 47 | /* 48 | * Make controller allocate number of queues before issuing them. 49 | */ 50 | int nvm_admin_request_num_queues(nvm_aq_ref ref, uint16_t* n_cqs, uint16_t* n_sqs); 51 | 52 | 53 | /* 54 | * Create IO completion queue (CQ) 55 | * Caller must set queue memory to zero manually. 56 | */ 57 | int nvm_admin_cq_create(nvm_aq_ref ref, // AQ pair reference 58 | nvm_queue_t* cq, // CQ descriptor 59 | uint16_t id, // Queue identifier 60 | void* qmem, // Queue memory (virtual memory) 61 | uint64_t ioaddr); // Bus address to queue memory as seen by controller 62 | 63 | 64 | /* 65 | * Delete IO completion queue (CQ) 66 | * After calling this, the queue is no longer used and must be recreated. 67 | * All associated submission queues must be deleted first. 68 | */ 69 | int nvm_admin_cq_delete(nvm_aq_ref ref, nvm_queue_t* cq); 70 | 71 | 72 | 73 | /* 74 | * Create IO submission queue (SQ) 75 | * Caller must set queue memory to zero manually. 76 | */ 77 | int nvm_admin_sq_create(nvm_aq_ref ref, // AQ pair reference 78 | nvm_queue_t* sq, // SQ descriptor 79 | const nvm_queue_t* cq, // Descriptor to paired CQ 80 | uint16_t id, // Queue identifier 81 | void* qmem, // Queue memory (virtual) 82 | uint64_t ioaddr); // Bus address to queue as seen by controller 83 | 84 | 85 | 86 | /* 87 | * Delete IO submission queue (SQ) 88 | * After calling this, the queue is no longer used and must be recreated. 89 | */ 90 | int nvm_admin_sq_delete(nvm_aq_ref ref, 91 | nvm_queue_t* sq, 92 | const nvm_queue_t* cq); 93 | 94 | 95 | #ifdef __cplusplus 96 | } 97 | #endif 98 | #endif /* #ifdef __NVM_ADMIN_H__ */ 99 | -------------------------------------------------------------------------------- /include/nvm_aq.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_AQ_H__ 2 | #define __NVM_AQ_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | 14 | /* 15 | * Create admin queue pair 16 | * 17 | * Take exclusive ownership of an NVM controller. This function resets the 18 | * controller and configures NVM admin queues. 19 | * 20 | * Returns a reference handle that can be used for admin RPC calls. 21 | */ 22 | int nvm_aq_create(nvm_aq_ref* ref, const nvm_ctrl_t* ctrl, const nvm_dma_t* aq_mem); 23 | 24 | 25 | 26 | /* 27 | * Destroy admin queues and references. 28 | * 29 | * Send NVM abort command to controller and deallocate admin queues. 30 | * 31 | * After calling this function, all admin queue references are invalid. 32 | * This also means that remote references will no longer be valid. 33 | * 34 | * This function will also work for unbinding remote references. 35 | */ 36 | void nvm_aq_destroy(nvm_aq_ref ref); 37 | 38 | 39 | 40 | //int nvm_tcp_rpc_enable(nvm_aq_ref ref, uint16_t port, nvm_rpc_cb_t filter, void* data); 41 | //int nvm_tcp_rpc_disable(nvm_aq_ref ref, uint16_t port); 42 | 43 | 44 | 45 | #ifdef __DIS_CLUSTER__ 46 | 47 | /* 48 | * Maximum number of adapters. 49 | */ 50 | #define NVM_DIS_RPC_MAX_ADAPTER 4 51 | 52 | 53 | /* 54 | * Callback function invoked whenever a remote NVM admin command is received. 55 | * Should indicate whether or not a remote admin command is accepted and can 56 | * be enqueued by using the return value. 57 | * 58 | * The remote command can also be modified if necessary. 59 | */ 60 | typedef bool (*nvm_dis_rpc_cb_t)(nvm_cmd_t* cmd, uint32_t dis_adapter, uint32_t dis_node_id); 61 | 62 | 63 | 64 | /* 65 | * Enable remote admin commands. 66 | * Allows remote processes to relay NVM admin commands to the local process. 67 | */ 68 | int nvm_dis_rpc_enable(nvm_aq_ref ref, // NVM admin queue-pair reference 69 | uint32_t dis_adapter, // Local adapter to enable interrupt on 70 | nvm_dis_rpc_cb_t filter); // Filter callback (can be NULL) 71 | 72 | 73 | 74 | /* 75 | * Disable remote admin commands. 76 | * Stop processing admin commands from remote processes. 77 | */ 78 | void nvm_dis_rpc_disable(nvm_aq_ref ref, uint32_t dis_adapter); 79 | 80 | #endif /* __DIS_CLUSTER__ */ 81 | 82 | 83 | 84 | #ifdef __cplusplus 85 | } 86 | #endif 87 | #endif /* #ifdef __NVM_AQ_H__ */ 88 | -------------------------------------------------------------------------------- /include/nvm_cmd.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_COMMAND_H__ 2 | #define __NVM_COMMAND_H__ 3 | 4 | #ifndef __CUDACC__ 5 | #define __device__ 6 | #define __host__ 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | 15 | 16 | /* All namespaces identifier */ 17 | #define NVM_CMD_NS_ALL 0xffffffff 18 | 19 | 20 | /* List of NVM IO command opcodes */ 21 | enum nvm_io_command_set 22 | { 23 | NVM_IO_FLUSH = (0x00 << 7) | (0x00 << 2) | 0x00, // 00h 24 | NVM_IO_WRITE = (0x00 << 7) | (0x00 << 2) | 0x01, // 01h 25 | NVM_IO_READ = (0x00 << 7) | (0x00 << 2) | 0x02, // 02h 26 | NVM_IO_WRITE_ZEROES = (0x00 << 7) | (0x02 << 2) | 0x00 // 08h 27 | }; 28 | 29 | 30 | 31 | /* List of NVM admin command opcodes */ 32 | enum nvm_admin_command_set 33 | { 34 | NVM_ADMIN_DELETE_SUBMISSION_QUEUE = (0x00 << 7) | (0x00 << 2) | 0x00, 35 | NVM_ADMIN_CREATE_SUBMISSION_QUEUE = (0x00 << 7) | (0x00 << 2) | 0x01, 36 | NVM_ADMIN_DELETE_COMPLETION_QUEUE = (0x00 << 7) | (0x01 << 2) | 0x00, 37 | NVM_ADMIN_CREATE_COMPLETION_QUEUE = (0x00 << 7) | (0x01 << 2) | 0x01, 38 | NVM_ADMIN_IDENTIFY = (0x00 << 7) | (0x01 << 2) | 0x02, 39 | NVM_ADMIN_GET_LOG_PAGE = (0x00 << 7) | (0x00 << 2) | 0x02, 40 | NVM_ADMIN_ABORT = (0x00 << 7) | (0x02 << 2) | 0x00, 41 | NVM_ADMIN_SET_FEATURES = (0x00 << 7) | (0x02 << 2) | 0x01, 42 | NVM_ADMIN_GET_FEATURES = (0x00 << 7) | (0x02 << 2) | 0x02 43 | }; 44 | 45 | 46 | 47 | 48 | 49 | /* 50 | * Set command's DWORD0 and DWORD1 51 | */ 52 | __device__ __host__ static inline 53 | void nvm_cmd_header(nvm_cmd_t* cmd, uint16_t cid, uint8_t opcode, uint32_t ns_id) 54 | { 55 | cmd->dword[0] = ((uint32_t) cid << 16) | (0x00 << 14) | (0x00 << 8) | (opcode & 0x7f); 56 | cmd->dword[1] = ns_id; 57 | } 58 | 59 | 60 | 61 | /* 62 | * Set command's DPTR field (DWORD6-9) 63 | */ 64 | __device__ __host__ static inline 65 | void nvm_cmd_data_ptr(nvm_cmd_t* cmd, uint64_t prp1, uint64_t prp2) 66 | { 67 | cmd->dword[0] &= ~( (0x03 << 14) | (0x03 << 8) ); 68 | 69 | cmd->dword[6] = (uint32_t) prp1; 70 | cmd->dword[7] = (uint32_t) (prp1 >> 32UL); 71 | cmd->dword[8] = (uint32_t) prp2; 72 | cmd->dword[9] = (uint32_t) (prp2 >> 32UL); 73 | } 74 | 75 | 76 | 77 | /* 78 | * Set command's block fields (DWORD10-12) 79 | */ 80 | __device__ __host__ static inline 81 | void nvm_cmd_rw_blks(nvm_cmd_t* cmd, uint64_t start_lba, uint16_t n_blks) 82 | { 83 | cmd->dword[10] = start_lba; 84 | cmd->dword[11] = start_lba >> 32; 85 | cmd->dword[12] = (cmd->dword[12] & 0xffff0000) | ((n_blks - 1) & 0xffff); 86 | } 87 | 88 | 89 | 90 | /* 91 | * Set command's dataset management (DSM) field (DWORD13) 92 | */ 93 | //__device__ __host__ static inline 94 | //void nvm_cmd_dataset(nvm_cmd_t* cmd, bool sequential, bool low_latency) 95 | //{ 96 | // cmd->dword[13] = 0; // not supported yet 97 | //} 98 | 99 | 100 | 101 | 102 | /* 103 | * Build a PRP list consisting of PRP entries. 104 | * 105 | * Populate a memory page with PRP entries required for a transfer. 106 | * Returns the number of PRP entries used. Number of pages should 107 | * always be max_data_size (MDTS) for IO commands. 108 | * 109 | * Note: currently, PRP lists can only be a single page 110 | */ 111 | __host__ __device__ static inline 112 | size_t nvm_prp_list(size_t page_size, size_t n_pages, void* list_ptr, const uint64_t* data_ioaddrs) 113 | { 114 | // TODO #ifdef __NO_COHERENCE__, make a nvm_prp_list_far variant that does not call nvm_cache_flush() 115 | size_t prps_per_page = page_size / sizeof(uint64_t); 116 | size_t i_prp; 117 | uint64_t* list; 118 | 119 | if (prps_per_page < n_pages) 120 | { 121 | n_pages = prps_per_page; 122 | } 123 | 124 | list = (uint64_t*) list_ptr; 125 | for (i_prp = 0; i_prp < n_pages; ++i_prp) 126 | { 127 | list[i_prp] = data_ioaddrs[i_prp]; 128 | } 129 | 130 | nvm_cache_flush(list_ptr, sizeof(uint64_t) * i_prp); 131 | 132 | return i_prp; 133 | } 134 | 135 | 136 | 137 | /* 138 | * Helper function to build a PRP list and set a command's data pointer fields. 139 | */ 140 | __host__ __device__ static inline 141 | size_t nvm_cmd_data(nvm_cmd_t* cmd, 142 | size_t page_size, 143 | size_t n_pages, 144 | void* list_ptr, 145 | uint64_t list_ioaddr, 146 | const uint64_t* data_ioaddrs) 147 | { 148 | size_t prp = 0; 149 | uint64_t dptr0 = 0; 150 | uint64_t dptr1 = 0; 151 | 152 | #if !defined( NDEBUG ) && !defined( __CUDA_ARCH__ ) 153 | if (n_pages == 0) 154 | { 155 | return 0; 156 | } 157 | #endif 158 | 159 | dptr0 = data_ioaddrs[prp++]; 160 | 161 | if (n_pages > 2 && list_ptr != NULL) 162 | { 163 | dptr1 = list_ioaddr; 164 | prp += nvm_prp_list(page_size, n_pages - 1, list_ptr, &data_ioaddrs[prp]); 165 | } 166 | else if (n_pages >= 2) 167 | { 168 | dptr1 = data_ioaddrs[prp++]; 169 | } 170 | 171 | nvm_cmd_data_ptr(cmd, dptr0, dptr1); 172 | return prp; 173 | } 174 | 175 | 176 | 177 | #ifndef __CUDACC__ 178 | #undef __device__ 179 | #undef __host__ 180 | #endif 181 | 182 | #endif /* __NVM_COMMAND_H__ */ 183 | -------------------------------------------------------------------------------- /include/nvm_ctrl.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_CTRL_H__ 2 | #define __NVM_CTRL_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef __DIS_CLUSTER__ 12 | #include 13 | #endif 14 | 15 | 16 | 17 | /* 18 | * Minimum size of mapped controller memory. 19 | */ 20 | #define NVM_CTRL_MEM_MINSIZE 0x2000 21 | 22 | 23 | 24 | #if defined (__unix__) 25 | /* 26 | * Initialize NVM controller handle. 27 | * 28 | * Read from controller registers and initialize controller handle. 29 | * This function should be used when using the kernel module or to manually 30 | * read from sysfs. 31 | * 32 | * Note: fd must be opened with O_RDWR and O_NONBLOCK 33 | */ 34 | int nvm_ctrl_init(nvm_ctrl_t** ctrl, int fd); 35 | #endif 36 | 37 | 38 | 39 | /* 40 | * Initialize NVM controller handle. 41 | * 42 | * Read from controller registers and initialize the controller handle using 43 | * a memory-mapped pointer to the PCI device BAR. 44 | * 45 | * This function should be used when neither SmartIO nor the disnvme kernel 46 | * module are used. 47 | * 48 | * Note: ctrl_mem must be at least NVM_CTRL_MEM_MINSIZE large and mapped 49 | * as IO memory. See arguments for mmap() for more info. 50 | */ 51 | int nvm_raw_ctrl_init(nvm_ctrl_t** ctrl, volatile void* mm_ptr, size_t mm_size); 52 | 53 | 54 | 55 | /* 56 | * Release controller handle. 57 | */ 58 | void nvm_ctrl_free(nvm_ctrl_t* ctrl); 59 | 60 | 61 | 62 | /* 63 | * Reset NVM controller. 64 | * 65 | * The queue memory must be memset to zero and be exactly one page size large. 66 | * IO addresses must align to the controller page size. 67 | * 68 | * Note: The controller must be unbound from any driver before attempting to 69 | * reset the controller. 70 | * 71 | * Note: This function is implicitly called by the controller manager, so it 72 | * should not be necessary to call it directly. 73 | */ 74 | int nvm_raw_ctrl_reset(const nvm_ctrl_t* ctrl, uint64_t acq_ioaddr, uint64_t asq_ioaddr); 75 | 76 | 77 | 78 | #ifdef __DIS_CLUSTER__ 79 | /* 80 | * Initialize NVM controller handle. 81 | * 82 | * Read from device registers and initialize controller handle. 83 | * This function should be used when SmartIO is being used. 84 | */ 85 | int nvm_dis_ctrl_init(nvm_ctrl_t** ctrl, uint64_t smartio_dev_id, uint32_t dis_adapter); 86 | #endif 87 | 88 | 89 | 90 | #ifdef __DIS_CLUSTER__ 91 | int nvm_dis_ctrl_map_p2p_device(const nvm_ctrl_t* ctrl, sci_device_t dev, uint64_t* ioaddr); 92 | #endif 93 | 94 | 95 | 96 | #ifdef __DIS_CLUSTER__ 97 | void nvm_dis_ctrl_unmap_p2p_device(const nvm_ctrl_t* ctrl, sci_device_t dev); 98 | #endif 99 | 100 | 101 | #ifdef __cplusplus 102 | } 103 | #endif 104 | #endif /* __NVM_CTRL_H__ */ 105 | -------------------------------------------------------------------------------- /include/nvm_dma.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_DMA_H__ 2 | #define __NVM_DMA_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #ifdef __DIS_CLUSTER__ 13 | #include 14 | #endif 15 | 16 | 17 | 18 | /* 19 | * Create DMA mapping descriptor from physical/bus addresses. 20 | * 21 | * Create a DMA mapping descriptor, describing a region of memory that is 22 | * accessible for the NVM controller. The caller must supply physical/bus 23 | * addresses of physical memory pages, page size and total number of pages. 24 | * As the host's page size may differ from the controller's page size (MPS), 25 | * this function will calculate the necessary offsets into the actual memory 26 | * pages. 27 | * 28 | * While virtual memory is assumed to be continuous, the physical pages do not 29 | * need to be contiguous. Physical/bus addresses must be aligned to the 30 | * controller's page size. 31 | * 32 | * Note: vaddr can be NULL. 33 | */ 34 | int nvm_dma_map(nvm_dma_t** map, // Mapping descriptor reference 35 | const nvm_ctrl_t* ctrl, // NVM controller reference 36 | void* vaddr, // Pointer to userspace memory (can be NULL if not required) 37 | size_t page_size, // Physical page size 38 | size_t n_pages, // Number of pages to map 39 | const uint64_t* page_addrs); // List of physical/bus addresses to the pages 40 | 41 | 42 | 43 | /* 44 | * Remove DMA mapping descriptor. 45 | * 46 | * Unmap DMA mappings (if necessary) and remove the descriptor. 47 | * This function destroys the descriptor. 48 | */ 49 | void nvm_dma_unmap(nvm_dma_t* map); 50 | 51 | 52 | 53 | /* 54 | * Create DMA mapping descriptor from virtual address using the kernel module. 55 | * This function is similar to nvm_dma_map, except the user is not required 56 | * to pass physical/bus addresses. 57 | * 58 | * Note: vaddr can not be NULL, and must be aligned to system page size. 59 | */ 60 | int nvm_dma_map_host(nvm_dma_t** map, const nvm_ctrl_t* ctrl, void* vaddr, size_t size); 61 | 62 | 63 | 64 | #if ( defined( __CUDA__ ) || defined( __CUDACC__ ) ) 65 | 66 | /* 67 | * Create DMA mapping descriptor from CUDA device pointer using the kernel 68 | * module. This function is similar to nvm_dma_map_host, except the memory 69 | * pointer must be a valid CUDA device pointer (see manual for 70 | * cudaGetPointerAttributes). 71 | * 72 | * The controller handle must have been created using the kernel module. 73 | * 74 | * Note: vaddr can not be NULL, and must be aligned to GPU page size. 75 | */ 76 | int nvm_dma_map_device(nvm_dma_t** map, const nvm_ctrl_t* ctrl, void* devptr, size_t size); 77 | 78 | #endif /* __CUDA__ */ 79 | 80 | 81 | 82 | #if defined( __DIS_CLUSTER__ ) 83 | 84 | /* 85 | * Create DMA mapping descriptor from local SISCI segment. 86 | * 87 | * Create DMA mapping descriptor from a local segment handler, and 88 | * reverse-map the segment making it accessible from the controller. 89 | * As segment memory is always continuous and page-aligned, it is not 90 | * necessary to calculate physical memory addresses. However, the user 91 | * should ensure that the mapping size is aligned to a controller 92 | * page-size (MPS). 93 | * 94 | * The controller handle must have been created using SmartIO, and 95 | * the segment must already be prepared on the local adapter. 96 | * 97 | * Note: vaddr can be NULL. 98 | */ 99 | int nvm_dis_dma_map_local(nvm_dma_t** map, // Mapping descriptor reference 100 | const nvm_ctrl_t* ctrl, // NVM controller handle 101 | uint32_t dis_adapter, // Local DIS adapter 102 | sci_local_segment_t segment, // Local segment descriptor 103 | bool map_vaddr); // Should function also map segment into local space 104 | 105 | #endif /* __DIS_CLUSTER__ */ 106 | 107 | 108 | 109 | #if defined( __DIS_CLUSTER__ ) 110 | 111 | /* 112 | * Create DMA mapping descriptor from remote SISCI segment. 113 | * 114 | * Create DMA mapping descriptor from a remote segment handler, and 115 | * reverse-map the segment making it accessible from the controller. 116 | * This function is similar to nvm_dis_dma_map_local. 117 | * 118 | * The remote segment must already be connected. 119 | * 120 | * Note: vaddr can be NULL. 121 | */ 122 | int nvm_dis_dma_map_remote(nvm_dma_t** map, // Mapping descriptor reference 123 | const nvm_ctrl_t* ctrl, // NVM controller handle 124 | sci_remote_segment_t segment,// Remote segment descriptor 125 | bool map_vaddr, // Should function also map segment into local space 126 | bool map_wc); // Should function map with write combining 127 | 128 | #endif /* __DIS_CLUSTER__ */ 129 | 130 | 131 | 132 | #if ( !defined( __CUDA__ ) && !defined( __CUDACC__ ) ) && ( defined (__unix__) ) 133 | /* 134 | * Short-hand function for allocating a page aligned buffer and mapping it 135 | * for the controller. 136 | * 137 | * Note: this function will not work if you are using the CUDA API 138 | */ 139 | int nvm_dma_create(nvm_dma_t** map, 140 | const nvm_ctrl_t* ctrl, 141 | size_t size); 142 | #endif 143 | 144 | 145 | 146 | #if defined( __DIS_CLUSTER__ ) 147 | 148 | /* 149 | * Create segment and map it for the controller. 150 | * Short-hand function for creating a local segment. 151 | */ 152 | int nvm_dis_dma_create(nvm_dma_t** map, 153 | const nvm_ctrl_t* ctrl, 154 | uint32_t dis_adapter, 155 | uint32_t id, 156 | size_t size); 157 | 158 | #endif /* __DIS_CLUSTER__ */ 159 | 160 | 161 | 162 | #if defined( __DIS_CLUSTER__ ) 163 | 164 | /* 165 | * Connect to device memory. 166 | * Short-hand function for connecting to device memory. 167 | */ 168 | int nvm_dis_dma_connect(nvm_dma_t** map, 169 | const nvm_ctrl_t* ctrl, 170 | uint32_t dis_adapter, 171 | uint32_t segment_no, 172 | size_t size, 173 | bool shared); 174 | 175 | #endif /* __DIS_CLUSTER__ */ 176 | 177 | 178 | 179 | #if defined ( __DIS_CLUSTER__ ) 180 | 181 | /* 182 | * Note: This function requires the IOMMU to be enabled. 183 | */ 184 | int nvm_dis_dma_map_host(nvm_dma_t** map, 185 | const nvm_ctrl_t* ctrl, 186 | uint32_t dis_adapter, 187 | uint32_t id, 188 | void* vaddr, 189 | size_t size); 190 | 191 | #endif 192 | 193 | 194 | #if ( ( defined( __CUDA__ ) || defined( __CUDACC__ ) ) && defined( __DIS_CLUSTER__ ) ) 195 | 196 | int nvm_dis_dma_map_device(nvm_dma_t** map, 197 | const nvm_ctrl_t* ctrl, 198 | uint32_t dis_adapter, 199 | uint32_t id, 200 | void* devptr, 201 | size_t size); 202 | 203 | #endif /* __DIS_CLUSTER__ && __CUDA__ */ 204 | 205 | 206 | 207 | #ifdef __cplusplus 208 | } 209 | #endif 210 | #endif /* __NVM_DMA_H__ */ 211 | -------------------------------------------------------------------------------- /include/nvm_error.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_ERROR_H__ 2 | #define __NVM_ERROR_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | 13 | 14 | /* Get the status code type of an NVM completion. */ 15 | #define NVM_ERR_SCT(cpl) ((uint8_t) _RB(*NVM_CPL_STATUS(cpl), 11, 9)) 16 | 17 | 18 | 19 | /* Get the status code of an NVM completion */ 20 | #define NVM_ERR_SC(cpl) ((uint8_t) _RB(*NVM_CPL_STATUS(cpl), 8, 1)) 21 | 22 | 23 | 24 | /* Is do not retry flag set? */ 25 | #define NVM_ERR_DNR(cpl) ((uint8_t) _RB(*NVM_CPL_STATUS(cpl), 15, 15)) 26 | 27 | 28 | 29 | /* Extract value from status field from NVM completion */ 30 | #define NVM_ERR_STATUS(cpl) \ 31 | ((int) ( (cpl) != NULL ? -((NVM_ERR_SCT(cpl) << 8) | NVM_ERR_SC(cpl)) : 0 )) 32 | 33 | 34 | 35 | /* Convenience macro for checking if an NVM completion indicates success. */ 36 | #define NVM_ERR_OK(cpl) ( !NVM_ERR_SCT(cpl) && !NVM_ERR_SC(cpl) ) 37 | 38 | 39 | 40 | /* Pack errno and NVM completion status into a single status variable */ 41 | #define NVM_ERR_PACK(cpl, err) \ 42 | ((int) ( (err) != 0 ? (err) : NVM_ERR_STATUS(cpl) ) ) 43 | 44 | 45 | 46 | /* Extract values from packed status */ 47 | #define NVM_ERR_UNPACK_ERRNO(status) ((status > 0) ? (status) : 0) 48 | #define NVM_ERR_UNPACK_SCT(status) ((status < 0) ? (((-status) >> 8) & 0xff) : 0) 49 | #define NVM_ERR_UNPACK_SC(status) ((status < 0) ? ((-status) & 0xff) : 0) 50 | 51 | 52 | /* Check if everything is okay */ 53 | #define nvm_ok(status) ( !(status) ) 54 | 55 | 56 | 57 | /* 58 | * Get an error string associated with the status code type and status code. 59 | * This function calls strerror() if the packed status is a regular errno. 60 | */ 61 | const char* nvm_strerror(int status); 62 | 63 | 64 | 65 | #ifdef __cplusplus 66 | } 67 | #endif 68 | #endif /* __NVM_ERROR_H__ */ 69 | -------------------------------------------------------------------------------- /include/nvm_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_QUEUE_H__ 2 | #define __NVM_QUEUE_H__ 3 | 4 | #ifndef __CUDACC__ 5 | #define __device__ 6 | #define __host__ 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | 15 | /* 16 | * Clear queue descriptor. 17 | * 18 | * Initialize an empty queue descriptor. 19 | * The user must clear the queue memory manually before using the handle. 20 | * 21 | * Note: vaddr must be page-aligned and at least one page. 22 | */ 23 | #ifdef __cplusplus 24 | extern "C" { 25 | #endif 26 | __host__ 27 | void nvm_queue_clear(nvm_queue_t* q, // NVM queue descriptor 28 | const nvm_ctrl_t* ctrl, // NVM controller handle 29 | bool cq, // Is this a completion queue or submission queue? 30 | uint16_t no, // Queue number 31 | void* vaddr, // Virtual address to queue memory 32 | uint64_t ioaddr); // Bus address to queue memory (as seen from the controller) 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | 38 | 39 | /* 40 | * Enqueue a submission command. 41 | * 42 | * Enqueue a submission command in the specified SQ and return a pointer to 43 | * the queue slot in order to build the command inline in queue memory. 44 | * 45 | * Returns a pointer to the queue entry. or NULL if the queue is full. 46 | */ 47 | __host__ __device__ static inline 48 | nvm_cmd_t* nvm_sq_enqueue(nvm_queue_t* sq) 49 | { 50 | // Check if queue is full 51 | if ((uint16_t) ((sq->tail - sq->head) % sq->max_entries) == sq->max_entries - 1) 52 | { 53 | return NULL; 54 | } 55 | 56 | // Take slot and end of queue 57 | nvm_cmd_t* cmd = (nvm_cmd_t*) (((unsigned char*) sq->vaddr) + sq->entry_size * sq->tail); 58 | 59 | // Increase tail pointer and wrap around if necessary 60 | if (++sq->tail == sq->max_entries) 61 | { 62 | sq->phase = !sq->phase; 63 | sq->tail = 0; 64 | } 65 | 66 | return cmd; 67 | } 68 | 69 | 70 | 71 | /* 72 | * Enqueue command the i'th of n threads. 73 | * 74 | * This function does not check actual queue state, so it is important 75 | * that all completions are consumed before making successive calls to this function. 76 | * It is also assumed that n < max_entries. 77 | */ 78 | #ifdef __CUDACC__ 79 | __device__ static inline 80 | nvm_cmd_t* nvm_sq_enqueue_n(nvm_queue_t* sq, nvm_cmd_t* last, uint16_t n, uint16_t i) 81 | { 82 | unsigned char* start = (unsigned char*) sq->vaddr; 83 | unsigned char* end = start + (sq->max_entries * sq->entry_size); 84 | nvm_cmd_t* cmd = NULL; 85 | 86 | if (last == NULL) 87 | { 88 | cmd = (nvm_cmd_t*) (start + sq->entry_size * i); 89 | } 90 | else 91 | { 92 | cmd = (nvm_cmd_t*) (((unsigned char*) last) + n * sq->entry_size); 93 | 94 | if (((nvm_cmd_t*) end) <= cmd) 95 | { 96 | cmd = (nvm_cmd_t*) (start + (((unsigned char*) cmd) - end)); 97 | } 98 | } 99 | 100 | if (i == 0) 101 | { 102 | sq->tail = (((uint32_t) sq->tail) + ((uint32_t) n)) % sq->max_entries; 103 | } 104 | 105 | //#ifdef __CUDA_ARCH__ 106 | __syncthreads(); 107 | //#endif 108 | 109 | return cmd; 110 | } 111 | #endif 112 | 113 | 114 | 115 | /* 116 | * Poll completion queue. 117 | * 118 | * Check the head of a completion queue for a new entry. The caller must 119 | * call dequeue manually. 120 | * 121 | * Returns a pointer to an unprocessed completion entry, or NULL if the queue 122 | * is empty. 123 | */ 124 | __host__ __device__ static inline 125 | nvm_cpl_t* nvm_cq_poll(const nvm_queue_t* cq) 126 | { 127 | nvm_cpl_t* cpl = (nvm_cpl_t*) (((unsigned char*) cq->vaddr) + cq->entry_size * cq->head); 128 | 129 | nvm_cache_invalidate((void*) cpl, sizeof(nvm_cpl_t)); 130 | 131 | // Check if new completion is ready by checking the phase tag 132 | if (!!_RB(*NVM_CPL_STATUS(cpl), 0, 0) != cq->phase) 133 | { 134 | return NULL; 135 | } 136 | 137 | return cpl; 138 | } 139 | 140 | 141 | 142 | /* 143 | * Dequeue completion queue entry. 144 | * 145 | * Dequeue a completion entry from the completion queue. If there is no ready 146 | * completions, this function returns NULL. 147 | * 148 | * The caller must update the corresponding SQ manually. 149 | * 150 | * Returns a pointer to the completion entry, or NULL if the queue is empty. 151 | */ 152 | __host__ __device__ static inline 153 | nvm_cpl_t* nvm_cq_dequeue(nvm_queue_t* cq) 154 | { 155 | nvm_cpl_t* cpl = nvm_cq_poll(cq); 156 | 157 | if (cpl != NULL) 158 | { 159 | // Increase head pointer and wrap around if necessary 160 | if (++cq->head == cq->max_entries) 161 | { 162 | cq->head = 0; 163 | cq->phase = !cq->phase; 164 | } 165 | } 166 | 167 | return cpl; 168 | } 169 | 170 | 171 | 172 | /* 173 | * Dequeue completion queue entry. 174 | * 175 | * Dequeue a completion entry from the completion queue. If none are ready 176 | * at the time, this function will block until a controller timeout interval 177 | * or a ready completion. 178 | * 179 | * Returns a pointer to the completion entry, or NULL if the queue is empty or 180 | * on timeout. 181 | */ 182 | #ifdef __cplusplus 183 | extern "C" { 184 | #endif 185 | __host__ 186 | nvm_cpl_t* nvm_cq_dequeue_block(nvm_queue_t* cq, uint64_t timeout); 187 | #ifdef __cplusplus 188 | } 189 | #endif 190 | 191 | 192 | 193 | /* 194 | * Update SQ tail pointer. 195 | * 196 | * Submit all enqueued commands by ringing the doorbell. 197 | * The caller must make sure that all commands are prepared before calling 198 | * this. 199 | */ 200 | __host__ __device__ static inline 201 | void nvm_sq_submit(nvm_queue_t* sq) 202 | { 203 | if (sq->last != sq->tail && sq->db != NULL) 204 | { 205 | nvm_cache_flush((void*) sq->vaddr, sizeof(nvm_cmd_t) * sq->max_entries); 206 | nvm_wcb_flush(); 207 | 208 | *((volatile uint32_t*) sq->db) = sq->tail; 209 | sq->last = sq->tail; 210 | } 211 | } 212 | 213 | 214 | 215 | /* 216 | * Update SQ head pointer. 217 | */ 218 | __host__ __device__ static inline 219 | void nvm_sq_update(nvm_queue_t* sq) 220 | { 221 | // Update head pointer of submission queue 222 | if (sq->db != NULL && ++sq->head == sq->max_entries) 223 | { 224 | sq->head = 0; 225 | } 226 | } 227 | 228 | 229 | 230 | /* 231 | * Update controller's CQ head pointer. 232 | * 233 | * Indicate that all completions are processed by ringing the doorbell. 234 | * All completion pointers acquired before this must be discarded after 235 | * calling this. 236 | */ 237 | __host__ __device__ static inline 238 | void nvm_cq_update(nvm_queue_t* cq) 239 | { 240 | if (cq->last != cq->head && cq->db != NULL) 241 | { 242 | *((volatile uint32_t*) cq->db) = cq->head; 243 | cq->last = cq->head; 244 | nvm_wcb_flush(); 245 | } 246 | } 247 | 248 | 249 | #ifndef __CUDACC__ 250 | #undef __device__ 251 | #undef __host__ 252 | #endif 253 | 254 | #endif /* __NVM_QUEUE_H__ */ 255 | -------------------------------------------------------------------------------- /include/nvm_rpc.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_RPC_H__ 2 | #define __NVM_RPC_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | 10 | 11 | //int nvm_tcp_rpc_bind(nvm_aq_ref* ref, const char* hostname, uint16_t port); 12 | 13 | 14 | 15 | #ifdef __DIS_CLUSTER__ 16 | 17 | /* 18 | * Bind admin queue-pair reference to remote handle. 19 | * The user should call the nvm_aq_destroy() to remove binding. 20 | */ 21 | int nvm_dis_rpc_bind(nvm_aq_ref* ref, const nvm_ctrl_t* ctrl, uint32_t adapter); 22 | 23 | #endif 24 | 25 | 26 | 27 | /* 28 | * Unbind admin queue-pair reference. 29 | * If reference is not bound (i.e., it is local), this function will do nothing. 30 | */ 31 | void nvm_rpc_unbind(nvm_aq_ref ref); 32 | 33 | 34 | 35 | /* 36 | * Relay NVM admin command. 37 | * 38 | * Use a local AQ pair reference to relay a NVM admin command to ASQ and get 39 | * a corresponding completion from the ACQ. This function will block until 40 | * either a timeout occurs or until the command is completed. 41 | * 42 | * Return value: 43 | * - If return value is zero, it indicates success. 44 | * - If return value is positive, it indicates an errno. 45 | * - If return value is negative, it indicates an NVM error. 46 | * 47 | * Use the error handling macros in nvm_error.h 48 | * 49 | * Note: The command can be modified. 50 | */ 51 | int nvm_raw_rpc(nvm_aq_ref ref, nvm_cmd_t* cmd, nvm_cpl_t* cpl); 52 | 53 | 54 | 55 | #ifdef __cplusplus 56 | } 57 | #endif 58 | #endif /* #ifdef __NVM_RPC_H__ */ 59 | -------------------------------------------------------------------------------- /include/nvm_types.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_TYPES_H__ 2 | #define __NVM_TYPES_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | 7 | #include 8 | #include 9 | 10 | #ifndef __CUDACC__ 11 | #define __align__(x) 12 | #endif 13 | 14 | 15 | 16 | /* 17 | * NVM controller handle. 18 | */ 19 | typedef struct 20 | { 21 | size_t page_size; // Memory page size used by the controller (MPS) 22 | uint8_t dstrd; // Doorbell stride (in encoded form) 23 | uint64_t timeout; // Controller timeout in milliseconds (TO) 24 | uint16_t max_entries; // Maximum queue entries supported (MQES) 25 | size_t mm_size; // Size of memory-mapped region 26 | volatile void* mm_ptr; // Memory-mapped pointer to BAR0 of the physical device 27 | } nvm_ctrl_t; 28 | 29 | 30 | 31 | 32 | /* 33 | * NVM admin queue-pair reference handle. 34 | * 35 | * As only a single process can be responsible of resetting the controller and 36 | * setting administration queues, this structure represents a remote handle to 37 | * that process. It is used as a descriptor for executing RPC calls to the 38 | * remote process owning the admin queues. 39 | */ 40 | struct nvm_admin_reference; 41 | typedef struct nvm_admin_reference* nvm_aq_ref; 42 | 43 | 44 | 45 | /* 46 | * DMA mapping descriptor. 47 | * 48 | * This structure describes a region of memory that is accessible for the 49 | * NVM controller using DMA. The API assumes a continuous virtual memory 50 | * address, but the physical pages do not need to be contiguous. 51 | * 52 | * The structure contains a variably sized array of bus addresses that maps 53 | * to the physical memory pages. The user should therefore not create a local 54 | * instance of this descriptor, but rather rely on the API to allocate and 55 | * instantiate members. 56 | * 57 | * Note: Only page-aligned addresses are supported in NVM Express 58 | */ 59 | typedef struct __align__(32) 60 | { 61 | void* vaddr; // Virtual address to start of region (NB! can be NULL) 62 | size_t page_size; // Controller's page size (MPS) 63 | size_t n_ioaddrs; // Number of MPS-sized pages 64 | uint64_t ioaddrs[]; // Physical/IO addresses of the memory pages 65 | } __attribute__((aligned (32))) nvm_dma_t; 66 | 67 | 68 | 69 | /* 70 | * NVM queue descriptor. 71 | * 72 | * This structure represents an NVM IO queue and holds information 73 | * about memory addresses, queue entries as well as a memory mapped pointer to 74 | * the device doorbell register. Maximum queue size is limited to a single 75 | * page. 76 | * 77 | * Note: This descriptor represents both completion and submission queues. 78 | */ 79 | typedef struct __align__(64) 80 | { 81 | uint16_t no; // Queue number (must be unique per SQ/CQ pair) 82 | uint16_t max_entries; // Maximum number of queue entries supported 83 | uint16_t entry_size; // Queue entry size 84 | uint32_t head; // Queue's head pointer 85 | uint32_t tail; // Queue's tail pointer 86 | // TODO: Create bitfield for phase, add a remote field indicating 87 | // if queue is far memory nor not, in which case we whould NOT do 88 | // cache operations 89 | int16_t phase; // Current phase bit 90 | uint32_t last; // Used internally to check db writes 91 | volatile uint32_t* db; // Pointer to doorbell register (NB! write only) 92 | volatile void* vaddr; // Virtual address to start of queue memory 93 | uint64_t ioaddr; // Physical/IO address of the memory page 94 | } __attribute__((aligned (64))) nvm_queue_t; 95 | 96 | 97 | 98 | /* 99 | * NVM completion queue entry type (16 bytes) 100 | */ 101 | typedef struct __align__(16) 102 | { 103 | uint32_t dword[4]; // The name DWORD is chosen to reflect the specification 104 | } __attribute__((aligned (16))) nvm_cpl_t; 105 | 106 | 107 | 108 | /* 109 | * NVM command queue entry type (64 bytes) 110 | */ 111 | typedef struct __align__(64) 112 | { 113 | uint32_t dword[16]; 114 | } __attribute__((aligned (64))) nvm_cmd_t; 115 | 116 | 117 | 118 | /* 119 | * Controller information structure. 120 | * 121 | * Holds information about an NVM controller retrieved from reading on-board 122 | * registers and running an IDENTIFY CONTROLLER admin command. 123 | */ 124 | struct nvm_ctrl_info 125 | { 126 | uint32_t nvme_version; // NVM Express version number 127 | size_t page_size; // Memory page size used by the controller (MPS) 128 | size_t db_stride; // Doorbell stride (DSTRD) 129 | uint64_t timeout; // Controller timeout in milliseconds (TO) 130 | int contiguous; // Contiguous queues required (CQR) 131 | uint16_t max_entries; // Maximum queue entries supported (MQES) 132 | uint8_t pci_vendor[4]; // PCI vendor and subsystem vendor identifier 133 | char serial_no[20]; // Serial number (NB! not null terminated) 134 | char model_no[40]; // Model number (NB! not null terminated) 135 | char firmware[8]; // Firmware revision 136 | size_t max_data_size; // Maximum data transfer size (MDTS) 137 | size_t max_data_pages; // Maximum data transfer size (in controller pages) 138 | size_t cq_entry_size; // CQ entry size (CQES) 139 | size_t sq_entry_size; // SQ entry size (SQES) 140 | size_t max_out_cmds; // Maximum outstanding commands (MAXCMD) 141 | size_t max_n_ns; // Maximum number of namespaces (NN) 142 | }; 143 | 144 | 145 | 146 | /* 147 | * Namespace information structure. 148 | * 149 | * Holds informaiton about an NVM namespace. 150 | */ 151 | struct nvm_ns_info 152 | { 153 | uint32_t ns_id; // Namespace identifier 154 | uint64_t size; // Size in logical blocks (NSZE) 155 | uint64_t capacity; // Capacity in logical blocks (NCAP) 156 | uint64_t utilization; // Utilization in logical blocks (NUSE) 157 | size_t lba_data_size; // Logical block size (LBADS) 158 | size_t metadata_size; // Metadata size (MS) 159 | }; 160 | 161 | 162 | 163 | #ifndef __CUDACC__ 164 | #undef __align__ 165 | #endif 166 | 167 | #ifdef __cplusplus 168 | } 169 | #endif 170 | #endif /* __NVM_TYPES_H__ */ 171 | -------------------------------------------------------------------------------- /include/nvm_util.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_UTIL_H__ 2 | #define __NVM_UTIL_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef __DIS_CLUSTER__ 9 | #include 10 | #include 11 | #endif 12 | 13 | 14 | 15 | /* Convenience function for creating a bit mask */ 16 | #ifdef __CUDACC__ 17 | __host__ __device__ 18 | #endif 19 | static inline 20 | uint64_t _nvm_bitmask(int hi, int lo) 21 | { 22 | uint64_t mask = 0; 23 | 24 | for (int i = lo; i <= hi; ++i) 25 | { 26 | mask |= 1UL << i; 27 | } 28 | 29 | return mask; 30 | } 31 | 32 | 33 | #if defined( __NO_COHERENCE__ ) && defined( __DIS_CLUSTER__ ) 34 | #ifdef __CUDACC__ 35 | __host__ __device__ 36 | #endif 37 | static inline 38 | void _nvm_cache_flush(void* ptr, size_t size) 39 | { 40 | #ifndef __CUDA_ARCH__ 41 | sci_error_t err; 42 | SCICacheSync(NULL, ptr, size, SCI_FLAG_CACHE_FLUSH, &err); 43 | #endif 44 | } 45 | 46 | #define nvm_cache_flush(ptr, size) _nvm_cache_flush(ptr, size) 47 | #else 48 | #define nvm_cache_flush(ptr, size) 49 | #endif 50 | 51 | 52 | 53 | #if defined( __NO_COHERENCE__ ) && defined( __DIS_CLUSTER__ ) 54 | #ifdef __CUDACC__ 55 | __host__ __device__ 56 | #endif 57 | static inline 58 | void _nvm_cache_invalidate(void* ptr, size_t size) 59 | { 60 | #ifndef __CUDA_ARCH__ 61 | sci_error_t err; 62 | SCICacheSync(NULL, ptr, size, SCI_FLAG_CACHE_FLUSH | SCI_FLAG_CACHE_INVALIDATE, &err); 63 | #endif 64 | } 65 | 66 | #define nvm_cache_invalidate(ptr, size) _nvm_cache_invalidate(ptr, size) 67 | #else 68 | #define nvm_cache_invalidate(ptr, size) 69 | #endif 70 | 71 | 72 | 73 | #if defined( __DIS_CLUSTER__ ) 74 | #ifdef __CUDACC__ 75 | __host__ __device__ 76 | #endif 77 | static inline 78 | void _nvm_wcb_flush() 79 | { 80 | #ifndef __CUDA_ARCH__ 81 | SCIFlush(NULL, 0); 82 | #endif 83 | } 84 | 85 | #define nvm_wcb_flush() _nvm_wcb_flush() 86 | #else 87 | #define nvm_wcb_flush() 88 | #endif 89 | 90 | 91 | 92 | /* Extract specific bits */ 93 | #define _RB(v, hi, lo) \ 94 | ( ( (v) & _nvm_bitmask((hi), (lo)) ) >> (lo) ) 95 | 96 | 97 | /* Set specifics bits */ 98 | #define _WB(v, hi, lo) \ 99 | ( ( (v) << (lo) ) & _nvm_bitmask((hi), (lo)) ) 100 | 101 | 102 | /* Offset to a register */ 103 | #define _REG(p, offs, bits) \ 104 | ((volatile uint##bits##_t *) (((volatile unsigned char*) ((volatile void*) (p))) + (offs))) 105 | 106 | 107 | /* 108 | * Calculate block number from page number. 109 | */ 110 | #define NVM_PAGE_TO_BLOCK(page_size, block_size, pageno) \ 111 | (((page_size) * (pageno)) / (block_size)) 112 | 113 | 114 | 115 | /* 116 | * Calculate page number from block number. 117 | */ 118 | #define NVM_BLOCK_TO_PAGE(page_size, block_size, blockno) \ 119 | (((block_size) * (blockno)) / (page_size)) 120 | 121 | 122 | /* 123 | * Create mask to clear away address offset. 124 | */ 125 | #define NVM_PAGE_MASK(page_size) \ 126 | ~((page_size) - 1) 127 | 128 | 129 | /* 130 | * Round address down to nearest page alignment. 131 | */ 132 | #define NVM_ADDR_MASK(addr, page_size) \ 133 | (((uint64_t) (addr)) & NVM_PAGE_MASK((page_size))) 134 | 135 | 136 | 137 | /* 138 | * Align size to page boundary. 139 | */ 140 | #define NVM_PAGE_ALIGN(size, page_size) \ 141 | (((size) + (page_size) - 1) & NVM_PAGE_MASK((page_size))) 142 | 143 | 144 | /* 145 | * Calculate page-aligned offset into address. 146 | */ 147 | #define NVM_ADDR_OFFSET(addr, page_size, pageno) \ 148 | (((uint64_t) (addr)) + ((page_size) * (pageno))) 149 | 150 | 151 | /* 152 | * Calculate page-aligned offset into pointer. 153 | */ 154 | #define NVM_PTR_OFFSET(ptr, page_size, pageno) \ 155 | ((void*) (((unsigned char*) (ptr)) + ((page_size) * (pageno)))) 156 | 157 | 158 | /* 159 | * Align size to controller pages. 160 | */ 161 | #define NVM_CTRL_ALIGN(ctrl_ptr, size) \ 162 | NVM_PAGE_ALIGN((size), (ctrl_ptr)->page_size) 163 | 164 | 165 | /* 166 | * Convert size to number of controller pages. 167 | */ 168 | #define NVM_CTRL_PAGES(ctrl_ptr, size) \ 169 | (NVM_CTRL_ALIGN((ctrl_ptr), (size)) / (ctrl_ptr)->page_size) 170 | 171 | 172 | /* 173 | * Align size to page size. 174 | */ 175 | #define NVM_DMA_ALIGN(dma_ptr, size) \ 176 | NVM_PAGE_ALIGN((size), (dma_ptr)->page_size) 177 | 178 | 179 | /* 180 | * Calculate controller page-aligned offset into DMA handle pointer. 181 | */ 182 | #define NVM_DMA_OFFSET(dma_ptr, pageno) \ 183 | NVM_PTR_OFFSET((dma_ptr)->vaddr, (dma_ptr)->page_size, (pageno)) 184 | 185 | 186 | 187 | 188 | /* Standard fields in a command */ 189 | #define NVM_CMD_CID(p) _REG(p, 2, 16) 190 | #define NVM_CMD_NSID(p) _REG(p, 1, 32) 191 | 192 | 193 | /* Standard fields in a completion */ 194 | #define NVM_CPL_CID(p) _REG(p, 12, 16) 195 | #define NVM_CPL_SQHD(p) _REG(p, 8, 16) 196 | #define NVM_CPL_SQID(p) _REG(p, 10, 16) 197 | #define NVM_CPL_STATUS(p) _REG(p, 14, 16) 198 | 199 | 200 | /* Convenience macro for creating a default CID based on submission queue */ 201 | #define NVM_DEFAULT_CID(sq) ((sq)->tail + (!(sq)->phase) * (sq)->max_entries) 202 | 203 | 204 | #ifdef __cplusplus 205 | extern "C" { 206 | #endif 207 | /* 208 | * Get controller associated with admin queue-pair reference. 209 | */ 210 | const nvm_ctrl_t* nvm_ctrl_from_aq_ref(nvm_aq_ref ref); 211 | #ifdef __cplusplus 212 | } 213 | #endif 214 | 215 | 216 | #endif /* __NVM_UTIL_H__ */ 217 | -------------------------------------------------------------------------------- /module/Makefile.in: -------------------------------------------------------------------------------- 1 | 2 | ifneq ($(KERNELRELEASE),) 3 | src := @module_root@ 4 | obj-m := @CMAKE_PROJECT_NAME@.o 5 | @CMAKE_PROJECT_NAME@-objs := pci.o list.o ctrl.o map.o 6 | ccflags-y += @module_ccflags@ 7 | KBUILD_EXTRA_SYMBOLS := @module_symbols@ 8 | else 9 | 10 | .PHONY: default reload unload load clean install 11 | 12 | default: 13 | $(MAKE) -C @KERNEL@ M=@module_output@ modules 14 | 15 | clean: 16 | $(MAKE) -C @KERNEL@ M=@module_output@ clean 17 | 18 | reload: unload load 19 | 20 | unload: 21 | -rmmod @CMAKE_PROJECT_NAME@.ko 22 | 23 | load: 24 | insmod @CMAKE_PROJECT_NAME@.ko num_ctrls=64 25 | 26 | install: default 27 | $(MAKE) -C @KERNEL@ M=@module_output@ modules_install 28 | #$(MAKE) INSTALL_MOD_DIR=@CMAKE_PROJECT_NAME@ -C @KERNEL@ M=@module_output@ modules_install 29 | 30 | endif 31 | 32 | -------------------------------------------------------------------------------- /module/ctrl.c: -------------------------------------------------------------------------------- 1 | #include "ctrl.h" 2 | #include "list.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | 12 | struct ctrl* ctrl_get(struct list* list, struct class* cls, struct pci_dev* pdev, int number) 13 | { 14 | struct ctrl* ctrl = NULL; 15 | 16 | ctrl = kmalloc(sizeof(struct ctrl), GFP_KERNEL | GFP_NOWAIT); 17 | if (ctrl == NULL) 18 | { 19 | printk(KERN_CRIT "Failed to allocate controller reference\n"); 20 | return ERR_PTR(-ENOMEM); 21 | } 22 | 23 | list_node_init(&ctrl->list); 24 | 25 | ctrl->pdev = pdev; 26 | ctrl->number = number; 27 | ctrl->rdev = 0; 28 | ctrl->cls = cls; 29 | ctrl->chrdev = NULL; 30 | 31 | snprintf(ctrl->name, sizeof(ctrl->name), "%s%d", KBUILD_MODNAME, ctrl->number); 32 | ctrl->name[sizeof(ctrl->name) - 1] = '\0'; 33 | 34 | list_insert(list, &ctrl->list); 35 | 36 | return ctrl; 37 | } 38 | 39 | 40 | 41 | void ctrl_put(struct ctrl* ctrl) 42 | { 43 | if (ctrl != NULL) 44 | { 45 | list_remove(&ctrl->list); 46 | ctrl_chrdev_remove(ctrl); 47 | kfree(ctrl); 48 | } 49 | } 50 | 51 | 52 | 53 | struct ctrl* ctrl_find_by_pci_dev(const struct list* list, const struct pci_dev* pdev) 54 | { 55 | const struct list_node* element = list_next(&list->head); 56 | struct ctrl* ctrl; 57 | 58 | while (element != NULL) 59 | { 60 | ctrl = container_of(element, struct ctrl, list); 61 | 62 | if (ctrl->pdev == pdev) 63 | { 64 | return ctrl; 65 | } 66 | 67 | element = list_next(element); 68 | } 69 | 70 | return NULL; 71 | } 72 | 73 | 74 | 75 | struct ctrl* ctrl_find_by_inode(const struct list* list, const struct inode* inode) 76 | { 77 | const struct list_node* element = list_next(&list->head); 78 | struct ctrl* ctrl; 79 | 80 | while (element != NULL) 81 | { 82 | ctrl = container_of(element, struct ctrl, list); 83 | 84 | if (&ctrl->cdev == inode->i_cdev) 85 | { 86 | return ctrl; 87 | } 88 | 89 | element = list_next(element); 90 | } 91 | 92 | return NULL; 93 | } 94 | 95 | 96 | 97 | int ctrl_chrdev_create(struct ctrl* ctrl, dev_t first, const struct file_operations* fops) 98 | { 99 | int err; 100 | struct device* chrdev = NULL; 101 | 102 | if (ctrl->chrdev != NULL) 103 | { 104 | printk(KERN_WARNING "Character device is already created\n"); 105 | return 0; 106 | } 107 | 108 | ctrl->rdev = MKDEV(MAJOR(first), MINOR(first) + ctrl->number); 109 | 110 | cdev_init(&ctrl->cdev, fops); 111 | err = cdev_add(&ctrl->cdev, ctrl->rdev, 1); 112 | if (err != 0) 113 | { 114 | printk(KERN_ERR "Failed to add cdev\n"); 115 | return err; 116 | } 117 | 118 | chrdev = device_create(ctrl->cls, NULL, ctrl->rdev, NULL, ctrl->name); 119 | if (IS_ERR(chrdev)) 120 | { 121 | cdev_del(&ctrl->cdev); 122 | printk(KERN_ERR "Failed to create character device\n"); 123 | return PTR_ERR(chrdev); 124 | } 125 | 126 | ctrl->chrdev = chrdev; 127 | 128 | printk(KERN_INFO "Character device /dev/%s created (%d.%d)\n", 129 | ctrl->name, MAJOR(ctrl->rdev), MINOR(ctrl->rdev)); 130 | 131 | return 0; 132 | } 133 | 134 | 135 | 136 | void ctrl_chrdev_remove(struct ctrl* ctrl) 137 | { 138 | if (ctrl->chrdev != NULL) 139 | { 140 | device_destroy(ctrl->cls, ctrl->rdev); 141 | cdev_del(&ctrl->cdev); 142 | ctrl->chrdev = NULL; 143 | 144 | printk(KERN_DEBUG "Character device /dev/%s removed (%d.%d)\n", 145 | ctrl->name, MAJOR(ctrl->rdev), MINOR(ctrl->rdev)); 146 | } 147 | } 148 | 149 | -------------------------------------------------------------------------------- /module/ctrl.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_HELPER_CTRL_H__ 2 | #define __LIBNVM_HELPER_CTRL_H__ 3 | 4 | #include "list.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | /* 12 | * Represents an NVM controller. 13 | */ 14 | struct ctrl 15 | { 16 | struct list_node list; /* Linked list head */ 17 | struct pci_dev* pdev; /* Reference to physical PCI device */ 18 | char name[64]; /* Character device name */ 19 | int number; /* Controller number */ 20 | dev_t rdev; /* Character device register */ 21 | struct class* cls; /* Character device class */ 22 | struct cdev cdev; /* Character device */ 23 | struct device* chrdev; /* Character device handle */ 24 | }; 25 | 26 | 27 | 28 | /* 29 | * Acquire a controller reference. 30 | */ 31 | struct ctrl* ctrl_get(struct list* list, struct class* cls, struct pci_dev* pdev, int number); 32 | 33 | 34 | 35 | /* 36 | * Release controller reference. 37 | */ 38 | void ctrl_put(struct ctrl* ctrl); 39 | 40 | 41 | 42 | /* 43 | * Find controller device. 44 | */ 45 | struct ctrl* ctrl_find_by_pci_dev(const struct list* list, const struct pci_dev* pdev); 46 | 47 | 48 | 49 | /* 50 | * Find controller reference. 51 | */ 52 | struct ctrl* ctrl_find_by_inode(const struct list* list, const struct inode* inode); 53 | 54 | 55 | 56 | /* 57 | * Create character device and set up file operations. 58 | */ 59 | int ctrl_chrdev_create(struct ctrl* ctrl, 60 | dev_t first, 61 | const struct file_operations* fops); 62 | 63 | 64 | 65 | /* 66 | * Remove character device. 67 | */ 68 | void ctrl_chrdev_remove(struct ctrl* ctrl); 69 | 70 | 71 | 72 | #endif /* __LIBNVM_HELPER_CTRL_H__ */ 73 | -------------------------------------------------------------------------------- /module/list.c: -------------------------------------------------------------------------------- 1 | #include "list.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | 10 | void list_init(struct list* list) 11 | { 12 | list->head.list = list; 13 | list->head.prev = &list->head; 14 | list->head.next = &list->head; 15 | 16 | spin_lock_init(&list->lock); 17 | } 18 | 19 | 20 | 21 | void list_remove(struct list_node* element) 22 | { 23 | if (likely(element != NULL && element->list != NULL && element != &element->list->head)) 24 | { 25 | spin_lock(&element->list->lock); 26 | element->prev->next = element->next; 27 | element->next->prev = element->prev; 28 | spin_unlock(&element->list->lock); 29 | 30 | element->list = NULL; 31 | element->next = NULL; 32 | element->prev = NULL; 33 | } 34 | } 35 | 36 | 37 | 38 | void list_insert(struct list* list, struct list_node* element) 39 | { 40 | struct list_node* last = NULL; 41 | 42 | spin_lock(&list->lock); 43 | last = list->head.prev; 44 | last->next = element; 45 | 46 | element->list = list; 47 | element->prev = last; 48 | element->next = &list->head; 49 | 50 | list->head.prev = element; 51 | 52 | spin_unlock(&list->lock); 53 | } 54 | 55 | -------------------------------------------------------------------------------- /module/list.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_HELPER_LIST_H__ 2 | #define __LIBNVM_HELPER_LIST_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | /* Forward declaration */ 10 | struct list; 11 | 12 | 13 | /* 14 | * Doubly linked list element. 15 | */ 16 | struct list_node 17 | { 18 | struct list* list; /* Reference to list */ 19 | struct list_node* next; /* Pointer to next element in list */ 20 | struct list_node* prev; /* Pointer to previous element in list */ 21 | }; 22 | 23 | 24 | /* 25 | * Doubly linked list. 26 | * This implementation expects there always be an empty head. 27 | */ 28 | struct list 29 | { 30 | struct list_node head; /* Start of the list */ 31 | spinlock_t lock; /* Ensure exclusive access to list */ 32 | }; 33 | 34 | 35 | 36 | /* 37 | * Initialize element. 38 | */ 39 | static void __always_inline list_node_init(struct list_node* element) 40 | { 41 | element->list = NULL; 42 | element->next = NULL; 43 | element->prev = NULL; 44 | } 45 | 46 | 47 | 48 | /* 49 | * Get next element in list (if there are any) 50 | */ 51 | #define list_next(current) \ 52 | ( ((current)->next != &(current)->list->head) ? (current)->next : NULL ) 53 | 54 | 55 | 56 | /* 57 | * Initialize list. 58 | */ 59 | void list_init(struct list* list); 60 | 61 | 62 | 63 | /* 64 | * Insert element into list. 65 | */ 66 | void list_insert(struct list* list, struct list_node* element); 67 | 68 | 69 | 70 | /* 71 | * Remove element from list. 72 | */ 73 | void list_remove(struct list_node* element); 74 | 75 | 76 | 77 | #endif /* __LIBNVM_HELPER_LIST_H__ */ 78 | -------------------------------------------------------------------------------- /module/map.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBNVM_HELPER_MAP_H__ 2 | #define __LIBNVM_HELPER_MAP_H__ 3 | 4 | #include "list.h" 5 | #include 6 | #include 7 | 8 | 9 | /* Forward declaration */ 10 | struct ctrl; 11 | struct map; 12 | 13 | 14 | typedef void (*release)(struct map*); 15 | 16 | 17 | /* 18 | * Describes a range of mapped memory. 19 | */ 20 | struct map 21 | { 22 | struct list_node list; /* Linked list header */ 23 | struct task_struct* owner; /* Owner of mapping */ 24 | u64 vaddr; /* Starting virtual address */ 25 | struct pci_dev* pdev; /* Reference to physical PCI device */ 26 | unsigned long page_size; /* Logical page size */ 27 | void* data; /* Custom data */ 28 | release release; /* Custom callback for unmapping and releasing memory */ 29 | unsigned long n_addrs; /* Number of mapped pages */ 30 | uint64_t addrs[1]; /* Bus addresses */ 31 | }; 32 | 33 | 34 | 35 | /* 36 | * Lock and map userspace pages for DMA. 37 | */ 38 | struct map* map_userspace(struct list* list, const struct ctrl* ctrl, u64 vaddr, unsigned long n_pages); 39 | 40 | 41 | 42 | /* 43 | * Unmap and release memory. 44 | */ 45 | void unmap_and_release(struct map* map); 46 | 47 | 48 | 49 | #ifdef _CUDA 50 | /* 51 | * Lock and map GPU device memory. 52 | */ 53 | struct map* map_device_memory(struct list* list, const struct ctrl* ctrl, u64 vaddr, unsigned long n_pages); 54 | #endif 55 | 56 | 57 | 58 | /* 59 | * Find memory mapping from vaddr and current task 60 | */ 61 | struct map* map_find(const struct list* list, u64 vaddr); 62 | 63 | 64 | #endif /* __LIBNVM_HELPER_MAP_H__ */ 65 | -------------------------------------------------------------------------------- /src/admin.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_ADMIN_H__ 2 | #define __NVM_INTERNAL_ADMIN_H__ 3 | 4 | 5 | 6 | /* 7 | * Create IO completion queue (CQ). 8 | * 9 | * Build an NVM admin command for creating a CQ. 10 | */ 11 | void _nvm_admin_cq_create(nvm_cmd_t* cmd, const nvm_queue_t* cq); 12 | 13 | 14 | 15 | /* 16 | * Create IO submission queue (SQ). 17 | * 18 | * Build an NVM admin command for creating an SQ. Note that the associated 19 | * CQ must have been created first. 20 | */ 21 | void _nvm_admin_sq_create(nvm_cmd_t* cmd, const nvm_queue_t* cq, const nvm_queue_t* sq); 22 | 23 | 24 | 25 | /* 26 | * Delete IO submission queue (SQ). 27 | * 28 | * Build an NVM admin command for deleting an SQ. 29 | */ 30 | void _nvm_admin_sq_delete(nvm_cmd_t* cmd, const nvm_queue_t* sq); 31 | 32 | 33 | 34 | /* 35 | * Delete IO completion queue (CQ). 36 | * 37 | * Build an NVM admin command for deleting a CQ. Note that the associated 38 | * SQ must have been deleted first. 39 | */ 40 | void _nvm_admin_cq_delete(nvm_cmd_t* cmd, const nvm_queue_t* cq); 41 | 42 | 43 | 44 | /* 45 | * Identify controller. 46 | * 47 | * Build an NVM admin command for identifying the controller. 48 | */ 49 | void _nvm_admin_identify_ctrl(nvm_cmd_t* cmd, uint64_t ioaddr); 50 | 51 | 52 | 53 | /* 54 | * Identify namespace. 55 | */ 56 | void _nvm_admin_identify_ns(nvm_cmd_t* cmd, uint32_t ns_id, uint64_t ioaddr); 57 | 58 | 59 | 60 | /* 61 | * Set/get current number of queues. 62 | */ 63 | void _nvm_admin_current_num_queues(nvm_cmd_t* cmd, bool set, uint16_t n_cqs, uint16_t n_sqs); 64 | 65 | /* 66 | * Get log page. 67 | */ 68 | int nvm_admin_get_log_page(nvm_aq_ref ref, uint32_t ns_id, void* ptr, uint64_t ioaddr, uint8_t log_id, uint64_t log_offset); 69 | 70 | #endif /* __NVM_INTERNAL_ADMIN_H__ */ 71 | -------------------------------------------------------------------------------- /src/ctrl.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_CTRL_H__ 2 | #define __NVM_INTERNAL_CTRL_H__ 3 | 4 | #include 5 | 6 | 7 | /* Forward declaration */ 8 | struct device; 9 | 10 | 11 | 12 | /* 13 | * Look up file descriptor from controller handle. 14 | * Returns a valid file descriptor (positive integer) or -EBADF if the 15 | * controller memory is not mapped through this library. 16 | */ 17 | int _nvm_fd_from_ctrl(const nvm_ctrl_t* ctrl); 18 | 19 | 20 | 21 | #ifdef _SISCI 22 | /* 23 | * Look up device reference from controller handle. 24 | * Returns a device reference if used, or NULL if not used. 25 | */ 26 | const struct device* _nvm_device_from_ctrl(const nvm_ctrl_t* ctrl); 27 | #endif 28 | 29 | 30 | 31 | #endif /* __NVM_INTERNAL_CTRL_H__ */ 32 | -------------------------------------------------------------------------------- /src/dis/device.c: -------------------------------------------------------------------------------- 1 | #ifndef _SISCI 2 | #error "Must compile with SISCI support" 3 | #endif 4 | 5 | #ifndef __DIS_CLUSTER__ 6 | #define __DIS_CLUSTER__ 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "dis/device.h" 14 | #include "dprintf.h" 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | 21 | /* 22 | * Increase device reference. 23 | */ 24 | int _nvm_device_get(struct device* dev, uint64_t dev_id) 25 | { 26 | sci_error_t err; 27 | 28 | if (dev == NULL) 29 | { 30 | return EINVAL; 31 | } 32 | 33 | dev->device_id = dev_id; 34 | 35 | SCIOpen(&dev->sd, 0, &err); 36 | if (err != SCI_ERR_OK) 37 | { 38 | dprintf("Failed to create virtual device: %s\n", _SCIGetErrorString(err)); 39 | return EIO; 40 | } 41 | 42 | SCIBorrowDevice(dev->sd, &dev->device, dev_id, 0, &err); 43 | if (err != SCI_ERR_OK) 44 | { 45 | dprintf("Failed to increase device reference: %s\n", _SCIGetErrorString(err)); 46 | SCIClose(dev->sd, 0, &err); 47 | return ENODEV; 48 | } 49 | 50 | return 0; 51 | } 52 | 53 | 54 | 55 | /* 56 | * Decrease device reference. 57 | */ 58 | void _nvm_device_put(struct device* dev) 59 | { 60 | sci_error_t err; 61 | 62 | if (dev != NULL) 63 | { 64 | SCIReturnDevice(dev->device, 0, &err); 65 | SCIClose(dev->sd, 0, &err); 66 | } 67 | } 68 | 69 | 70 | 71 | /* 72 | * Helper function to connect to a segment. 73 | */ 74 | int _nvm_device_memory_get(struct device_memory* mem, 75 | const struct device* dev, 76 | uint32_t adapter, 77 | uint32_t segment_no, 78 | uint32_t flags) 79 | { 80 | sci_error_t err; 81 | int status = 0; 82 | 83 | status = _nvm_device_get(&mem->device, dev->device_id); 84 | if (status != 0) 85 | { 86 | goto leave; 87 | } 88 | 89 | const struct device* d = &mem->device; 90 | mem->adapter = adapter; 91 | mem->segment_no = segment_no; 92 | mem->flags = flags; 93 | 94 | SCIConnectDeviceMemory(d->sd, &mem->segment, mem->adapter, d->device, segment_no, 0, flags, &err); 95 | if (err != SCI_ERR_OK) 96 | { 97 | dprintf("Failed to connect to device memory: %s\n", _SCIGetErrorString(err)); 98 | status = ENODEV; 99 | goto release; 100 | } 101 | 102 | return 0; 103 | 104 | release: 105 | _nvm_device_put(&mem->device); 106 | 107 | leave: 108 | return status; 109 | } 110 | 111 | 112 | 113 | /* 114 | * Disconnect from device memory. 115 | */ 116 | void _nvm_device_memory_put(struct device_memory* mem) 117 | { 118 | sci_error_t err = SCI_ERR_OK; 119 | 120 | do 121 | { 122 | SCIDisconnectSegment(mem->segment, 0, &err); 123 | } 124 | while (err == SCI_ERR_BUSY); 125 | 126 | #ifndef NDEBUG 127 | if (err != SCI_ERR_OK) 128 | { 129 | dprintf("Failed to disconnect from device memory: %s\n", _SCIGetErrorString(err)); 130 | } 131 | #endif 132 | 133 | _nvm_device_put(&mem->device); 134 | } 135 | 136 | -------------------------------------------------------------------------------- /src/dis/device.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_DIS_DEVICE_H__ 2 | #define __NVM_INTERNAL_DIS_DEVICE_H__ 3 | 4 | /* Forward declarations */ 5 | struct device; 6 | struct device_memory; 7 | 8 | 9 | #ifdef _SISCI 10 | 11 | /* Make sure everything is defined as needed */ 12 | #ifndef __DIS_CLUSTER__ 13 | #define __DIS_CLUSTER__ 14 | #endif 15 | 16 | /* Necessary includes */ 17 | #include 18 | #include 19 | #include 20 | #include "dis/map.h" 21 | 22 | 23 | 24 | /* 25 | * Device descriptor. 26 | * 27 | * Holds a "borrowed" reference to a SISCI SmartIO device. 28 | * Handles that require a device or controller reference should take this. 29 | */ 30 | struct device 31 | { 32 | sci_desc_t sd; // SISCI virtual device descriptor 33 | uint64_t device_id; // SISCI SmartIO device identifier 34 | sci_device_t device; // SmartIO device handle 35 | }; 36 | 37 | 38 | 39 | /* 40 | * Device memory segment descriptor. 41 | * 42 | * Describes mapping to device-local memory, e.g. PCI BARs or memory residing 43 | * on the same host as the device. 44 | */ 45 | struct device_memory 46 | { 47 | struct device device; // Device reference 48 | uint32_t adapter; // DIS adapter number 49 | uint32_t segment_no; // Device segment number 50 | sci_remote_segment_t segment; // SISCI remote segment to device memory 51 | uint32_t flags; // SISCI flags used when connecting segment 52 | }; 53 | 54 | 55 | 56 | /* 57 | * Acquire device reference (increase). 58 | */ 59 | int _nvm_device_get(struct device* dev, uint64_t dev_id); 60 | 61 | 62 | 63 | /* 64 | * Release device reference (decrease). 65 | */ 66 | void _nvm_device_put(struct device* dev); 67 | 68 | 69 | 70 | /* 71 | * Connect to device memory. 72 | * This function will increase the device reference. 73 | */ 74 | int _nvm_device_memory_get(struct device_memory* mem, 75 | const struct device* dev, 76 | uint32_t adapter, 77 | uint32_t segment_no, 78 | uint32_t flags); 79 | 80 | 81 | 82 | /* 83 | * Disconnect from device memory. 84 | * This function will decrease the device reference. 85 | */ 86 | void _nvm_device_memory_put(struct device_memory* mem); 87 | 88 | 89 | 90 | #endif /* _SISCI */ 91 | #endif /* __NVM_INTERNAL_DIS_DEVICE_H__ */ 92 | -------------------------------------------------------------------------------- /src/dis/interrupt.c: -------------------------------------------------------------------------------- 1 | #ifndef _SISCI 2 | #error "Must compile with SISCI support" 3 | #endif 4 | 5 | #ifndef __DIS_CLUSTER__ 6 | #define __DIS_CLUSTER__ 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "dis/interrupt.h" 14 | #include "dprintf.h" 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | 21 | /* 22 | * Do some sanity checking and then call supplied callback. 23 | */ 24 | static sci_callback_action_t interrupt_callback(struct local_intr* interrupt, 25 | sci_local_data_interrupt_t intr, 26 | void* data, 27 | uint32_t length, 28 | sci_error_t status) 29 | { 30 | #ifndef NDEBUG 31 | if (status != SCI_ERR_OK) 32 | { 33 | dprintf("Unexpected status in interrupt handler routine: %s\n", _SCIGetErrorString(status)); 34 | return SCI_CALLBACK_CANCEL; 35 | } 36 | 37 | if (intr != interrupt->intr) 38 | { 39 | dprintf("Possible memory corruption\n"); 40 | return SCI_CALLBACK_CANCEL; 41 | } 42 | #endif 43 | 44 | interrupt->callback(interrupt->data, data, length); 45 | 46 | return SCI_CALLBACK_CONTINUE; 47 | } 48 | 49 | 50 | 51 | int _nvm_local_intr_get(struct local_intr* intr, uint32_t adapter, void* cb_data, intr_callback_t cb) 52 | { 53 | sci_error_t err = SCI_ERR_OK; 54 | 55 | // Get local node identifier 56 | SCIGetLocalNodeId(adapter, &intr->node_id, 0, &err); 57 | #ifndef NDEBUG 58 | if (err != SCI_ERR_OK) 59 | { 60 | dprintf("Unexpected error: %s\n", _SCIGetErrorString(err)); 61 | return EIO; 62 | } 63 | #endif 64 | 65 | // Open SISCI descriptor 66 | SCIOpen(&intr->sd, 0, &err); 67 | #ifndef NDEBUG 68 | if (err != SCI_ERR_OK) 69 | { 70 | dprintf("Failed to open SISCI virtual device: %s\n", _SCIGetErrorString(err)); 71 | return EIO; 72 | } 73 | #endif 74 | 75 | intr->data = cb_data; 76 | intr->callback = cb; 77 | 78 | uint32_t flags = 0; 79 | void* data = NULL; 80 | sci_cb_data_interrupt_t callback = NULL; 81 | 82 | // Callback was supplied, set up parameters 83 | if (cb != NULL) 84 | { 85 | data = (void*) intr; 86 | callback = (sci_cb_data_interrupt_t) interrupt_callback; 87 | flags |= SCI_FLAG_USE_CALLBACK; 88 | } 89 | 90 | // Create data interrupt 91 | SCICreateDataInterrupt(intr->sd, &intr->intr, adapter, &intr->intr_no, callback, data, flags, &err); 92 | if (err != SCI_ERR_OK) 93 | { 94 | dprintf("Failed to create data interrupt: %s\n", _SCIGetErrorString(err)); 95 | SCIClose(intr->sd, 0, &err); 96 | return ENOSPC; 97 | } 98 | 99 | return 0; 100 | } 101 | 102 | 103 | 104 | void _nvm_local_intr_put(struct local_intr* intr) 105 | { 106 | sci_error_t err = SCI_ERR_OK; 107 | 108 | do 109 | { 110 | SCIRemoveDataInterrupt(intr->intr, 0, &err); 111 | } 112 | while (err == SCI_ERR_BUSY); 113 | 114 | SCIClose(intr->sd, 0, &err); 115 | } 116 | 117 | 118 | 119 | int _nvm_local_intr_wait(struct local_intr* intr, void* data, uint16_t maxlen, uint32_t timeout) 120 | { 121 | sci_error_t err = SCI_ERR_OK; 122 | uint32_t len = maxlen; 123 | 124 | SCIWaitForDataInterrupt(intr->intr, data, &len, timeout, 0, &err); 125 | 126 | switch (err) 127 | { 128 | case SCI_ERR_OK: 129 | return 0; 130 | 131 | case SCI_ERR_TIMEOUT: 132 | return ETIMEDOUT; 133 | 134 | default: 135 | dprintf("Waiting for data interrupt unexpectedly failed: %s\n", _SCIGetErrorString(err)); 136 | return EIO; 137 | } 138 | } 139 | 140 | 141 | 142 | int _nvm_remote_intr_get(struct remote_intr* intr, uint32_t adapter, uint32_t node, uint32_t no) 143 | { 144 | sci_error_t err = SCI_ERR_OK; 145 | 146 | SCIOpen(&intr->sd, 0, &err); 147 | #ifndef NDEBUG 148 | if (err != SCI_ERR_OK) 149 | { 150 | dprintf("Failed to open SISCI virtual device: %s\n", _SCIGetErrorString(err)); 151 | return EIO; 152 | } 153 | #endif 154 | 155 | SCIConnectDataInterrupt(intr->sd, &intr->intr, node, adapter, no, SCI_INFINITE_TIMEOUT, 0, &err); 156 | if (err != SCI_ERR_OK) 157 | { 158 | SCIClose(intr->sd, 0, &err); 159 | return ECONNREFUSED; 160 | } 161 | 162 | return 0; 163 | } 164 | 165 | 166 | 167 | void _nvm_remote_intr_put(struct remote_intr* intr) 168 | { 169 | sci_error_t err = SCI_ERR_OK; 170 | SCIDisconnectDataInterrupt(intr->intr, 0, &err); 171 | SCIClose(intr->sd, 0, &err); 172 | } 173 | 174 | 175 | 176 | /* 177 | * Trigger remote interrupt with data. 178 | */ 179 | int _nvm_remote_intr_trigger(const struct remote_intr* intr, void* data, uint16_t length) 180 | { 181 | sci_error_t err = SCI_ERR_OK; 182 | 183 | SCITriggerDataInterrupt(intr->intr, data, length, 0, &err); 184 | if (err != SCI_ERR_OK) 185 | { 186 | dprintf("Failed to trigger data interrupt\n"); 187 | return ENOTCONN; 188 | } 189 | 190 | return 0; 191 | } 192 | 193 | 194 | 195 | /* 196 | * Convenience function for easy remote interrupt triggering. 197 | */ 198 | int _nvm_remote_intr_fire_and_forget(uint32_t adapter, uint32_t node, uint32_t no, void* data, uint16_t len) 199 | { 200 | int status = 0; 201 | struct remote_intr intr; 202 | 203 | status = _nvm_remote_intr_get(&intr, adapter, node, no); 204 | if (status != 0) 205 | { 206 | return status; 207 | } 208 | 209 | status = _nvm_remote_intr_trigger(&intr, data, len); 210 | _nvm_remote_intr_put(&intr); 211 | return status; 212 | } 213 | 214 | -------------------------------------------------------------------------------- /src/dis/interrupt.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_DIS_INTERRUPT_H__ 2 | #define __NVM_INTERNAL_DIS_INTERRUPT_H__ 3 | 4 | /* Forward declarations */ 5 | struct local_intr; 6 | struct remote_intr; 7 | 8 | 9 | #ifdef _SISCI 10 | 11 | /* Make sure everything is defined as needed */ 12 | #ifndef __DIS_CLUSTER__ 13 | #define __DIS_CLUSTER__ 14 | #endif 15 | 16 | /* Necessary includes */ 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | 23 | /* 24 | * Interrupt callback. 25 | */ 26 | typedef void (*intr_callback_t)(void* user_data, void* recv_data, uint16_t length); 27 | 28 | 29 | 30 | /* 31 | * Local interrupt descriptor. 32 | * Data must be free'd manually. 33 | */ 34 | struct local_intr 35 | { 36 | sci_desc_t sd; // SISCI virtual device descriptor 37 | sci_local_data_interrupt_t intr; // SISCI data interrupt handle 38 | uint32_t adapter; // DIS adapter 39 | uint32_t intr_no; // Interrupt number 40 | uint32_t node_id; // DIS node identifier 41 | void* data; // User data 42 | intr_callback_t callback; // Interrupt callback 43 | }; 44 | 45 | 46 | 47 | /* 48 | * Remote interrupt descriptor. 49 | */ 50 | struct remote_intr 51 | { 52 | sci_desc_t sd; // SISCI virtual device descriptor 53 | sci_remote_data_interrupt_t intr; // SISCI data interrupt reference 54 | }; 55 | 56 | 57 | 58 | /* 59 | * Create a local data interrupt. 60 | */ 61 | int _nvm_local_intr_get(struct local_intr* intr, 62 | uint32_t adapter, 63 | void* cb_data, 64 | intr_callback_t cb_func); 65 | 66 | 67 | 68 | /* 69 | * Remove a local data interrupt. 70 | */ 71 | void _nvm_local_intr_put(struct local_intr* intr); 72 | 73 | 74 | 75 | /* 76 | * Block for a duration while waiting for an interrupt and removes interrupt afterwards. 77 | * Returns success if length of received data matches expected length. 78 | */ 79 | int _nvm_local_intr_wait(struct local_intr* intr, void* data, uint16_t maxlen, uint32_t timeout); 80 | 81 | 82 | 83 | /* 84 | * Connect to remote interrupt. 85 | */ 86 | int _nvm_remote_intr_get(struct remote_intr* intr, uint32_t adapter, uint32_t node_id, uint32_t intr_no); 87 | 88 | 89 | 90 | /* 91 | * Disconnect from remote interrupt. 92 | */ 93 | void _nvm_remote_intr_put(struct remote_intr* intr); 94 | 95 | 96 | 97 | /* 98 | * Trigger remote interrupt with data. 99 | */ 100 | int _nvm_remote_intr_trigger(const struct remote_intr* intr, void* data, uint16_t len); 101 | 102 | 103 | 104 | /* 105 | * Connect to remote interrupt, send data, and disconnect. 106 | */ 107 | int _nvm_remote_intr_fire_and_forget(uint32_t adapter, 108 | uint32_t node_id, 109 | uint32_t intr_no, 110 | void* data, 111 | uint16_t len); 112 | 113 | #endif /* _SISCI */ 114 | #endif /* __NVM_INTERNAL_DIS_INTERRUPT_H__ */ 115 | -------------------------------------------------------------------------------- /src/dis/local.c: -------------------------------------------------------------------------------- 1 | #ifndef _SISCI 2 | #error "Must compile with SISCI support" 3 | #endif 4 | 5 | #ifndef __DIS_CLUSTER__ 6 | #define __DIS_CLUSTER__ 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "dis/local.h" 14 | #include "dprintf.h" 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | 21 | /* 22 | * Helper function to create a local segment. 23 | */ 24 | static int create_segment(struct local_memory* m, uint32_t id, size_t size, uint32_t flags) 25 | { 26 | sci_error_t err; 27 | sci_error_t status = SCI_ERR_OK; 28 | 29 | m->id = id; 30 | m->size = size; 31 | m->flags = flags; 32 | 33 | SCIOpen(&m->sd, 0, &err); 34 | if (err != SCI_ERR_OK) 35 | { 36 | dprintf("Failed to open SISCI descriptor: %s\n", _SCIGetErrorString(err)); 37 | return EIO; 38 | } 39 | 40 | SCICreateSegment(m->sd, &m->segment, id, size, NULL, NULL, flags, &err); 41 | if (err != SCI_ERR_OK) 42 | { 43 | status = err; 44 | SCIClose(m->sd, 0, &err); 45 | } 46 | 47 | switch (status) 48 | { 49 | case SCI_ERR_OK: 50 | return 0; 51 | 52 | case SCI_ERR_SEGMENTID_USED: 53 | return EEXIST; 54 | 55 | default: 56 | dprintf("Unknown error while creating local segment: %s\n", _SCIGetErrorString(status)); 57 | return EIO; 58 | } 59 | } 60 | 61 | 62 | 63 | void _nvm_local_memory_put(struct local_memory* mem) 64 | { 65 | sci_error_t err = SCI_ERR_OK; 66 | 67 | do 68 | { 69 | SCIRemoveSegment(mem->segment, 0, &err); 70 | } 71 | while (err == SCI_ERR_BUSY); 72 | 73 | #ifndef NDEBUG 74 | if (err != SCI_ERR_OK) 75 | { 76 | dprintf("Unknown error while removing local segment: %s\n", _SCIGetErrorString(err)); 77 | } 78 | #endif 79 | } 80 | 81 | 82 | 83 | int _nvm_local_memory_get(struct local_memory* mem, uint32_t id, size_t size) 84 | { 85 | int err; 86 | 87 | // FIXME: When support is added, remove id argument and pass SCI_FLAG_PRIVATE 88 | err = create_segment(mem, id, size, 0); 89 | //err = create_segment(mem, id, size, SCI_FLAG_PRIVATE); 90 | if (err != 0) 91 | { 92 | return err; 93 | } 94 | 95 | return 0; 96 | } 97 | 98 | 99 | 100 | int _nvm_local_memory_get_attached(struct local_memory* mem, uint32_t id, void* ptr, size_t size) 101 | { 102 | sci_error_t err; 103 | int status; 104 | 105 | // FIXME: When support is added, use SCI_FLAG_PRIVATE 106 | //status = create_segment(mem, id, size, SCI_FLAG_EMPTY | SCI_FLAG_PRIVATE); 107 | status = create_segment(mem, id, size, SCI_FLAG_EMPTY); 108 | if (status != 0) 109 | { 110 | return status; 111 | } 112 | 113 | SCIAttachPhysicalMemory(0, ptr, 0, size, mem->segment, SCI_FLAG_CUDA_BUFFER, &err); 114 | if (err != SCI_ERR_OK) 115 | { 116 | _nvm_local_memory_put(mem); 117 | dprintf("Failed to attach memory to local segment: %s\n", _SCIGetErrorString(err)); 118 | return EIO; 119 | } 120 | 121 | return 0; 122 | } 123 | 124 | 125 | 126 | int _nvm_local_memory_get_registered(struct local_memory* mem, uint32_t id, void* ptr, size_t size) 127 | { 128 | sci_error_t err; 129 | int status; 130 | 131 | // FIXME: Add support for SCI_FLAG_PRIVATE 132 | status = create_segment(mem, id, size, SCI_FLAG_EMPTY /* | SCI_FLAG_PRIVATE */); 133 | if (status != 0) 134 | { 135 | return status; 136 | } 137 | 138 | SCIRegisterSegmentMemory(ptr, size, mem->segment, SCI_FLAG_LOCK_USER_MEM, &err); 139 | 140 | switch (err) 141 | { 142 | case SCI_ERR_OK: 143 | // Do nothing 144 | break; 145 | 146 | case SCI_ERR_SIZE_ALIGNMENT: 147 | _nvm_local_memory_put(mem); 148 | return EINVAL; 149 | 150 | case SCI_ERR_OUT_OF_RANGE: 151 | _nvm_local_memory_put(mem); 152 | return ERANGE; 153 | 154 | case SCI_ERR_ILLEGAL_ADDRESS: 155 | _nvm_local_memory_put(mem); 156 | return EPERM; 157 | 158 | case SCI_ERR_NOT_SUPPORTED: 159 | _nvm_local_memory_put(mem); 160 | dprintf("Function is not supported on platform\n"); 161 | return ENOTSUP; 162 | 163 | default: 164 | _nvm_local_memory_put(mem); 165 | dprintf("Unknown error when registering host memory: %s\n", _SCIGetErrorString(err)); 166 | return EIO; 167 | } 168 | 169 | return 0; 170 | } 171 | 172 | -------------------------------------------------------------------------------- /src/dis/local.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_DIS_LOCAL_MEMORY_H__ 2 | #define __NVM_INTERNAL_DIS_LOCAL_MEMORY_H__ 3 | 4 | /* Forward declarations */ 5 | struct local_memory; 6 | 7 | 8 | #ifdef _SISCI 9 | 10 | /* Make sure everything is defined as needed */ 11 | #ifndef __DIS_CLUSTER__ 12 | #define __DIS_CLUSTER__ 13 | #endif 14 | 15 | /* Necessary includes */ 16 | #include 17 | #include 18 | #include 19 | 20 | 21 | 22 | /* 23 | * Local memory descriptor. 24 | * 25 | * Describes a local memory segment and its mapping into address space. 26 | */ 27 | struct local_memory 28 | { 29 | sci_desc_t sd; // SISCI virtual device descriptor 30 | sci_local_segment_t segment; // SISCI local segment descriptor 31 | uint32_t id; // Local segment identifier 32 | size_t size; // Size of local memory 33 | uint32_t flags; // SISCI flags used when creating segment 34 | }; 35 | 36 | 37 | 38 | /* 39 | * Create local segment and map it into local address space. 40 | */ 41 | int _nvm_local_memory_get(struct local_memory* mem, uint32_t id, size_t size); 42 | 43 | 44 | 45 | /* 46 | * Create local segment with attached memory. 47 | * This function will NOT map segment into address space. 48 | */ 49 | int _nvm_local_memory_get_attached(struct local_memory* mem, uint32_t id, void* ptr, size_t size); 50 | 51 | 52 | 53 | /* 54 | * Create local segment with registered host memory. 55 | * This function will not map segment into address space (as it is already mapped). 56 | */ 57 | int _nvm_local_memory_get_registered(struct local_memory* mem, uint32_t id, void* ptr, size_t size); 58 | 59 | 60 | /* 61 | * Remove local segment. 62 | */ 63 | void _nvm_local_memory_put(struct local_memory* mem); 64 | 65 | 66 | 67 | #endif /* _SISCI */ 68 | #endif /* __NVM_INTERNAL_DIS_LOCAL_MEMORY_H__ */ 69 | -------------------------------------------------------------------------------- /src/dis/map.c: -------------------------------------------------------------------------------- 1 | #ifndef _SISCI 2 | #error "Must compile with SISCI support" 3 | #endif 4 | 5 | #ifndef __DIS_CLUSTER__ 6 | #define __DIS_CLUSTER__ 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "dis/map.h" 14 | #include "dis/local.h" 15 | #include "dis/device.h" 16 | #include "dprintf.h" 17 | #include 18 | #include 19 | #include 20 | 21 | 22 | 23 | int _nvm_va_map_local(struct va_map* m, size_t size, sci_local_segment_t segment) 24 | { 25 | sci_error_t err = SCI_ERR_OK; 26 | 27 | m->type = _VA_MAP_NOT_MAPPED; 28 | 29 | m->vaddr = SCIMapLocalSegment(segment, &m->md, 0, size, NULL, 0, &err); 30 | 31 | switch (err) 32 | { 33 | case SCI_ERR_OK: 34 | m->type = _VA_MAP_MAPPED_LOCAL; 35 | return 0; 36 | 37 | case SCI_ERR_OUT_OF_RANGE: 38 | case SCI_ERR_SIZE_ALIGNMENT: 39 | case SCI_ERR_OFFSET_ALIGNMENT: 40 | return EINVAL; 41 | 42 | default: 43 | dprintf("Mapping segment into virtual address space failed: %s\n", _SCIGetErrorString(err)); 44 | return EIO; 45 | } 46 | } 47 | 48 | 49 | int _nvm_va_map_remote(struct va_map* m, size_t size, sci_remote_segment_t segment, bool write, bool wc) 50 | { 51 | sci_error_t err = SCI_ERR_OK; 52 | uint32_t flags = 0; 53 | 54 | flags |= !write ? SCI_FLAG_READONLY_MAP : 0; 55 | flags |= !wc ? SCI_FLAG_IO_MAP_IOSPACE : 0; 56 | 57 | m->type = _VA_MAP_NOT_MAPPED; 58 | 59 | m->vaddr = SCIMapRemoteSegment(segment, &m->md, 0, size, NULL, flags, &err); 60 | 61 | switch (err) 62 | { 63 | case SCI_ERR_OK: 64 | m->type = _VA_MAP_MAPPED_REMOTE; 65 | return 0; 66 | 67 | case SCI_ERR_OUT_OF_RANGE: 68 | case SCI_ERR_SIZE_ALIGNMENT: 69 | case SCI_ERR_OFFSET_ALIGNMENT: 70 | return EINVAL; 71 | 72 | case SCI_ERR_NOT_CONNECTED: 73 | return ENOTCONN; 74 | 75 | default: 76 | dprintf("Mapping segment into virtual address space failed: %s\n", _SCIGetErrorString(err)); 77 | return EIO; 78 | } 79 | } 80 | 81 | 82 | 83 | void _nvm_va_unmap(struct va_map* m) 84 | { 85 | sci_error_t err = SCI_ERR_OK; 86 | 87 | switch (m->type) 88 | { 89 | case _VA_MAP_MAPPED_LOCAL: 90 | case _VA_MAP_MAPPED_REMOTE: 91 | do 92 | { 93 | SCIUnmapSegment(m->md, 0, &err); 94 | } 95 | while (err == SCI_ERR_BUSY); 96 | break; 97 | 98 | case _VA_MAP_NOT_MAPPED: 99 | // Do nothing 100 | break; 101 | 102 | default: 103 | dprintf("Unknown mapping type\n"); 104 | break; 105 | } 106 | 107 | m->type = _VA_MAP_NOT_MAPPED; 108 | 109 | #ifndef NDEBUG 110 | if (err != SCI_ERR_OK) 111 | { 112 | dprintf("Unmapping segment from virtual address space failed: %s\n", _SCIGetErrorString(err)); 113 | } 114 | #endif 115 | } 116 | 117 | 118 | 119 | int _nvm_io_map_local(struct io_map* m, sci_device_t device, sci_local_segment_t segment, uint32_t adapter) 120 | { 121 | sci_error_t err = SCI_ERR_OK; 122 | 123 | m->type = _IO_MAP_NOT_MAPPED; 124 | m->lsegment = segment; 125 | m->rsegment = NULL; 126 | m->device = device; 127 | m->adapter = adapter; 128 | m->ioaddr = 0; 129 | 130 | // FIXME: Remove this call when Lars adds support for segments created with SCI_FLAG_PRIVATE 131 | SCISetSegmentAvailable(m->lsegment, m->adapter, 0, &err); 132 | switch (err) 133 | { 134 | case SCI_ERR_OK: 135 | break; 136 | 137 | case SCI_ERR_ILLEGAL_OPERATION: 138 | case SCI_ERR_SEGMENT_NOT_PREPARED: 139 | return EINVAL; 140 | 141 | default: 142 | dprintf("Failed to set segment available: %s\n", _SCIGetErrorString(err)); 143 | return EIO; 144 | } 145 | 146 | SCIMapLocalSegmentForDevice(m->lsegment, m->adapter, m->device, &m->ioaddr, 0, 0, &err); 147 | if (err != SCI_ERR_OK) 148 | { 149 | dprintf("Failed to map segment for device: %s\n", _SCIGetErrorString(err)); 150 | return EIO; 151 | } 152 | 153 | m->type = _IO_MAP_MAPPED_LOCAL; 154 | return 0; 155 | } 156 | 157 | 158 | 159 | int _nvm_io_map_remote(struct io_map* m, sci_device_t device, sci_remote_segment_t segment) 160 | { 161 | sci_error_t err = SCI_ERR_OK; 162 | 163 | m->type = _IO_MAP_NOT_MAPPED; 164 | m->lsegment = NULL; 165 | m->rsegment = segment; 166 | m->device = device; 167 | m->adapter = 0; 168 | m->ioaddr = 0; 169 | 170 | SCIMapRemoteSegmentForDevice(m->rsegment, m->device, &m->ioaddr, 0, 0, &err); 171 | 172 | if (err != SCI_ERR_OK) 173 | { 174 | dprintf("Failed to map segment for device: %s\n", _SCIGetErrorString(err)); 175 | return EIO; 176 | } 177 | 178 | m->type = _IO_MAP_MAPPED_REMOTE; 179 | return 0; 180 | } 181 | 182 | 183 | 184 | void _nvm_io_unmap(struct io_map* m) 185 | { 186 | sci_error_t err = SCI_ERR_OK; 187 | 188 | switch (m->type) 189 | { 190 | case _IO_MAP_NOT_MAPPED: 191 | // Do nothing 192 | break; 193 | 194 | case _IO_MAP_MAPPED_LOCAL: 195 | do 196 | { 197 | SCIUnmapLocalSegmentForDevice(m->lsegment, m->adapter, m->device, 0, &err); 198 | } 199 | while (err == SCI_ERR_BUSY); 200 | break; 201 | 202 | case _IO_MAP_MAPPED_REMOTE: 203 | do 204 | { 205 | SCIUnmapRemoteSegmentForDevice(m->rsegment, m->device, 0, &err); 206 | } 207 | while (err == SCI_ERR_BUSY); 208 | break; 209 | 210 | default: 211 | dprintf("Unknown mapping type\n"); 212 | break; 213 | } 214 | 215 | #ifndef NDEBUG 216 | if (err != SCI_ERR_OK) 217 | { 218 | dprintf("Unmapping segment for device failed: %s\n", _SCIGetErrorString(err)); 219 | } 220 | #endif 221 | 222 | m->type = _IO_MAP_NOT_MAPPED; 223 | } 224 | 225 | 226 | 227 | int _nvm_io_map_device_memory(struct io_map* m, const struct device_memory* mem) 228 | { 229 | return _nvm_io_map_remote(m, mem->device.device, mem->segment); 230 | } 231 | 232 | 233 | 234 | int _nvm_io_map_local_memory(struct io_map* m, const struct device* dev, const struct local_memory* mem, uint32_t adapter) 235 | { 236 | sci_error_t err = SCI_ERR_OK; 237 | 238 | SCIPrepareSegment(mem->segment, adapter, 0, &err); 239 | if (err != SCI_ERR_OK) 240 | { 241 | dprintf("Failed to prepare local segment: %s\n", _SCIGetErrorString(err)); 242 | return ENOSPC; 243 | } 244 | 245 | return _nvm_io_map_local(m, dev->device, mem->segment, adapter); 246 | } 247 | 248 | -------------------------------------------------------------------------------- /src/dis/map.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_DIS_MAPPING_H__ 2 | #define __NVM_INTERNAL_DIS_MAPPING_H__ 3 | 4 | /* Forward declarations */ 5 | struct va_map; 6 | struct io_map; 7 | struct device; 8 | struct device_memory; 9 | struct local_memory; 10 | 11 | 12 | #ifdef _SISCI 13 | 14 | /* Make sure everything is defined as needed */ 15 | #ifndef __DIS_CLUSTER__ 16 | #define __DIS_CLUSTER__ 17 | #endif 18 | 19 | /* Necessary includes */ 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | 26 | 27 | /* 28 | * VA mapping kind. 29 | * Indicates whether or not a mapping is in use. 30 | */ 31 | enum va_map_type 32 | { 33 | _VA_MAP_NOT_MAPPED = 0x00, 34 | _VA_MAP_MAPPED_LOCAL = 0x01, 35 | _VA_MAP_MAPPED_REMOTE = 0x02 36 | }; 37 | 38 | /* 39 | * Virtual address mapping descriptor. 40 | * 41 | * Describes a mapping of a segment in local virtual address space. 42 | */ 43 | struct va_map 44 | { 45 | enum va_map_type type; // Type of mapping 46 | sci_map_t md; // SISCI mapping descriptor 47 | volatile void* vaddr; // Pointer to mapped memory 48 | }; 49 | 50 | #define VA_MAP_CLEAR(p) \ 51 | do { (p)->type = _VA_MAP_NOT_MAPPED; (p)->vaddr = NULL; } while (0) 52 | 53 | #define VA_MAP_INIT { .type = _VA_MAP_NOT_MAPPED, .md = NULL, .vaddr = NULL } 54 | 55 | 56 | 57 | /* 58 | * IO mapping kind. 59 | * Indicates whether or not a mapping is in use. 60 | */ 61 | enum io_map_type 62 | { 63 | _IO_MAP_NOT_MAPPED = 0x00, 64 | _IO_MAP_MAPPED_REMOTE = 0x01, 65 | _IO_MAP_MAPPED_LOCAL = 0x02, 66 | }; 67 | 68 | /* 69 | * IO address space mapping descriptor. 70 | * 71 | * Describes a mapping of a segment as seen from the device. 72 | */ 73 | struct io_map 74 | { 75 | enum io_map_type type; // Type of mapping 76 | sci_local_segment_t lsegment; // Local segment descriptor 77 | sci_remote_segment_t rsegment; // Remote segment descriptor 78 | sci_device_t device; // SmartIO device the segment is mapped for 79 | uint32_t adapter; // Local adapter 80 | sci_ioaddr_t ioaddr; // Device-local address to segment 81 | }; 82 | 83 | #define IO_MAP_CLEAR(p) \ 84 | do { (p)->type = _IO_MAP_NOT_MAPPED; (p)->ioaddr = 0; } while (0) 85 | 86 | #define IO_MAP_INIT \ 87 | { .type = _IO_MAP_NOT_MAPPED, .lsegment = NULL, .rsegment = NULL, .device = NULL, .adapter = 0, .ioaddr = 0 } 88 | 89 | 90 | 91 | 92 | 93 | /* 94 | * Map local segment into virtual address space. 95 | */ 96 | int _nvm_va_map_local(struct va_map* map, size_t size, sci_local_segment_t segment); 97 | 98 | 99 | /* 100 | * Map remote segment into virtual address space. 101 | */ 102 | int _nvm_va_map_remote(struct va_map* map, 103 | size_t size, 104 | sci_remote_segment_t segment, 105 | bool write, 106 | bool wc); 107 | 108 | 109 | /* 110 | * Unmap segment from virtual address space. 111 | */ 112 | void _nvm_va_unmap(struct va_map* map); 113 | 114 | 115 | 116 | /* 117 | * Map local segment for device. 118 | */ 119 | int _nvm_io_map_local(struct io_map* map, 120 | sci_device_t device, 121 | sci_local_segment_t segment, 122 | uint32_t adapter); 123 | 124 | 125 | /* 126 | * Map remote segment for device. 127 | */ 128 | int _nvm_io_map_remote(struct io_map* map, 129 | sci_device_t device, 130 | sci_remote_segment_t segment); 131 | 132 | 133 | /* 134 | * Unmap segment for device. 135 | */ 136 | void _nvm_io_unmap(struct io_map* map); 137 | 138 | 139 | 140 | /* 141 | * Convenience function to map device memory for device. 142 | */ 143 | int _nvm_io_map_device_memory(struct io_map* map, const struct device_memory* mem); 144 | 145 | 146 | 147 | /* 148 | * Convenience function to map local segment for device. 149 | * This will implicitly prepare segment on the specified adapter. 150 | */ 151 | int _nvm_io_map_local_memory(struct io_map* map, 152 | const struct device* dev, 153 | const struct local_memory* mem, 154 | uint32_t adapter); 155 | 156 | 157 | 158 | #endif /* _SISCI */ 159 | #endif /* __NVM_INTERNAL_DIS_MAPPING_H__ */ 160 | -------------------------------------------------------------------------------- /src/dma.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_DMA_H__ 2 | #define __NVM_INTERNAL_DMA_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | /* Forward declaration */ 9 | struct dma_map; 10 | 11 | 12 | /* Callback type for unmapping and deleting custom mapping */ 13 | typedef void (*dma_map_free_t)(struct dma_map* map); 14 | 15 | 16 | 17 | /* 18 | * DMA mapping descriptor. 19 | * 20 | * This structure describes a custom address range mapping. 21 | * Custom types should include this structure in head. 22 | */ 23 | struct __attribute__((aligned (64))) dma_map 24 | { 25 | void* vaddr; // Virtual address of mapped address range 26 | size_t page_size; // Page size of address range 27 | size_t n_pages; // Number of pages for address range 28 | }; 29 | 30 | 31 | 32 | /* 33 | * Create a DMA handle container. 34 | */ 35 | int _nvm_dma_create(nvm_dma_t** handle, 36 | const nvm_ctrl_t* ctrl, 37 | struct dma_map* map, 38 | dma_map_free_t release); 39 | 40 | 41 | /* 42 | * Initialize DMA handle. 43 | */ 44 | void _nvm_dma_handle_populate(nvm_dma_t* handle, 45 | const nvm_ctrl_t* ctrl, 46 | const uint64_t* ioaddrs); 47 | 48 | 49 | 50 | /* 51 | * Invoke release callback and remove a DMA handle container. 52 | */ 53 | void _nvm_dma_remove(nvm_dma_t* handle); 54 | 55 | 56 | 57 | #endif /* __NVM_INTERNAL_DMA_H__ */ 58 | -------------------------------------------------------------------------------- /src/dprintf.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_DPRINTF_H__ 2 | #define __NVM_INTERNAL_DPRINTF_H__ 3 | 4 | #ifndef NDEBUG 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | /* Debug printf */ 12 | static void _nvm_dprintf(const char* func, const char* format, ...) 13 | { 14 | va_list args; 15 | va_start(args, format); 16 | fprintf(stderr, "[%s] ", func); 17 | vfprintf(stderr, format, args); 18 | va_end(args); 19 | } 20 | 21 | #define dprintf(...) _nvm_dprintf(__func__, __VA_ARGS__) 22 | 23 | #define _nvm_strerror(status) nvm_strerror(status) 24 | #define _SCIGetErrorString(err) SCIGetErrorString(err) 25 | 26 | #endif /* ! NDEBUG */ 27 | 28 | 29 | 30 | /* If no debug print, just swallow message */ 31 | #ifndef dprintf 32 | #define dprintf(...) 33 | #endif 34 | 35 | 36 | 37 | /* If no debug print, don't lookup completions */ 38 | #ifndef _nvm_strerror 39 | #define _nvm_strerror(status) 40 | #define _SCIGetErrorString(err) 41 | #endif 42 | 43 | 44 | #endif /* __NVM_INTERNAL_DPRINTF_H__ */ 45 | -------------------------------------------------------------------------------- /src/error.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | static const char* generic_status[] = 10 | { 11 | "Success", 12 | "Invalid command opcode", 13 | "Invalid field in command", 14 | "Command ID conflict", 15 | "Data transfer error", 16 | "Commands aborted due to power loss notification", 17 | "Internal error", 18 | "Command abort requested", 19 | "Command aborted due to SQ deletion", 20 | "Command aborted due to failed fused command", 21 | "Command aborted due to missing fused command", 22 | "Invalid namespace or format", 23 | "Command sequence error", 24 | "Invalid SGL segment descriptor", 25 | "Invalid number of SQL descriptors", 26 | "Data SGL length invalid", 27 | "Metadata SGL length invalid", 28 | "SGL descriptor type invalid", 29 | "Invalid use of controller memory buffer", 30 | "PRP offset invalid", 31 | "Atomic write unit exceeded", 32 | "Operation denied", 33 | "SGL offset invalid", 34 | "Unknown/reserved", 35 | "Host identifier inconsistent format", 36 | "Keep alive timer expired", 37 | "Keep alive timer invalid", 38 | "Command aborted due to preempt and abort", 39 | "Sanitize failed", 40 | "Sanitize in progress", 41 | "SGL data block granularity invalid", 42 | "Command not supported for queue in CMB" 43 | }; 44 | 45 | 46 | 47 | static const char* generic_status_nvm_commands[] = 48 | { 49 | "LBA out of range", 50 | "Capacity exceeded", 51 | "Namespace not ready", 52 | "Reservation conflict", 53 | "Format in progress" 54 | }; 55 | 56 | 57 | 58 | static const char* command_specific_status[] = 59 | { 60 | "Completion queue invalid", 61 | "Invalid queue identifier", 62 | "Invalid queue size", 63 | "Abort command limit exceeded", 64 | "Unknown/reserved", 65 | "Asynchronous event request limit exceeded", 66 | "Invalid firmware slot", 67 | "Invalid firmware image", 68 | "Invalid interrupt vector", 69 | "Invalid log page", 70 | "Invalid format", 71 | "Firmware activation requires conventional reset", 72 | "Invalid queue deletion", 73 | "Feature identifier not saveable", 74 | "Feature not changeable", 75 | "Feature not namespace specific", 76 | "Firmware activation requires NVM subsystem reset", 77 | "Firmware activation requires reset", 78 | "Firmware activation requires maximum time violation", 79 | "Firmware activation prohibited", 80 | "Overlapping range", 81 | "Namespace insufficient capacity", 82 | "Namespace identifier unavailable", 83 | "Unknown/reserved", 84 | "Namespace already attached", 85 | "Namespace is private", 86 | "Namespace not attached", 87 | "Thin provisioning not supported", 88 | "Controller list invalid", 89 | "Device self-test in progress", 90 | "Boot partition write prohibited", 91 | "Invalid controller identifier", 92 | "Invalid secondary controller state", 93 | "Invalid number of controller resources", 94 | "Invalid resource identifier" 95 | }; 96 | 97 | 98 | 99 | static const char* command_specific_status_nvm_commands[] = 100 | { 101 | "Conflicting attributes", 102 | "Invalid protection information", 103 | "Attempted write to read only range" 104 | }; 105 | 106 | 107 | 108 | static const char* media_and_data_integrity_nvm_commands[] = 109 | { 110 | "Write fault", 111 | "Unrecovered read error", 112 | "End-to-end guard check error", 113 | "End-to-end application tag check error", 114 | "End-to-end reference tag check error", 115 | "Compare failure", 116 | "Access denied", 117 | "Deallocated or unwritten logical block" 118 | }; 119 | 120 | 121 | 122 | static const char* lookup_string(uint8_t status_code_type, uint8_t status_code) 123 | { 124 | switch (status_code_type) 125 | { 126 | case 0x00: // Generic command status 127 | if (status_code < 0x20) 128 | { 129 | return generic_status[status_code]; 130 | } 131 | else if (0x80 <= status_code && status_code <= 0x84) 132 | { 133 | return generic_status_nvm_commands[status_code - 0x80]; 134 | } 135 | return "Unknown generic error"; 136 | 137 | case 0x01: // Command specific status 138 | if (status_code < 0x23) 139 | { 140 | return command_specific_status[status_code]; 141 | } 142 | else if (0x80 <= status_code && status_code <= 0x82) 143 | { 144 | return command_specific_status_nvm_commands[status_code - 0x80]; 145 | } 146 | return "Unknown command specific error"; 147 | 148 | case 0x02: // Media and data integrity errors 149 | if (0x80 <= status_code && status_code <= 0x87) 150 | { 151 | return media_and_data_integrity_nvm_commands[status_code - 0x80]; 152 | } 153 | return "Unknown media or data integrity error"; 154 | 155 | default: 156 | return "Unknown status code type"; 157 | } 158 | } 159 | 160 | 161 | 162 | const char* nvm_strerror(int status) 163 | { 164 | int err; 165 | uint8_t sct; 166 | uint8_t sc; 167 | 168 | err = NVM_ERR_UNPACK_ERRNO(status); 169 | sct = NVM_ERR_UNPACK_SCT(status); 170 | sc = NVM_ERR_UNPACK_SC(status); 171 | 172 | if (sct != 0 || sc != 0) 173 | { 174 | return lookup_string(sct, sc); 175 | } 176 | 177 | return strerror(err); 178 | } 179 | 180 | -------------------------------------------------------------------------------- /src/ioctl.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_IOCTL_H__ 2 | #define __NVM_INTERNAL_IOCTL_H__ 3 | 4 | #include 5 | #include 6 | 7 | #define NVM_IOCTL_TYPE 0x80 8 | 9 | 10 | 11 | /* Memory map request */ 12 | struct nvm_ioctl_map 13 | { 14 | uint64_t vaddr_start; 15 | size_t n_pages; 16 | uint64_t* ioaddrs; 17 | }; 18 | 19 | 20 | 21 | /* Supported operations */ 22 | enum nvm_ioctl_type 23 | { 24 | NVM_MAP_HOST_MEMORY = _IOW(NVM_IOCTL_TYPE, 1, struct nvm_ioctl_map), 25 | #ifdef _CUDA 26 | NVM_MAP_DEVICE_MEMORY = _IOW(NVM_IOCTL_TYPE, 2, struct nvm_ioctl_map), 27 | #endif 28 | NVM_UNMAP_MEMORY = _IOW(NVM_IOCTL_TYPE, 3, uint64_t) 29 | }; 30 | 31 | 32 | 33 | #endif /* __NVM_INTERNAL_IOCTL_H__ */ 34 | -------------------------------------------------------------------------------- /src/queue.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "regs.h" 8 | #include "util.h" 9 | 10 | 11 | 12 | void nvm_queue_clear(nvm_queue_t* queue, const nvm_ctrl_t* ctrl, bool cq, uint16_t no, void* vaddr, uint64_t ioaddr) 13 | { 14 | queue->no = no; 15 | queue->max_entries = 0; 16 | queue->entry_size = cq ? sizeof(nvm_cpl_t) : sizeof(nvm_cmd_t); 17 | queue->head = 0; 18 | queue->tail = 0; 19 | queue->phase = 1; 20 | queue->last = 0; 21 | queue->vaddr = vaddr; 22 | queue->ioaddr = ioaddr; 23 | queue->db = cq ? CQ_DBL(ctrl->mm_ptr, queue->no, ctrl->dstrd) : SQ_DBL(ctrl->mm_ptr, queue->no, ctrl->dstrd); 24 | queue->max_entries = _MIN(ctrl->max_entries, ctrl->page_size / queue->entry_size); 25 | } 26 | 27 | 28 | 29 | nvm_cpl_t* nvm_cq_dequeue_block(nvm_queue_t* cq, uint64_t timeout) 30 | { 31 | uint64_t nsecs = timeout * 1000000UL; 32 | nvm_cpl_t* cpl = nvm_cq_dequeue(cq); 33 | 34 | while (cpl == NULL && nsecs > 0) 35 | { 36 | nsecs = _nvm_delay_remain(nsecs); 37 | cpl = nvm_cq_dequeue(cq); 38 | } 39 | 40 | return cpl; 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/regs.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_REGS_H__ 2 | #define __NVM_INTERNAL_REGS_H__ 3 | 4 | #include 5 | #include 6 | 7 | 8 | /* Controller registers */ 9 | #define CAP(p) _REG(p, 0x0000, 64) // Controller Capabilities 10 | #define VER(p) _REG(p, 0x0008, 32) // NVM Express version 11 | #define CC(p) _REG(p, 0x0014, 32) // Controller Configuration 12 | #define CSTS(p) _REG(p, 0x001c, 32) // Controller Status 13 | #define AQA(p) _REG(p, 0x0024, 32) // Admin Queue Attributes 14 | #define ASQ(p) _REG(p, 0x0028, 64) // Admin Submission Queue Base Address 15 | #define ACQ(p) _REG(p, 0x0030, 64) // Admin Completion Queue Base Address 16 | 17 | 18 | /* Read bit fields */ 19 | #define CAP$MPSMAX(p) _RB(*CAP(p), 55, 52) // Memory Page Size Maximum 20 | #define CAP$MPSMIN(p) _RB(*CAP(p), 51, 48) // Memory Page Size Minimum 21 | #define CAP$DSTRD(p) _RB(*CAP(p), 35, 32) // Doorbell Stride 22 | #define CAP$TO(p) _RB(*CAP(p), 31, 24) // Timeout 23 | #define CAP$CQR(p) _RB(*CAP(p), 16, 16) // Contiguous Queues Required 24 | #define CAP$MQES(p) _RB(*CAP(p), 15, 0) // Maximum Queue Entries Supported 25 | 26 | #define CSTS$RDY(p) _RB(*CSTS(p), 0, 0) // Ready indicator 27 | 28 | 29 | /* Write bit fields */ 30 | #define CC$IOCQES(v) _WB(v, 23, 20) // IO Completion Queue Entry Size 31 | #define CC$IOSQES(v) _WB(v, 19, 16) // IO Submission Queue Entry Size 32 | #define CC$MPS(v) _WB(v, 10, 7) // Memory Page Size 33 | #define CC$CSS(v) _WB(0, 3, 1) // IO Command Set Selected (0=NVM Command Set) 34 | #define CC$EN(v) _WB(v, 0, 0) // Enable 35 | 36 | #define AQA$AQS(v) _WB(v, 27, 16) // Admin Completion Queue Size 37 | #define AQA$AQC(v) _WB(v, 11, 0) // Admin Submission Queue Size 38 | 39 | 40 | /* SQ doorbell register offset */ 41 | #define SQ_DBL(p, y, dstrd) \ 42 | ((volatile uint32_t*) (((volatile unsigned char*) (p)) + 0x1000 + ((2*(y)) * (4 << (dstrd)))) ) 43 | 44 | 45 | /* CQ doorbell register offset */ 46 | #define CQ_DBL(p, y, dstrd) \ 47 | ((volatile uint32_t*) (((volatile unsigned char*) (p)) + 0x1000 + ((2*(y) + 1) * (4 << (dstrd)))) ) 48 | 49 | #endif /* __NVM_INTERNAL_REGS_H__ */ 50 | -------------------------------------------------------------------------------- /src/rpc.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_RPC_H__ 2 | #define __NVM_INTERNAL_RPC_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | /* Forward declaration */ 12 | struct nvm_admin_reference; 13 | 14 | 15 | 16 | /* 17 | * Callback to delete custom instance data. 18 | */ 19 | typedef void (*rpc_deleter_t)(void* data, uint32_t key, int remaining_handles); 20 | 21 | 22 | 23 | /* 24 | * RPC client-side stub definition. 25 | * 26 | * Should perform the following actions. 27 | * - marshal command 28 | * - send command to remote host 29 | * - wait for completion (or timeout) 30 | * - unmarshal completion and return status 31 | */ 32 | typedef int (*rpc_stub_t)(void*, nvm_cmd_t*, nvm_cpl_t*); 33 | 34 | 35 | 36 | /* 37 | * Allocate a reference wrapper and increase controller reference. 38 | */ 39 | int _nvm_ref_get(nvm_aq_ref* handle, const nvm_ctrl_t* ctrl); 40 | 41 | 42 | 43 | /* 44 | * Free reference wrapper and decrease controller reference. 45 | */ 46 | void _nvm_ref_put(nvm_aq_ref ref); 47 | 48 | 49 | 50 | /* 51 | * Insert binding handle to server's list of handles. 52 | * If key is already found, this function will fail. 53 | */ 54 | int _nvm_rpc_handle_insert(nvm_aq_ref ref, uint32_t key, void* data, rpc_deleter_t release); 55 | 56 | 57 | 58 | /* 59 | * Remove local binding handle. 60 | * This function will call the release callback. 61 | */ 62 | void _nvm_rpc_handle_remove(nvm_aq_ref ref, uint32_t key); 63 | 64 | 65 | 66 | /* 67 | * Bind reference to remote handle. 68 | */ 69 | int _nvm_rpc_bind(nvm_aq_ref ref, void* data, rpc_deleter_t deleter, rpc_stub_t stub); 70 | 71 | 72 | 73 | /* 74 | * Execute a local admin command. 75 | */ 76 | int _nvm_local_admin(nvm_aq_ref ref, const nvm_cmd_t* cmd, nvm_cpl_t* cpl); 77 | 78 | 79 | 80 | #endif /* __NVM_INTERNAL_RPC_H__ */ 81 | -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- 1 | #ifndef __NVM_INTERNAL_UTIL_H__ 2 | #define __NVM_INTERNAL_UTIL_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | //#include 10 | 11 | #ifndef NDEBUG 12 | #include 13 | #include 14 | #include "dprintf.h" 15 | #endif 16 | 17 | 18 | /* Get minimum of two values */ 19 | #define _MIN(a, b) ( (a) <= (b) ? (a) : (b) ) 20 | 21 | 22 | /* Get the maximum of two values */ 23 | #define _MAX(a, b) ( (a) > (b) ? (a) : (b) ) 24 | 25 | 26 | 27 | /* Calculate the base-2 logarithm of a number n */ 28 | static inline uint32_t _nvm_b2log(uint32_t n) 29 | { 30 | uint32_t count = 0; 31 | 32 | while (n > 0) 33 | { 34 | ++count; 35 | n >>= 1; 36 | } 37 | 38 | return count - 1; 39 | } 40 | 41 | 42 | /* Delay the minimum of one millisecond and a time remainder */ 43 | static inline uint64_t _nvm_delay_remain(uint64_t remaining_nanoseconds) 44 | { 45 | struct timespec ts; 46 | 47 | if (remaining_nanoseconds == 0) 48 | { 49 | return 0; 50 | } 51 | 52 | ts.tv_sec = 0; 53 | ts.tv_nsec = _MIN(1000000UL, remaining_nanoseconds); 54 | 55 | //pthread_yield(); 56 | clock_nanosleep(CLOCK_REALTIME, 0, &ts, NULL); 57 | 58 | remaining_nanoseconds -= _MIN(1000000UL, remaining_nanoseconds); 59 | return remaining_nanoseconds; 60 | } 61 | 62 | 63 | /* Get the system page size */ 64 | static inline size_t _nvm_host_page_size() 65 | { 66 | long page_size = sysconf(_SC_PAGESIZE); 67 | 68 | #ifndef NDEBUG 69 | if (page_size < 0) 70 | { 71 | dprintf("Failed to look up system page size: %s\n", strerror(errno)); 72 | return 0; 73 | } 74 | #endif 75 | 76 | return page_size; 77 | } 78 | 79 | 80 | #endif /* __NVM_INTERNAL_UTIL_H__ */ 81 | --------------------------------------------------------------------------------