├── .gitignore ├── COPYING ├── Documentation ├── ntc.txt └── ntrdma.txt ├── GNUmakefile ├── MAINTAINERS ├── Makefile ├── README.md ├── build.gradle ├── build └── build.sh ├── drivers ├── Makefile ├── infiniband │ ├── Makefile │ └── hw │ │ ├── Makefile │ │ └── ntrdma │ │ ├── Makefile │ │ ├── ntc.h │ │ ├── ntrdma-trace.h │ │ ├── ntrdma.h │ │ ├── ntrdma_cm.c │ │ ├── ntrdma_cm.h │ │ ├── ntrdma_cmd.c │ │ ├── ntrdma_cmd.h │ │ ├── ntrdma_cq.c │ │ ├── ntrdma_cq.h │ │ ├── ntrdma_debugfs.c │ │ ├── ntrdma_dev.c │ │ ├── ntrdma_dev.h │ │ ├── ntrdma_eth.c │ │ ├── ntrdma_eth.h │ │ ├── ntrdma_file.c │ │ ├── ntrdma_file.h │ │ ├── ntrdma_hello.c │ │ ├── ntrdma_hello.h │ │ ├── ntrdma_ib.c │ │ ├── ntrdma_ib.h │ │ ├── ntrdma_ib_4_19.h │ │ ├── ntrdma_ib_5_3.h │ │ ├── ntrdma_ioctl.h │ │ ├── ntrdma_main.c │ │ ├── ntrdma_mr.c │ │ ├── ntrdma_mr.h │ │ ├── ntrdma_obj.h │ │ ├── ntrdma_pd.c │ │ ├── ntrdma_pd.h │ │ ├── ntrdma_qp.c │ │ ├── ntrdma_qp.h │ │ ├── ntrdma_res.c │ │ ├── ntrdma_res.h │ │ ├── ntrdma_ring.h │ │ ├── ntrdma_sg.h │ │ ├── ntrdma_util.c │ │ ├── ntrdma_util.h │ │ ├── ntrdma_vbell.c │ │ ├── ntrdma_vbell.h │ │ ├── ntrdma_wr.h │ │ ├── ntrdma_zip.c │ │ └── ntrdma_zip.h └── ntc │ ├── Makefile │ ├── ntc-trace.h │ ├── ntc.c │ ├── ntc.h │ ├── ntc_internal.h │ └── ntc_ntb_msi.c ├── include └── linux │ ├── ntc.h │ ├── ntc_mm.h │ ├── ntc_pfn.h │ └── ntc_trace.h ├── makefile ├── patches ├── 0001-iommu-vt-d-Implement-dma_-un-map_resource.patch └── ntb_no_msix │ └── 0001-ntb_hw_intel-add-no_msix-parameter.patch └── scripts └── getstats.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.o 3 | *.ko 4 | *.mod.c 5 | config.h 6 | Module.symvers 7 | modules.order 8 | *.o.parts 9 | !.gitignore 10 | devops-scripts/ 11 | output/ 12 | -------------------------------------------------------------------------------- /Documentation/ntc.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ntrdma/ntrdma-ext/6eeba7c9e028841d48ce32270136e31fb0fb84f2/Documentation/ntc.txt -------------------------------------------------------------------------------- /Documentation/ntrdma.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ntrdma/ntrdma-ext/6eeba7c9e028841d48ce32270136e31fb0fb84f2/Documentation/ntrdma.txt -------------------------------------------------------------------------------- /GNUmakefile: -------------------------------------------------------------------------------- 1 | ../../utils/GNUmakefile -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | NTC DRIVER FOR NTB 2 | M: Leonid Ravich 3 | M: Eyal BenNer 4 | M: Alexander Barabash 5 | M: Yuval Berger 6 | S: Maintained 7 | F: include/linux/ntc.h 8 | F: include/linux/ntc_mm.h 9 | F: include/linux/ntc_internal.h 10 | F: include/linux/ntc_trace.h 11 | F: drivers/ntc/ntc.c 12 | F: drivers/ntc/ntc_ntb_msi.c 13 | F: drivers/ntc/ntc_phys.c 14 | F: drivers/ntc/Makefile 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS_MODULE := -Wall -Werror -DNTRDMA_GIT_HASH=\"$(shell git --git-dir=$M/.git rev-list -1 --abbrev-commit HEAD)\" 2 | obj-y += drivers/ 3 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | // Default to building the entire repo 2 | defaultTasks gradle.build.buildTaskName 3 | -------------------------------------------------------------------------------- /build/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | TOP="$(git rev-parse --show-toplevel)" 4 | DEVOPS_TOP="$TOP"/devops-scripts 5 | 6 | . "$TOP"/devops-scripts/bashlib/init.libraries 7 | 8 | devops.parseBuildArgs "$0" "$@" 2> /dev/null 9 | devops.getBuildInputs 10 | OUTPUT_DIR=$(devops.outputDir) 11 | OUTPUT_FILE=$OUTPUT_DIR/ntrdma_modules/ntrdma_modules.tar.gz 12 | TMP_DIR=$TOP/output 13 | 14 | echo "INFO: builder file is ${build_inputs[sles15sp2-sles_builder]}" 15 | BUILD_DOCKER=$(<"${build_inputs[sles15sp2-sles_builder]}") 16 | KERNEL_HEADERS_PATH=${build_inputs[coreos-linux-headers-gcached]} 17 | KERNEL_VERSION_PATH="${build_inputs[coreos-version-text]:-${build_inputs[coreos-version]}}" 18 | 19 | echo "INFO: kernel headers are $KERNEL_HEADERS_PATH version is $KERNEL_VERSION_PATH" 20 | 21 | mkdir -p $TMP_DIR 22 | 23 | if [[ $(/bin/diff -qN $TMP_DIR/version.txt $KERNEL_VERSION_PATH) ]]; then 24 | echo Extracting kernel headers 25 | rm -fr $TMP_DIR/* 26 | tar -I pbzip2 -C $TMP_DIR -xf $KERNEL_HEADERS_PATH 27 | cp $KERNEL_VERSION_PATH $TMP_DIR 28 | fi 29 | 30 | # Use devtoolset 7 (Updated toolchain) with gcc 7.3 instead of default 4.8 31 | CMD="set +o nounset; source /opt/rh/devtoolset-7/enable; set -o nounset; 32 | gcc --version; 33 | make -j CONFIG_NTC=m CONFIG_NTRDMA=m DEBUG=1 -C ${TMP_DIR}/lib/modules/*/build M=${TOP} modules" 34 | 35 | echo "Kernel header ready firing make from docker..." 36 | 37 | devops.run_in_builder_docker "$CMD" "$TOP" "$BUILD_DOCKER" 38 | 39 | echo "NTRDMA compilation completed, starting packaging..." 40 | mkdir -p $OUTPUT_DIR/ntrdma_modules/ 41 | 42 | tar -czvf $OUTPUT_FILE --xform s:^.*/:: drivers/ntc/ntc_ntb.ko drivers/infiniband/hw/ntrdma/ntrdma.ko 43 | 44 | -------------------------------------------------------------------------------- /drivers/Makefile: -------------------------------------------------------------------------------- 1 | obj-y += ntc/ infiniband/ 2 | -------------------------------------------------------------------------------- /drivers/infiniband/Makefile: -------------------------------------------------------------------------------- 1 | obj-y += hw/ 2 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/Makefile: -------------------------------------------------------------------------------- 1 | obj-y += ntrdma/ 2 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_NTRDMA) += ntrdma.o 2 | CFLAGS_ntrdma_zip.o = -I$(src) 3 | 4 | ntrdma-y += ntrdma_cmd.o 5 | ntrdma-y += ntrdma_cq.o 6 | ntrdma-y += ntrdma_debugfs.o 7 | ntrdma-y += ntrdma_dev.o 8 | ntrdma-y += ntrdma_eth.o 9 | ntrdma-y += ntrdma_file.o 10 | ntrdma-y += ntrdma_hello.o 11 | ntrdma-y += ntrdma_ib.o 12 | ntrdma-y += ntrdma_main.o 13 | ntrdma-y += ntrdma_mr.o 14 | ntrdma-y += ntrdma_pd.o 15 | ntrdma-y += ntrdma_qp.o 16 | ntrdma-y += ntrdma_res.o 17 | ntrdma-y += ntrdma_util.o 18 | ntrdma-y += ntrdma_vbell.o 19 | ntrdma-y += ntrdma_zip.o 20 | ntrdma-y += ntrdma_cm.o 21 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntc.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #pragma once 3 | #include "../../../../include/linux/ntc.h" 4 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma-trace.h: -------------------------------------------------------------------------------- 1 | #undef TRACE_SYSTEM 2 | #define TRACE_SYSTEM ntrdma 3 | 4 | #if !defined(_NTRDMA_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) 5 | #define _NTRDMA_TRACE_H 6 | 7 | #include 8 | 9 | DECLARE_EVENT_CLASS(imm_template, 10 | TP_PROTO(u64 wrid, u64 data, u64 dva, u64 dpa, u64 len, u32 rc), 11 | TP_ARGS(wrid, data, dva, dpa, len, rc), 12 | TP_STRUCT__entry( 13 | __field(u64, wrid) 14 | __field(u64, data) 15 | __field(u64, dva) 16 | __field(u64, dpa) 17 | __field(u64, len) 18 | __field(u32, rc) 19 | ), 20 | TP_fast_assign( 21 | __entry->wrid = wrid; 22 | __entry->data = data; 23 | __entry->dva = dva; 24 | __entry->dpa = dpa; 25 | __entry->len = len; 26 | __entry->rc = rc; 27 | ), 28 | TP_printk( 29 | "wrid 0x%llx snd data 0x%llx, dst phy 0x%llx dst vir 0x%llx, len 0x%llx, rc %d", 30 | __entry->wrid, __entry->data, __entry->dpa, 31 | __entry->dva, __entry->len, __entry->rc) 32 | ); 33 | 34 | /*trace_imm_dma_cpy*/ 35 | DEFINE_EVENT(imm_template, imm_dma_cpy, 36 | TP_PROTO(u64 wrid, u64 data, u64 dva, u64 dpa, u64 len, u32 rc), 37 | TP_ARGS(wrid, data, dva, dpa, len, rc)); 38 | 39 | DECLARE_EVENT_CLASS(dma_template, 40 | TP_PROTO(u64 wrid, u64 sva, u64 spa, u64 dva, 41 | u64 dpa, u64 len, u32 rc), 42 | TP_ARGS(wrid, sva, spa, dva, dpa, len, rc), 43 | TP_STRUCT__entry( 44 | __field(u64, wrid) 45 | __field(u64, sva) 46 | __field(u64, spa) 47 | __field(u64, dva) 48 | __field(u64, dpa) 49 | __field(u64, len) 50 | __field(u32, rc) 51 | ), 52 | TP_fast_assign( 53 | __entry->wrid = wrid; 54 | __entry->sva = sva; 55 | __entry->spa = spa; 56 | __entry->dva = dva; 57 | __entry->dpa = dpa; 58 | __entry->len = len; 59 | __entry->rc = rc; 60 | ), 61 | TP_printk( 62 | "wrid 0x%llx src phy 0x%llx src vir 0x%llx, dst phy 0x%llx dst vir 0x%llx, len 0x%llx, rc %d", 63 | __entry->wrid, __entry->spa, __entry->sva, __entry->dpa, 64 | __entry->dva, __entry->len, __entry->rc) 65 | ); 66 | 67 | /*trace_dma_cpy*/ 68 | DEFINE_EVENT(dma_template, dma_cpy, 69 | TP_PROTO(u64 wrid, u64 sva, u64 spa, u64 dva, 70 | u64 dpa, u64 len, u32 rc), 71 | TP_ARGS(wrid, sva, spa, dva, dpa, len, rc)); 72 | 73 | DECLARE_EVENT_CLASS(poll_ioctl_template, 74 | TP_PROTO(u64 wrid, u32 wc_opcode, u32 cq_opcode, s32 qp, 75 | s32 status, u32 pos, u32 end, u32 flags), 76 | TP_ARGS(wrid, wc_opcode, cq_opcode, qp, status, pos, end, flags), 77 | TP_STRUCT__entry( 78 | __field(u64, wrid) 79 | __field(u32, wc_opcode) 80 | __field(u32, cq_opcode) 81 | __field(u32, qp) 82 | __field(u32, status) 83 | __field(u32, pos) 84 | __field(u32, end) 85 | __field(u32, flags) 86 | ), 87 | TP_fast_assign( 88 | __entry->wrid = wrid; 89 | __entry->wc_opcode = wc_opcode; 90 | __entry->cq_opcode = cq_opcode; 91 | __entry->qp = qp; 92 | __entry->status = status; 93 | __entry->pos = pos; 94 | __entry->end = end; 95 | __entry->flags = flags; 96 | ), 97 | TP_printk( 98 | "OPCODE %d(%d): wrid %llu QP %d status %d pos %u end %u flags %d\n", 99 | __entry->wc_opcode, 100 | __entry->cq_opcode, 101 | __entry->wrid, 102 | __entry->qp, 103 | __entry->status, 104 | __entry->pos, 105 | __entry->end, 106 | __entry->flags) 107 | ); 108 | 109 | /*trace_poll_ioctl*/ 110 | DEFINE_EVENT(poll_ioctl_template, poll_ioctl, 111 | TP_PROTO(u64 wrid, u32 wc_opcode, u32 cq_opcode, s32 qp, 112 | s32 status, u32 pos, u32 end, u32 flags), 113 | TP_ARGS(wrid, wc_opcode, cq_opcode, qp, status, pos, end, flags)); 114 | /*trace_poll_cq*/ 115 | DEFINE_EVENT(poll_ioctl_template, poll_cq, 116 | TP_PROTO(u64 wrid, u32 wc_opcode, u32 cq_opcode, s32 qp, 117 | s32 status, u32 pos, u32 end, u32 flags), 118 | TP_ARGS(wrid, wc_opcode, cq_opcode, qp, status, pos, end, flags)); 119 | 120 | DECLARE_EVENT_CLASS(rqp_work_template, 121 | TP_PROTO(u32 opcode, u32 flags, u64 addr, s32 qp, 122 | s32 num_segs, u64 wrid, s32 status), 123 | TP_ARGS(opcode, flags, addr, qp, num_segs, wrid, status), 124 | TP_STRUCT__entry( 125 | __field(u32, opcode) 126 | __field(u32, flags) 127 | __field(u64, addr) 128 | __field(s32, qp) 129 | __field(u32, num_segs) 130 | __field(u64, wrid) 131 | __field(s32, status) 132 | ), 133 | TP_fast_assign( 134 | __entry->opcode = opcode; 135 | __entry->flags = flags; 136 | __entry->addr = addr; 137 | __entry->qp = qp; 138 | __entry->num_segs = num_segs; 139 | __entry->wrid = wrid; 140 | __entry->status = status; 141 | ), 142 | TP_printk( 143 | "OPCODE %d: flags %x, addr %llx QP %d num sges %d wrid %llu status %d", 144 | __entry->opcode, __entry->flags, __entry->addr, 145 | __entry->qp, __entry->num_segs, __entry->wrid, 146 | __entry->status) 147 | ); 148 | 149 | /*trace_rqp_work*/ 150 | DEFINE_EVENT(rqp_work_template, rqp_work, 151 | TP_PROTO(u32 opcode, u32 flags, u64 addr, s32 qp, 152 | s32 num_segs, u64 wrid, s32 status), 153 | TP_ARGS(opcode, flags, addr, qp, num_segs, wrid, status)); 154 | 155 | #endif /* _NTRDMA_TRACE_H */ 156 | 157 | /* This part must be outside protection */ 158 | #undef TRACE_INCLUDE_PATH 159 | #define TRACE_INCLUDE_PATH . 160 | #define TRACE_INCLUDE_FILE ntrdma-trace 161 | #include 162 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_H 34 | #define NTRDMA_H 35 | 36 | #include 37 | #include "ntc.h" 38 | #include "ntrdma_file.h" 39 | 40 | /* #define NTRDMA_QP_DEBUG */ 41 | 42 | #define NTRDMA_PERF_MEASURE(name) do { \ 43 | if (ntrdma_measure_perf) NTC_PERF_MEASURE(name); \ 44 | } while (false) 45 | 46 | #define NTRDMA_RESERVED_DMA_LEKY 0xFFFF 47 | 48 | #define to_ptrhld(ptr) ((unsigned long)(void *)(ptr)) 49 | #define of_ptrhld(ptr) ((void *)(ptr)) 50 | 51 | struct dentry; 52 | 53 | struct ntrdma_dev; 54 | struct ntrdma_eth; 55 | 56 | struct ntrdma_obj; 57 | struct ntrdma_res; 58 | struct ntrdma_rres; 59 | 60 | struct ntrdma_cq; 61 | struct ntrdma_mr; 62 | struct ntrdma_pd; 63 | struct ntrdma_qp; 64 | 65 | struct ntrdma_rmr; 66 | struct ntrdma_rpd; 67 | struct ntrdma_rqp; 68 | 69 | struct ntrdma_cqe; 70 | struct ntrdma_poll; 71 | struct ntrdma_recv_wqe; 72 | struct ntrdma_send_wqe; 73 | 74 | struct ntrdma_vbell; 75 | struct ntrdma_vbell_head; 76 | 77 | union ntrdma_cmd; 78 | union ntrdma_rsp; 79 | 80 | int __init ntrdma_vbell_module_init(void); 81 | void ntrdma_vbell_module_deinit(void); 82 | 83 | int __init ntrdma_qp_module_init(void); 84 | void ntrdma_qp_module_deinit(void); 85 | 86 | int __init ntrdma_ib_module_init(void); 87 | void ntrdma_ib_module_deinit(void); 88 | #ifdef NTRDMA_FULL_ETH 89 | int __init ntrdma_eth_module_init(void); 90 | void ntrdma_eth_module_deinit(void); 91 | void ntrdma_dev_eth_reset(struct ntrdma_dev *dev); 92 | #else 93 | static inline 94 | int __init ntrdma_eth_module_init(void) 95 | { 96 | return 0; 97 | } 98 | static inline 99 | void ntrdma_eth_module_deinit(void) {} 100 | static inline 101 | void ntrdma_dev_eth_reset(struct ntrdma_dev *dev) {} 102 | #endif 103 | 104 | int __init ntrdma_debugfs_init(void); 105 | void ntrdma_debugfs_deinit(void); 106 | 107 | void ntrdma_debugfs_dev_add(struct ntrdma_dev *dev); 108 | void ntrdma_debugfs_dev_del(struct ntrdma_dev *dev); 109 | void ntrdma_debugfs_cq_add(struct ntrdma_cq *cq); 110 | void ntrdma_debugfs_cq_del(struct ntrdma_cq *cq); 111 | void ntrdma_debugfs_pd_add(struct ntrdma_pd *pd); 112 | void ntrdma_debugfs_pd_del(struct ntrdma_pd *pd); 113 | void ntrdma_debugfs_mr_add(struct ntrdma_mr *mr); 114 | void ntrdma_debugfs_mr_del(struct ntrdma_mr *mr); 115 | void ntrdma_debugfs_qp_add(struct ntrdma_qp *qp); 116 | void ntrdma_debugfs_qp_del(struct ntrdma_qp *qp); 117 | void ntrdma_debugfs_rpd_add(struct ntrdma_rpd *rpd); 118 | void ntrdma_debugfs_rpd_del(struct ntrdma_rpd *rpd); 119 | void ntrdma_debugfs_rmr_add(struct ntrdma_rmr *rmr); 120 | void ntrdma_debugfs_rmr_del(struct ntrdma_rmr *rmr); 121 | void ntrdma_debugfs_rqp_add(struct ntrdma_rqp *rqp); 122 | void ntrdma_debugfs_rqp_del(struct ntrdma_rqp *rqp); 123 | 124 | int ntrdma_dev_init(struct ntrdma_dev *dev, 125 | struct ntc_dev *ntc); 126 | void ntrdma_dev_deinit(struct ntrdma_dev *dev); 127 | 128 | int ntrdma_dev_hello_init(struct ntrdma_dev *dev, struct ntc_dev *ntc); 129 | void ntrdma_dev_hello_deinit(struct ntrdma_dev *dev); 130 | 131 | int ntrdma_dev_ib_init(struct ntrdma_dev *dev); 132 | void ntrdma_dev_ib_deinit(struct ntrdma_dev *dev); 133 | 134 | int ntrdma_dev_eth_init(struct ntrdma_dev *dev, 135 | u32 vbell_idx, u32 rx_cap); 136 | void ntrdma_dev_eth_deinit(struct ntrdma_dev *dev); 137 | 138 | int ntrdma_dev_hello(struct ntrdma_dev *dev, int phase); 139 | 140 | int ntrdma_dev_res_init(struct ntrdma_dev *dev); 141 | void ntrdma_dev_res_deinit(struct ntrdma_dev *dev); 142 | 143 | int ntrdma_dev_enable(struct ntrdma_dev *dev); 144 | void ntrdma_dev_eth_enable(struct ntrdma_dev *dev); 145 | void ntrdma_dev_cmd_enable(struct ntrdma_dev *dev); 146 | int ntrdma_dev_res_enable(struct ntrdma_dev *dev); 147 | 148 | void ntrdma_dev_disable(struct ntrdma_dev *dev); 149 | void ntrdma_dev_eth_disable(struct ntrdma_dev *dev); 150 | void ntrdma_dev_cmd_disable(struct ntrdma_dev *dev); 151 | void ntrdma_dev_res_disable(struct ntrdma_dev *dev); 152 | 153 | void ntrdma_dev_quiesce(struct ntrdma_dev *dev); 154 | void ntrdma_dev_reset(struct ntrdma_dev *dev); 155 | void ntrdma_dev_eth_quiesce(struct ntrdma_dev *dev); 156 | void ntrdma_dev_cmd_reset(struct ntrdma_dev *dev); 157 | void ntrdma_dev_cmd_quiesce(struct ntrdma_dev *dev); 158 | void ntrdma_dev_res_reset(struct ntrdma_dev *dev); 159 | void ntrdma_dev_rres_reset(struct ntrdma_dev *dev); 160 | void ntrdma_dev_vbell_reset(struct ntrdma_dev *dev); 161 | void _ntrdma_unrecoverable_err(struct ntrdma_dev *dev, const char *f); 162 | #define ntrdma_unrecoverable_err(_dev) _ntrdma_unrecoverable_err(_dev, __func__) 163 | #endif 164 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_cm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ntrdma_cm.h 3 | * 4 | * Created on: Jan 1, 2020 5 | * Author: leonidr 6 | */ 7 | 8 | #ifndef DRIVERS_INFINIBAND_HW_NTRDMA_NTRDMA_CM_H_ 9 | #define DRIVERS_INFINIBAND_HW_NTRDMA_NTRDMA_CM_H_ 10 | 11 | #include 12 | #include "ntrdma_cmd.h" 13 | #include "ntrdma_dev.h" 14 | 15 | enum ntrdma_cm_state { 16 | NTRDMA_CM_STATE_IDLE = 0, 17 | NTRDMA_CM_STATE_ESTABLISHED, 18 | NTRDMA_CM_STATE_CONNECTING, 19 | NTRDMA_CM_STATE_KILLING, 20 | }; 21 | 22 | #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0) 23 | #define iwcm_2_ntrdma_iwcm(__iwcm) \ 24 | container_of(__iwcm, struct ntrdma_iw_cm, iwcm) 25 | 26 | #define ntrdma_iw_cm_from_ntrdma_dev(__dev) \ 27 | iwcm_2_ntrdma_iwcm((__dev)->ibdev.iwcm) 28 | 29 | struct ntrdma_iw_cm { 30 | struct iw_cm_verbs iwcm; 31 | struct ntrdma_dev *dev; 32 | }; 33 | struct iw_cm_verbs *ntrdma_cm_init(const char *name, struct ntrdma_dev *dev); 34 | void ntrdma_cm_deinit(struct iw_cm_verbs *iwcm); 35 | #else 36 | void ntrdma_cm_init(struct ib_device *ibdev); 37 | #endif 38 | 39 | struct ntrdma_iw_cm_id_node { 40 | struct list_head head; 41 | struct iw_cm_id *cm_id; 42 | int qpn; 43 | }; 44 | 45 | int ntrdma_cmd_recv_cm(struct ntrdma_dev *dev, 46 | const union ntrdma_cmd *cmd, 47 | union ntrdma_rsp *rsp); 48 | 49 | void ntrdma_cm_qp_shutdown(struct ntrdma_qp *qp); 50 | void ntrdma_cm_kill(struct ntrdma_qp *qp); 51 | int _ntrdma_modify_qp_local(struct ib_qp *ibqp, 52 | struct ib_qp_attr *ibqp_attr, 53 | int ibqp_mask, 54 | const char *func); 55 | 56 | #define ntrdma_modify_qp_local(_ibqp, _ibqp_attr, _ibqp_mask) \ 57 | _ntrdma_modify_qp_local(_ibqp, _ibqp_attr, _ibqp_mask, __func__) 58 | 59 | 60 | 61 | #endif /* DRIVERS_INFINIBAND_HW_NTRDMA_NTRDMA_CM_H_ */ 62 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_cmd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_CMD_H 34 | #define NTRDMA_CMD_H 35 | 36 | #include "ntc.h" 37 | #include 38 | 39 | #include "ntrdma_sg.h" 40 | 41 | struct ntrdma_dev; 42 | 43 | /* Command and response size in bytes */ 44 | 45 | #define NTRDMA_CMD_SIZE 0x100 46 | #define NTRDMA_RSP_SIZE 0x40 47 | 48 | /* Command op codes */ 49 | 50 | #define NTRDMA_CMD_NONE 0 51 | 52 | #define NTRDMA_CMD_MR 0x08 53 | #define NTRDMA_CMD_MR_CREATE (NTRDMA_CMD_MR + 0) 54 | #define NTRDMA_CMD_MR_DELETE (NTRDMA_CMD_MR + 1) 55 | #define NTRDMA_CMD_MR_APPEND (NTRDMA_CMD_MR + 2) 56 | 57 | #define NTRDMA_CMD_QP 0x0c 58 | #define NTRDMA_CMD_QP_CREATE (NTRDMA_CMD_QP + 0) 59 | #define NTRDMA_CMD_QP_DELETE (NTRDMA_CMD_QP + 1) 60 | #define NTRDMA_CMD_QP_MODIFY (NTRDMA_CMD_QP + 2) 61 | #define NTRDMA_CMD_IW_CM (NTRDMA_CMD_QP + 3) 62 | 63 | #define CMD_TIMEOUT_MSEC 5000 /*5 sec*/ 64 | 65 | #define MAX_SUM_ACCESS_FLAGS (1<<7) // enum ibv_access_flags (rdma-core) 66 | #define IB_MR_LIMIT_BYTES (1ULL << 35) /* 32GB */ 67 | #define INTEL_ALIGN 16 68 | 69 | struct ntrdma_rsp_hdr { 70 | u32 op; 71 | u32 status; 72 | u32 cmd_id; 73 | }; 74 | 75 | struct ntrdma_cmd_hdr { 76 | u32 op; 77 | u32 cmd_id; 78 | u64 cb_p; 79 | }; 80 | 81 | /* Create memory region command */ 82 | struct ntrdma_cmd_mr_create { 83 | struct ntrdma_cmd_hdr hdr; 84 | u32 mr_key; 85 | u32 pd_key; 86 | u32 access; 87 | u64 mr_addr; 88 | u64 mr_len; 89 | u32 sg_cap; 90 | u32 sg_count; 91 | struct ntc_remote_buf_desc sg_desc_list[]; 92 | }; 93 | 94 | /* Delete memory region command */ 95 | struct ntrdma_cmd_mr_delete { 96 | struct ntrdma_cmd_hdr hdr; 97 | u32 mr_key; 98 | }; 99 | 100 | /* Append to memory region command */ 101 | struct ntrdma_cmd_mr_append { 102 | struct ntrdma_cmd_hdr hdr; 103 | u32 mr_key; 104 | u32 sg_pos; 105 | u32 sg_count; 106 | struct ntc_remote_buf_desc sg_desc_list[]; 107 | }; 108 | 109 | /* Create, delete or append memory region response */ 110 | struct ntrdma_rsp_mr_status { 111 | struct ntrdma_rsp_hdr hdr; 112 | u32 mr_key; 113 | }; 114 | 115 | /* Create queue pair command */ 116 | struct ntrdma_cmd_qp_create { 117 | struct ntrdma_cmd_hdr hdr; 118 | u32 qp_key; 119 | u32 pd_key; 120 | u32 qp_type; 121 | u32 recv_wqe_cap; 122 | u32 recv_wqe_sg_cap; 123 | u32 recv_ring_idx; 124 | u32 send_wqe_cap; 125 | u32 send_wqe_sg_cap; 126 | u32 send_wqe_inline_cap; 127 | u32 send_ring_idx; 128 | struct ntc_remote_buf_desc send_cqe_buf_desc; 129 | u64 send_cons_shift; 130 | u32 cmpl_vbell_idx; 131 | }; 132 | 133 | /* Delete a queue pair command */ 134 | struct ntrdma_cmd_qp_delete { 135 | struct ntrdma_cmd_hdr hdr; 136 | u32 qp_key; 137 | }; 138 | 139 | /* Modify queue pair command */ 140 | struct ntrdma_cmd_qp_modify { 141 | struct ntrdma_cmd_hdr hdr; 142 | u32 src_qp_key; 143 | u32 access; 144 | u32 state; 145 | u32 dest_qp_key; 146 | }; 147 | 148 | /* Create queue pair response */ 149 | struct ntrdma_rsp_qp_create { 150 | struct ntrdma_rsp_hdr hdr; 151 | u32 qp_key; 152 | u32 send_vbell_idx; 153 | struct ntc_remote_buf_desc recv_wqe_buf_desc; 154 | u64 recv_prod_shift; 155 | struct ntc_remote_buf_desc send_wqe_buf_desc; 156 | u64 send_prod_shift; 157 | }; 158 | 159 | /* Delete or modify queue pair response */ 160 | struct ntrdma_rsp_qp_status { 161 | struct ntrdma_rsp_hdr hdr; 162 | u32 qp_key; 163 | }; 164 | 165 | /* Command union */ 166 | union ntrdma_cmd { 167 | struct ntrdma_cmd_hdr hdr; 168 | struct ntrdma_cmd_mr_create mr_create; 169 | struct ntrdma_cmd_mr_delete mr_delete; 170 | struct ntrdma_cmd_mr_append mr_append; 171 | struct ntrdma_cmd_qp_create qp_create; 172 | struct ntrdma_cmd_qp_delete qp_delete; 173 | struct ntrdma_cmd_qp_modify qp_modify; 174 | u8 buf[NTRDMA_CMD_SIZE]; 175 | }; 176 | 177 | /* Response union */ 178 | union ntrdma_rsp { 179 | struct ntrdma_rsp_hdr hdr; 180 | struct ntrdma_rsp_mr_status mr_create; 181 | struct ntrdma_rsp_mr_status mr_delete; 182 | struct ntrdma_rsp_mr_status mr_append; 183 | struct ntrdma_rsp_qp_create qp_create; 184 | struct ntrdma_rsp_qp_status qp_delete; 185 | struct ntrdma_rsp_qp_status qp_modify; 186 | u8 buf[NTRDMA_RSP_SIZE]; 187 | }; 188 | 189 | #define NTRDMA_CMD_MR_CREATE_SG_CAP \ 190 | ((sizeof(union ntrdma_cmd) - sizeof(struct ntrdma_cmd_mr_create)) \ 191 | / sizeof(struct ntc_remote_buf_desc)) 192 | 193 | #define NTRDMA_CMD_MR_APPEND_SG_CAP \ 194 | ((sizeof(union ntrdma_cmd) - sizeof(struct ntrdma_cmd_mr_append)) \ 195 | / sizeof(struct ntc_remote_buf_desc)) 196 | 197 | struct ntrdma_cmd_cb { 198 | /* entry in the device cmd pending or posted list */ 199 | struct list_head dev_entry; /* Protected by dev->cmd_send.lock */ 200 | 201 | /* prepare a command in-place in the ring buffer */ 202 | int (*cmd_prep)(struct ntrdma_cmd_cb *cb, 203 | union ntrdma_cmd *cmd); 204 | 205 | /* complete and free the command following a response */ 206 | void (*rsp_cmpl)(struct ntrdma_cmd_cb *cb, 207 | const union ntrdma_rsp *rsp); 208 | 209 | struct completion cmds_done; 210 | 211 | bool in_list; /* Protected by dev->cmd_send.lock */ 212 | int cmd_id; /* Protected by dev->cmd_send.lock */ 213 | int ret; 214 | }; 215 | 216 | int ntrdma_dev_cmd_init(struct ntrdma_dev *dev, 217 | u32 recv_vbell_idx, 218 | u32 send_vbell_idx, 219 | u32 send_cap); 220 | 221 | void ntrdma_dev_cmd_deinit(struct ntrdma_dev *dev); 222 | 223 | int ntrdma_dev_cmd_add(struct ntrdma_dev *dev, struct ntrdma_cmd_cb *cb); 224 | void ntrdma_dev_cmd_add_unsafe(struct ntrdma_dev *dev, 225 | struct ntrdma_cmd_cb *cb); 226 | inline int ntrdma_dev_cmd_submit(struct ntrdma_dev *dev); 227 | 228 | inline bool ntrdma_cmd_cb_unlink(struct ntrdma_dev *dev, 229 | struct ntrdma_cmd_cb *cb); 230 | 231 | int _ntrmda_rqp_modify_local(struct ntrdma_dev *dev, 232 | u32 src_qp_key, u32 access, 233 | u32 new_state, u32 dest_qp_key, 234 | const char *caller); 235 | 236 | #define ntrmda_rqp_modify_local(_dev, _src_qp_key, _access, _new_state, _dest_qp_key) \ 237 | _ntrmda_rqp_modify_local(_dev, _src_qp_key, _access, _new_state, _dest_qp_key, __func__) 238 | 239 | #endif 240 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_cq.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include "ntrdma_dev.h" 34 | #include "ntrdma_cq.h" 35 | #include "ntrdma_qp.h" 36 | 37 | static void ntrdma_cq_cue_work(unsigned long ptrhld); 38 | 39 | DECLARE_PER_CPU(struct ntrdma_dev_counters, dev_cnt); 40 | 41 | void ntrdma_cq_init(struct ntrdma_cq *cq, struct ntrdma_dev *dev) 42 | { 43 | ntrdma_obj_init(&cq->obj, dev); 44 | 45 | INIT_LIST_HEAD(&cq->poll_list); 46 | mutex_init(&cq->poll_lock); 47 | } 48 | 49 | void ntrdma_cq_vbell_init(struct ntrdma_cq *cq, int vbell_idx) 50 | { 51 | struct ntrdma_dev *dev = ntrdma_cq_dev(cq); 52 | 53 | cq->arm = 0; 54 | cq->need_cue = false; 55 | spin_lock_init(&cq->arm_lock); 56 | 57 | tasklet_init(&cq->cue_work, 58 | ntrdma_cq_cue_work, 59 | to_ptrhld(cq)); 60 | ntrdma_tasklet_vbell_init(dev, &cq->vbell, vbell_idx, &cq->cue_work); 61 | } 62 | 63 | void ntrdma_cq_arm_resync(struct ntrdma_dev *dev) 64 | { 65 | struct ntrdma_cq *cq; 66 | 67 | mutex_lock(&dev->res.lock); 68 | list_for_each_entry(cq, &dev->res.cq_list, obj.dev_entry) { 69 | spin_lock_bh(&cq->arm_lock); 70 | if (cq->arm) { 71 | ntrdma_vbell_add_clear(&cq->vbell); 72 | ntrdma_info(dev, "re arm cq %p vbell %d\n", cq, 73 | cq->vbell.idx); 74 | TRACE("re arm cq %p, vbell %d\n", cq, cq->vbell.idx); 75 | } 76 | spin_unlock_bh(&cq->arm_lock); 77 | } 78 | mutex_unlock(&dev->res.lock); 79 | } 80 | 81 | void ntrdma_cq_vbell_kill(struct ntrdma_cq *cq) 82 | { 83 | spin_lock_bh(&cq->arm_lock); 84 | { 85 | cq->arm = 0; 86 | cq->need_cue = false; 87 | } 88 | spin_unlock_bh(&cq->arm_lock); 89 | 90 | ntrdma_tasklet_vbell_kill(&cq->vbell); 91 | } 92 | 93 | void ntrdma_cq_arm(struct ntrdma_cq *cq) 94 | { 95 | bool need_cue; 96 | unsigned int arm; 97 | 98 | this_cpu_inc(dev_cnt.cqes_armed); 99 | 100 | spin_lock_bh(&cq->arm_lock); 101 | 102 | ntrdma_vbell_readd(&cq->vbell); 103 | ++cq->arm; 104 | arm = cq->arm; 105 | need_cue = cq->need_cue; 106 | cq->need_cue = false; 107 | 108 | spin_unlock_bh(&cq->arm_lock); 109 | 110 | TRACE("cq %p arm %d need cue %d vbell idx %d\n", 111 | cq, arm, need_cue, cq->vbell.idx); 112 | 113 | if (need_cue) 114 | ntrdma_cq_cue(cq); 115 | } 116 | 117 | void _ntrdma_cq_cue(struct ntrdma_cq *cq, const char *f) 118 | { 119 | unsigned int arm, initial_arm; 120 | 121 | spin_lock_bh(&cq->arm_lock); 122 | 123 | if (unlikely(!cq->ibcq_valid)) { 124 | spin_unlock_bh(&cq->arm_lock); 125 | return; 126 | } 127 | 128 | arm = cq->arm; 129 | 130 | initial_arm = arm = cq->arm; 131 | cq->arm = 0; 132 | 133 | if (!arm) 134 | cq->need_cue = true; 135 | 136 | for (; arm; --arm) { 137 | /*TODO: do we realy need to run this in spinlock???*/ 138 | cq->ibcq->ibcq.comp_handler(&cq->ibcq->ibcq, cq->ibcq->ibcq.cq_context); 139 | TRACE("cq %p arm %d %s\n", cq, initial_arm, f); 140 | } 141 | spin_unlock_bh(&cq->arm_lock); 142 | 143 | this_cpu_add(dev_cnt.cqes_notified, initial_arm); 144 | } 145 | 146 | void ntrdma_cq_add_poll(struct ntrdma_cq *cq, struct ntrdma_poll *poll) 147 | { 148 | mutex_lock(&cq->poll_lock); 149 | { 150 | list_add_tail(&poll->cq_entry, &cq->poll_list); 151 | } 152 | mutex_unlock(&cq->poll_lock); 153 | } 154 | 155 | void ntrdma_cq_del_poll(struct ntrdma_cq *cq, struct ntrdma_poll *poll) 156 | { 157 | mutex_lock(&cq->poll_lock); 158 | list_del(&poll->cq_entry); 159 | mutex_unlock(&cq->poll_lock); 160 | } 161 | 162 | int ntrdma_cq_cmpl_get(struct ntrdma_cq *cq, struct ntrdma_qp **qp, 163 | u32 *pos, u32 *end, u32 *base) 164 | { 165 | struct ntrdma_poll *poll; 166 | int rc = -EAGAIN; 167 | 168 | list_for_each_entry(poll, &cq->poll_list, cq_entry) { 169 | rc = poll->poll_start_and_get(poll, qp, pos, end, base); 170 | if (rc == -EAGAIN && qp && *qp) { 171 | if (rc == -EAGAIN) 172 | continue; 173 | } 174 | 175 | /* move the head to after the entry (rotate the list) */ 176 | list_move(&cq->poll_list, &poll->cq_entry); 177 | break; 178 | } 179 | 180 | return rc; 181 | } 182 | 183 | void ntrdma_cq_cmpl_put(struct ntrdma_cq *cq, 184 | u32 pos, u32 base) 185 | { 186 | struct ntrdma_poll *poll; 187 | 188 | poll = list_last_entry(&cq->poll_list, struct ntrdma_poll, cq_entry); 189 | 190 | poll->poll_put_and_done(poll, pos, base); 191 | } 192 | 193 | void ntrdma_cq_cmpl_cqe(struct ntrdma_cq *cq, 194 | struct ntrdma_cqe *outcqe, u32 pos) 195 | { 196 | struct ntrdma_poll *poll; 197 | 198 | poll = list_last_entry(&cq->poll_list, struct ntrdma_poll, cq_entry); 199 | 200 | return poll->poll_cqe(poll, outcqe, pos); 201 | } 202 | 203 | static void ntrdma_cq_cue_work(unsigned long ptrhld) 204 | { 205 | struct ntrdma_cq *cq = of_ptrhld(ptrhld); 206 | 207 | ntrdma_cq_cue(cq); 208 | } 209 | 210 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_cq.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_CQ_H 34 | #define NTRDMA_CQ_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #include 43 | 44 | #include "ntrdma_obj.h" 45 | #include "ntrdma_vbell.h" 46 | 47 | struct ntrdma_ib_cq; 48 | /* Completion Queue */ 49 | struct ntrdma_cq { 50 | /* Ofed cq structure */ 51 | struct ntrdma_ib_cq *ibcq; 52 | bool ibcq_valid; /* Protected by arm_lock. */ 53 | 54 | /* debugfs */ 55 | struct dentry *debug; 56 | 57 | /* Ntrdma cq is a local-only object */ 58 | struct ntrdma_obj obj; /* member of dev->cq_list */ 59 | 60 | /* number of application requests to be notified */ 61 | unsigned int arm; 62 | 63 | /* synchronize arming and cuing the completion queue */ 64 | spinlock_t arm_lock; 65 | 66 | /* poll entries for polling queue pairs */ 67 | struct list_head poll_list; 68 | 69 | /* synchronize poll_list modification and polling */ 70 | struct mutex poll_lock; 71 | 72 | /* work completion notification */ 73 | struct tasklet_struct cue_work; 74 | struct ntrdma_vbell vbell; 75 | bool need_cue; 76 | 77 | struct page *poll_page; 78 | }; 79 | 80 | struct ntrdma_ib_cq { 81 | struct ib_cq ibcq; 82 | struct ntrdma_cq *cq; 83 | }; 84 | 85 | #define ntrdma_ib_cq(__ibcq) \ 86 | container_of(__ibcq, struct ntrdma_ib_cq, ibcq)->cq 87 | 88 | #define ntrdma_cq_dev(__cq) \ 89 | ntrdma_obj_dev(&(__cq)->obj) 90 | 91 | #define ntrdma_cq_dbg(__cq, __args...) \ 92 | ntrdma_dbg(ntrdma_cq_dev(__cq), ## __args) 93 | 94 | #define ntrdma_cq_err(__cq, __fmt, __args...) \ 95 | ntrdma_err(ntrdma_cq_dev(__cq), __fmt, ## __args) 96 | 97 | #define ntrdma_cq_info(__cq, __fmt, __args...) \ 98 | ntrdma_info(ntrdma_cq_dev(__cq), __fmt, ## __args) 99 | 100 | #define ntrdma_cq_info_ratelimited(__cq, __fmt, __args...) \ 101 | ntrdma_info_ratelimited(ntrdma_cq_dev(__cq), __fmt, ## __args) 102 | 103 | void ntrdma_cq_init(struct ntrdma_cq *cq, struct ntrdma_dev *dev); 104 | void ntrdma_cq_vbell_init(struct ntrdma_cq *cq, int vbell_idx); 105 | void ntrdma_cq_vbell_kill(struct ntrdma_cq *cq); 106 | 107 | void ntrdma_cq_get(struct ntrdma_cq *cq); 108 | void ntrdma_cq_put(struct ntrdma_cq *cq); 109 | 110 | void ntrdma_cq_arm_resync(struct ntrdma_dev *dev); 111 | void ntrdma_cq_arm(struct ntrdma_cq *cq); 112 | #define ntrdma_cq_cue(__cq) _ntrdma_cq_cue(__cq, __func__) 113 | void _ntrdma_cq_cue(struct ntrdma_cq *cq, const char *f); 114 | 115 | void ntrdma_cq_add_poll(struct ntrdma_cq *cq, struct ntrdma_poll *poll); 116 | void ntrdma_cq_del_poll(struct ntrdma_cq *cq, struct ntrdma_poll *poll); 117 | 118 | int ntrdma_cq_cmpl_get(struct ntrdma_cq *cq, struct ntrdma_qp **qp, 119 | u32 *pos, u32 *end, u32 *base); 120 | void ntrdma_cq_cmpl_put(struct ntrdma_cq *cq, 121 | u32 pos, u32 base); 122 | void ntrdma_cq_cmpl_cqe(struct ntrdma_cq *cq, 123 | struct ntrdma_cqe *outcqe, u32 pos); 124 | 125 | #endif 126 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_dev.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include 34 | #include "ntrdma_cmd.h" 35 | #include "ntrdma_vbell.h" 36 | 37 | #include "ntrdma.h" 38 | #include "ntrdma_dev.h" 39 | #include "ntrdma_qp.h" 40 | #include "ntrdma_cm.h" 41 | 42 | #define NTRDMA_DEV_VBELL_START 0x8 43 | 44 | #define NTRDMA_DEV_CMD_RECV_VBELL_IDX 0 45 | #define NTRDMA_DEV_CMD_SEND_VBELL_IDX 1 46 | #define NTRDMA_DEV_CMD_SEND_CAP 0x10 47 | 48 | #define NTRDMA_DEV_ETH_VBELL_IDX 2 49 | #define NTRDMA_DEV_ETH_RX_CAP 0x100 50 | 51 | 52 | #define ntrdma_of_qp_close_work(__ws) \ 53 | container_of(__ws, struct ntrdma_dev, qp_close_work) 54 | 55 | 56 | DEFINE_PER_CPU(struct ntrdma_dev_counters, dev_cnt); 57 | EXPORT_PER_CPU_SYMBOL(dev_cnt); 58 | static void ntrdma_ntc_link_reset_cb(struct work_struct *ws); 59 | 60 | static int ntrdma_ntc_hello(void *ctx, int phase, 61 | void *in_buf, size_t in_size, 62 | void *out_buf, size_t out_size) 63 | { 64 | struct ntrdma_dev *dev = ctx; 65 | 66 | return ntrdma_dev_hello(dev, phase); 67 | } 68 | 69 | static int ntrdma_ntc_enable(void *ctx) 70 | { 71 | struct ntrdma_dev *dev = ctx; 72 | 73 | return ntrdma_dev_enable(dev); 74 | } 75 | 76 | static void ntrdma_ntc_disable(void *ctx) 77 | { 78 | struct ntrdma_dev *dev = ctx; 79 | 80 | ntrdma_dev_disable(dev); 81 | } 82 | 83 | static void ntrdma_ntc_quiesce(void *ctx) 84 | { 85 | struct ntrdma_dev *dev = ctx; 86 | 87 | ntrdma_dev_quiesce(dev); 88 | } 89 | 90 | static void ntrdma_ntc_reset(void *ctx) 91 | { 92 | struct ntrdma_dev *dev = ctx; 93 | 94 | ntrdma_dev_reset(dev); 95 | } 96 | 97 | static void ntrdma_ntc_signal(void *ctx, int vec) 98 | { 99 | struct ntrdma_dev *dev = ctx; 100 | 101 | tasklet_schedule(&dev->vbell.work[vec]); 102 | } 103 | 104 | static struct ntc_ctx_ops ntrdma_ntc_ctx_ops = { 105 | .hello = ntrdma_ntc_hello, 106 | .enable = ntrdma_ntc_enable, 107 | .disable = ntrdma_ntc_disable, 108 | .quiesce = ntrdma_ntc_quiesce, 109 | .reset = ntrdma_ntc_reset, 110 | .signal = ntrdma_ntc_signal, 111 | }; 112 | 113 | int ntrdma_dev_init(struct ntrdma_dev *dev, struct ntc_dev *ntc) 114 | { 115 | int rc; 116 | int i = 0; 117 | int num_cpus; 118 | 119 | dev->ntc = ntc; 120 | 121 | /* Must be before device is registered (in ntrdma_dev_ib_init) */ 122 | mutex_init(&dev->debugfs_lock); 123 | 124 | rc = ntrdma_dev_vbell_init(dev, 125 | NTRDMA_DEV_VBELL_COUNT, 126 | NTRDMA_DEV_VBELL_START); 127 | if (rc) 128 | goto err_vbell; 129 | 130 | rc = ntrdma_dev_cmd_init(dev, 131 | NTRDMA_DEV_CMD_RECV_VBELL_IDX, 132 | NTRDMA_DEV_CMD_SEND_VBELL_IDX, 133 | NTRDMA_DEV_CMD_SEND_CAP); 134 | if (rc) 135 | goto err_cmd; 136 | 137 | rc = ntrdma_dev_eth_init(dev, 138 | NTRDMA_DEV_ETH_VBELL_IDX, 139 | NTRDMA_DEV_ETH_RX_CAP); 140 | if (rc) 141 | goto err_eth; 142 | 143 | rc = ntrdma_dev_res_init(dev); 144 | if (rc) 145 | goto err_res; 146 | 147 | rc = ntrdma_dev_ib_init(dev); 148 | if (rc) 149 | goto err_ib; 150 | 151 | rc = ntc_set_ctx(ntc, dev, &ntrdma_ntc_ctx_ops); 152 | if (rc) 153 | goto err_ntc; 154 | 155 | rc = ntrdma_dev_hello_init(dev, ntc); 156 | if (rc) 157 | goto err_hello; 158 | 159 | /* counters */ 160 | num_cpus = num_online_cpus(); 161 | 162 | for (i = 0; i < num_cpus; i++) 163 | memset(per_cpu_ptr(&dev_cnt, i), 0, 164 | sizeof(struct ntrdma_dev_counters)); 165 | 166 | ntrdma_debugfs_dev_add(dev); 167 | 168 | atomic_set(&dev->qp_num, 0); 169 | atomic_set(&dev->cq_num, 0); 170 | atomic_set(&dev->mr_num, 0); 171 | atomic_set(&dev->pd_num, 0); 172 | INIT_WORK(&dev->ntc_link_reset_work, ntrdma_ntc_link_reset_cb); 173 | 174 | return 0; 175 | 176 | err_hello: 177 | ntrdma_dev_hello_deinit(dev); 178 | err_ntc: 179 | ntrdma_dev_ib_deinit(dev); 180 | err_ib: 181 | ntrdma_dev_eth_deinit(dev); 182 | err_eth: 183 | ntrdma_dev_res_deinit(dev); 184 | err_res: 185 | ntrdma_dev_cmd_deinit(dev); 186 | err_cmd: 187 | ntrdma_dev_vbell_deinit(dev); 188 | err_vbell: 189 | return rc; 190 | } 191 | 192 | int ntrdma_dev_hello_init(struct ntrdma_dev *dev, struct ntc_dev *ntc) 193 | { 194 | dev->hello.local_buf = ntc_local_hello_buf(ntc, &dev->hello.local_buf_size); 195 | dev->hello.peer_buf = ntc_peer_hello_buf(ntc, &dev->hello.peer_buf_size); 196 | 197 | ntrdma_dbg(dev, "local %p size %d peer %p size %d\n", 198 | dev->hello.local_buf, dev->hello.local_buf_size, 199 | dev->hello.peer_buf, dev->hello.peer_buf_size); 200 | 201 | return !(dev->hello.local_buf && dev->hello.peer_buf && 202 | dev->hello.local_buf_size > 0 && dev->hello.peer_buf_size > 0); 203 | } 204 | 205 | void ntrdma_dev_hello_deinit(struct ntrdma_dev *dev) 206 | { 207 | dev->hello.local_buf = NULL; 208 | dev->hello.peer_buf = NULL; 209 | dev->hello.local_buf_size = 0; 210 | dev->hello.peer_buf_size = 0; 211 | } 212 | 213 | void ntrdma_dev_deinit(struct ntrdma_dev *dev) 214 | { 215 | ntrdma_info(dev, "Starting dev deinit...\n"); 216 | ntrdma_debugfs_dev_del(dev); 217 | ntrdma_dev_eth_deinit(dev); 218 | ntrdma_dev_res_deinit(dev); 219 | ntrdma_dev_cmd_deinit(dev); 220 | ntrdma_dev_vbell_deinit(dev); 221 | cancel_work_sync(&dev->ntc_link_reset_work); 222 | ntrdma_info(dev, "Dev deinit finished\n"); 223 | } 224 | 225 | int ntrdma_dev_enable(struct ntrdma_dev *dev) 226 | { 227 | int rc; 228 | 229 | ntrdma_dev_eth_enable(dev); 230 | ntrdma_dev_cmd_enable(dev); 231 | rc = ntrdma_dev_res_enable(dev); 232 | 233 | return rc; 234 | } 235 | 236 | void ntrdma_dev_disable(struct ntrdma_dev *dev) 237 | { 238 | ntrdma_dev_cmd_disable(dev); 239 | ntrdma_dev_eth_disable(dev); 240 | } 241 | 242 | void ntrdma_dev_quiesce(struct ntrdma_dev *dev) 243 | { 244 | ntrdma_dev_vbell_disable(dev); 245 | ntrdma_dev_eth_quiesce(dev); 246 | /* resource disable should block from new commands to be submitted */ 247 | ntrdma_dev_res_disable(dev); 248 | /* cmd quiesce should block till all in progress commands completed */ 249 | ntrdma_dev_cmd_quiesce(dev); 250 | } 251 | 252 | void ntrdma_dev_reset(struct ntrdma_dev *dev) 253 | { 254 | ntrdma_dev_eth_reset(dev); 255 | ntrdma_dev_rres_reset(dev); 256 | ntrdma_dev_cmd_reset(dev); 257 | ntrdma_dev_vbell_reset(dev); 258 | } 259 | 260 | static void ntrdma_ntc_link_reset_cb(struct work_struct *ws) 261 | { 262 | struct ntrdma_dev *dev; 263 | 264 | dev = ntrdma_ntc_link_reset_work_dev(ws); 265 | ntc_link_reset(dev->ntc, ASYNC_RESET); 266 | } 267 | 268 | void _ntrdma_unrecoverable_err(struct ntrdma_dev *dev, 269 | const char *f) 270 | { 271 | ntrdma_err(dev, "unrecoverable error called by %s ", f); 272 | ntrdma_err(dev, "disabling NTB link...\n"); 273 | ntc_link_disable(dev->ntc); 274 | ntrdma_err(dev, "NTB link disabled, resetting NTC state machine\n"); 275 | schedule_work(&dev->ntc_link_reset_work); 276 | ntrdma_err(dev, "NTC state machine reset finished\n"); 277 | } 278 | 279 | 280 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_eth.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_ETH_H 34 | #define NTRDMA_ETH_H 35 | 36 | #include 37 | #include 38 | 39 | #include "ntrdma.h" 40 | #include "ntrdma_ring.h" 41 | 42 | struct ntrdma_eth { 43 | struct napi_struct napi; 44 | struct ntrdma_dev *dev; 45 | 46 | struct ntc_dma_chan *dma_chan; 47 | 48 | bool enable; 49 | bool ready; /* FIXME: dev enable common */ 50 | bool link; 51 | 52 | /* ethernet rx ring indices */ 53 | 54 | u32 rx_cap; 55 | u32 rx_post; 56 | u32 rx_prod; 57 | u32 rx_cmpl; 58 | 59 | /* ethernet rx ring buffers */ 60 | 61 | struct ntc_export_buf *rx_buf; 62 | struct ntc_local_buf rx_wqe_buf; 63 | struct ntc_export_buf rx_cqe_buf; 64 | 65 | struct ntc_export_buf rx_cons_buf; 66 | 67 | struct ntc_remote_buf peer_tx_wqe_buf; 68 | struct ntc_remote_buf peer_tx_prod_buf; 69 | 70 | /* ethernet tx ring indices */ 71 | 72 | u32 tx_cap; 73 | u32 tx_cons; 74 | u32 tx_cmpl; 75 | 76 | /* ethernet tx ring buffers */ 77 | 78 | struct ntc_export_buf tx_wqe_buf; 79 | struct ntc_local_buf tx_cqe_buf; 80 | 81 | struct ntc_export_buf tx_prod_buf; 82 | 83 | struct ntc_remote_buf peer_rx_cqe_buf; 84 | struct ntc_remote_buf peer_rx_cons_buf; 85 | u32 peer_vbell_idx; 86 | 87 | /* one at a time each: poster, producer, consumer, completer */ 88 | 89 | spinlock_t rx_prod_lock; 90 | spinlock_t rx_cmpl_lock; 91 | spinlock_t tx_cons_lock; 92 | 93 | /* notify napi of rx tx ring availability */ 94 | 95 | struct ntrdma_vbell vbell; 96 | bool is_hello_done; 97 | bool is_hello_prep; 98 | }; 99 | 100 | inline u32 ntrdma_eth_tx_prod(struct ntrdma_eth *eth); 101 | inline u32 ntrdma_eth_rx_cons(struct ntrdma_eth *eth); 102 | 103 | #define ntrdma_napi_eth(__napi) \ 104 | container_of(__napi, struct ntrdma_eth, napi) 105 | #define ntrdma_net_eth(__net) \ 106 | netdev_priv(__net) 107 | 108 | static inline struct net_device *ntrdma_get_net(struct ntrdma_dev *dev) 109 | { 110 | struct ntrdma_eth *eth = dev->eth; 111 | 112 | return eth->napi.dev; 113 | } 114 | #endif 115 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_file.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019, Dell Technologies, Inc. 3 | * 4 | * This program is free software; you can redistribute it and/or modify it 5 | * under the terms and conditions of the GNU General Public License, 6 | * version 2, as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope it will be useful, but WITHOUT 9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 | * more details. 12 | */ 13 | 14 | #include "ntrdma_file.h" 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #define NTRDMA_MINORS (1U << MINORBITS) 23 | 24 | #define NTRDMA_FILE_IOCTL_BASE 'F' 25 | 26 | #define NTRDMA_FILEIOCTL_REG _IOWR(NTRDMA_FILE_IOCTL_BASE, 0x30, u32) 27 | #define NTRDMA_FILEIOCTL_SEND _IOWR(NTRDMA_FILE_IOCTL_BASE, 0x31, u32) 28 | 29 | static int ntrdma_file_open(struct inode *inode, struct file *filp); 30 | static int ntrdma_file_release(struct inode *inode, struct file *filp); 31 | static long ntrdma_file_ioctl(struct file *filp, unsigned int cmd, 32 | unsigned long arg); 33 | 34 | static bool ntrdma_chrdev_registered; 35 | static int ntrdma_char_major; 36 | module_param(ntrdma_char_major, int, 0); 37 | static struct class *ntrdma_class; 38 | struct device *ntrdma_dev; 39 | 40 | static const struct file_operations ntrdma_file_ops = { 41 | .owner = THIS_MODULE, 42 | .open = ntrdma_file_open, 43 | .release = ntrdma_file_release, 44 | .unlocked_ioctl = ntrdma_file_ioctl, 45 | .compat_ioctl = ntrdma_file_ioctl, 46 | }; 47 | 48 | bool ntrdma_measure_perf; 49 | static ssize_t show_measure_perf(struct device *dev, 50 | struct device_attribute *attr, 51 | char *buf); 52 | static ssize_t write_measure_perf(struct device *dev, 53 | struct device_attribute *attr, 54 | const char *buf, size_t count); 55 | static DEVICE_ATTR(measure_perf, S_IRUGO | S_IWUSR, 56 | show_measure_perf, write_measure_perf); 57 | 58 | bool ntrdma_print_debug; 59 | static ssize_t show_print_debug(struct device *dev, 60 | struct device_attribute *attr, 61 | char *buf); 62 | static ssize_t write_print_debug(struct device *dev, 63 | struct device_attribute *attr, 64 | const char *buf, size_t count); 65 | static DEVICE_ATTR(print_debug, S_IRUGO | S_IWUSR, 66 | show_print_debug, write_print_debug); 67 | 68 | static struct attribute *ntrdma_file_attrs[] = { 69 | &dev_attr_measure_perf.attr, 70 | &dev_attr_print_debug.attr, 71 | NULL, 72 | }; 73 | 74 | static struct attribute_group ntrdma_file_attrs_group = { 75 | .attrs = ntrdma_file_attrs, 76 | }; 77 | 78 | static const struct attribute_group *ntrdma_file_attrs_groups[] = { 79 | &ntrdma_file_attrs_group, 80 | NULL, 81 | }; 82 | 83 | struct ntrdma_common_data { 84 | struct page *common_page; 85 | void volatile *common_ptr; 86 | }; 87 | 88 | static int ntrdma_file_open(struct inode *inode, struct file *filp) 89 | { 90 | struct ntrdma_common_data *common_data; 91 | 92 | common_data = kmalloc(sizeof(*common_data), GFP_KERNEL); 93 | if (!common_data) { 94 | pr_err("%s failed to alloc common data\n", __func__); 95 | return -ENOMEM; 96 | } 97 | 98 | common_data->common_page = NULL; 99 | common_data->common_ptr = NULL; 100 | 101 | filp->private_data = common_data; 102 | 103 | return 0; 104 | } 105 | 106 | static int ntrdma_file_release(struct inode *inode, struct file *filp) 107 | { 108 | struct ntrdma_common_data *common_data = filp->private_data; 109 | 110 | if (common_data->common_page) 111 | put_page(common_data->common_page); 112 | 113 | kfree(common_data); 114 | 115 | return 0; 116 | } 117 | 118 | static long ntrdma_file_ioctl(struct file *filp, unsigned int cmd, 119 | unsigned long arg) 120 | { 121 | struct ntrdma_common_data *common_data = filp->private_data; 122 | u8 data[64]; 123 | int i; 124 | int rc; 125 | 126 | switch (cmd) { 127 | case NTRDMA_FILEIOCTL_REG: 128 | if (common_data->common_page) { 129 | pr_err("%s has no common page\n", __func__); 130 | return -EINVAL; 131 | } 132 | rc = get_user_pages_fast(arg, 1, 1, &common_data->common_page); 133 | if (rc >= 0) 134 | common_data->common_ptr = 135 | page_address(common_data->common_page); 136 | return rc; 137 | case NTRDMA_FILEIOCTL_SEND: 138 | if (!common_data->common_page) { 139 | pr_err("%s has no common page\n", __func__); 140 | return -EINVAL; 141 | } 142 | memcpy(data, (void *)common_data->common_ptr, sizeof(data)); 143 | for (i = 0; i < sizeof(data); i++) 144 | data[i]++; 145 | memcpy((void *)common_data->common_ptr + 1024, 146 | data, sizeof(data)); 147 | return 0; 148 | default: 149 | pr_err("%s unsupported cmd %d\n", __func__, cmd); 150 | return -EINVAL; 151 | } 152 | return 0; 153 | } 154 | 155 | static ssize_t show_measure_perf(struct device *dev, 156 | struct device_attribute *attr, 157 | char *buf) 158 | { 159 | return sprintf(buf, "%d\n", ntrdma_measure_perf ? 1 : 0); 160 | } 161 | 162 | static ssize_t write_measure_perf(struct device *dev, 163 | struct device_attribute *attr, 164 | const char *buf, size_t count) 165 | { 166 | int rc; 167 | int val = 0; 168 | 169 | rc = sscanf(buf, "%i", &val); 170 | if (rc != 1) { 171 | pr_err("%s: Failed to read buffer\n", __func__); 172 | return -EINVAL; 173 | } 174 | 175 | ntrdma_measure_perf = !!val; 176 | 177 | return count; 178 | } 179 | 180 | static ssize_t show_print_debug(struct device *dev, 181 | struct device_attribute *attr, 182 | char *buf) 183 | { 184 | return sprintf(buf, "%d\n", ntrdma_print_debug ? 1 : 0); 185 | } 186 | 187 | static ssize_t write_print_debug(struct device *dev, 188 | struct device_attribute *attr, 189 | const char *buf, size_t count) 190 | { 191 | int rc; 192 | int val = 0; 193 | 194 | rc = sscanf(buf, "%i", &val); 195 | if (rc != 1) { 196 | pr_err("%s: Failed to read buffer\n", __func__); 197 | return -EINVAL; 198 | } 199 | 200 | ntrdma_print_debug = !!val; 201 | 202 | return count; 203 | } 204 | 205 | int __init ntrdma_file_register(void) 206 | { 207 | int rc; 208 | 209 | rc = __register_chrdev(ntrdma_char_major, 0, NTRDMA_MINORS, "ntrdma", 210 | &ntrdma_file_ops); 211 | if (rc < 0) { 212 | pr_err("%s: failed to register chrdev\n", __func__); 213 | goto err_out; 214 | } else { 215 | if (rc > 0) 216 | ntrdma_char_major = rc; 217 | 218 | ntrdma_chrdev_registered = true; 219 | } 220 | 221 | ntrdma_class = class_create(THIS_MODULE, "ntrdma"); 222 | if (IS_ERR(ntrdma_class)) { 223 | rc = PTR_ERR(ntrdma_class); 224 | ntrdma_class = NULL; 225 | pr_err("%s: failed to create ntrdma class\n", __func__); 226 | goto err_out; 227 | } 228 | 229 | ntrdma_dev = device_create_with_groups(ntrdma_class, NULL, 230 | MKDEV(ntrdma_char_major, 0), 231 | NULL, ntrdma_file_attrs_groups, 232 | "ntrdma"); 233 | if (IS_ERR(ntrdma_dev)) { 234 | rc = PTR_ERR(ntrdma_dev); 235 | ntrdma_dev = NULL; 236 | pr_err("%s: faile to create groups\n", __func__); 237 | goto err_out; 238 | } 239 | 240 | 241 | return 0; 242 | err_out: 243 | ntrdma_file_unregister(); 244 | return rc; 245 | } 246 | 247 | void ntrdma_file_unregister(void) 248 | { 249 | if (ntrdma_dev) { 250 | device_destroy(ntrdma_class, MKDEV(ntrdma_char_major, 0)); 251 | ntrdma_dev = NULL; 252 | } 253 | 254 | if (ntrdma_class) { 255 | class_destroy(ntrdma_class); 256 | ntrdma_class = NULL; 257 | } 258 | 259 | if (ntrdma_chrdev_registered) { 260 | __unregister_chrdev(ntrdma_char_major, 0, NTRDMA_MINORS, 261 | "ntrdma"); 262 | ntrdma_chrdev_registered = false; 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019, Dell Technologies, Inc. 3 | * 4 | * This program is free software; you can redistribute it and/or modify it 5 | * under the terms and conditions of the GNU General Public License, 6 | * version 2, as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope it will be useful, but WITHOUT 9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 | * more details. 12 | */ 13 | 14 | #ifndef _NTRDMA_FILE_H 15 | #define _NTRDMA_FILE_H 16 | 17 | #include 18 | 19 | int __init ntrdma_file_register(void); 20 | void ntrdma_file_unregister(void); 21 | 22 | extern bool ntrdma_measure_perf; 23 | extern bool ntrdma_print_debug; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_hello.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include "ntrdma_hello.h" 34 | #include "ntrdma_res.h" 35 | #include "ntrdma_vbell.h" 36 | #include "ntrdma_cq.h" 37 | 38 | enum status { 39 | NOT_DONE = 0, 40 | DONE 41 | }; 42 | 43 | #define MAX_SUPPORTED_VERSIONS 32 44 | 45 | static u32 supported_versions[] = { 46 | NTRDMA_VER_FIRST 47 | }; 48 | 49 | struct ntrdma_hello_phase1 { 50 | /* protocol negotiation */ 51 | u32 versions[MAX_SUPPORTED_VERSIONS]; 52 | u32 version_num; 53 | }; 54 | 55 | struct ntrdma_hello_phase2 { 56 | /* protocol validation */ 57 | u32 version_magic; 58 | u32 phase_magic; 59 | 60 | /* virtual doorbells */ 61 | struct ntc_remote_buf_desc vbell_ntc_buf_desc; 62 | u32 vbell_count; 63 | u32 vbell_reserved; 64 | 65 | /* resource commands */ 66 | struct ntrdma_cmd_hello_info cmd_info; 67 | 68 | /* ethernet frames */ 69 | struct ntrdma_eth_hello_info eth_info; 70 | }; 71 | 72 | struct ntrdma_hello_phase3 { 73 | /* protocol validation */ 74 | u32 version_magic; 75 | u32 phase_magic; 76 | 77 | /* resource commands */ 78 | struct ntrdma_cmd_hello_prep cmd_prep; 79 | 80 | /* ethernet frames */ 81 | struct ntrdma_eth_hello_prep eth_prep; 82 | }; 83 | 84 | static void 85 | ntrdma_buff_supported_versions(struct ntrdma_hello_phase1 __iomem *buff) 86 | { 87 | int i; 88 | 89 | for (i = 0; i < ARRAY_SIZE(supported_versions); i++) 90 | iowrite32(supported_versions[i], &buff->versions[i]); 91 | iowrite32(ARRAY_SIZE(supported_versions), &buff->version_num); 92 | BUILD_BUG_ON(ARRAY_SIZE(supported_versions) > MAX_SUPPORTED_VERSIONS); 93 | } 94 | 95 | static int ntrdma_dev_hello_phase0(struct ntrdma_dev *dev, 96 | const void *in_buf, size_t in_size, 97 | void __iomem *out_buf, size_t out_size) 98 | { 99 | struct ntrdma_hello_phase1 __iomem *out = out_buf; 100 | if (sizeof(struct ntrdma_hello_phase1) > out_size) { 101 | ntrdma_err(dev, "out size %zu is too small for %ld", 102 | out_size, sizeof(struct ntrdma_hello_phase1)); 103 | return -EINVAL; 104 | } 105 | 106 | ntrdma_buff_supported_versions(out); 107 | 108 | return NOT_DONE; 109 | } 110 | 111 | static inline u32 ntrdma_version_choose(struct ntrdma_dev *dev, 112 | const struct ntrdma_hello_phase1 *v1, 113 | const struct ntrdma_hello_phase1 *v2) 114 | { 115 | int i, j; 116 | 117 | for (j = v1->version_num - 1; j >= 0; j--) 118 | for (i = v2->version_num - 1; i >= 0; i--) 119 | if (v1->versions[j] == v2->versions[i]) 120 | return v1->versions[j]; 121 | 122 | ntrdma_err(dev, "Local supported versions (%d) are:\n", 123 | v1->version_num); 124 | for (j = 0; j < v1->version_num; j++) 125 | ntrdma_err(dev, "0x%08x\n", v1->versions[j]); 126 | ntrdma_err(dev, "Remote supported versions (%d) are:\n", 127 | v2->version_num); 128 | for (i = 0; i < v2->version_num; i++) 129 | ntrdma_err(dev, "0x%08x\n", v2->versions[i]); 130 | return NTRDMA_VER_NONE; 131 | } 132 | 133 | 134 | static int ntrdma_dev_hello_phase1(struct ntrdma_dev *dev, 135 | const void *in_buf, size_t in_size, 136 | void __iomem *out_buf, size_t out_size) 137 | { 138 | const struct ntrdma_hello_phase1 *in; 139 | struct ntrdma_hello_phase1 local; 140 | struct ntrdma_hello_phase2 __iomem *out; 141 | struct ntc_remote_buf_desc vbell_ntc_buf_desc; 142 | 143 | if (in_size < sizeof(*in) || out_size < sizeof(*out)) { 144 | ntrdma_err(dev, 145 | "in_size %zu *in size %ld, out_size %zu *out size %ld", 146 | in_size, sizeof(*in), out_size, sizeof(*out)); 147 | return -EINVAL; 148 | } 149 | 150 | in = in_buf; 151 | out = out_buf; 152 | ntrdma_buff_supported_versions(&local); 153 | 154 | dev->hello.latest_version = local.versions[local.version_num-1]; 155 | 156 | if (!in || (in->version_num > MAX_SUPPORTED_VERSIONS)) { 157 | ntrdma_err(dev, "version %d not supported", 158 | in ? in->version_num : -1); 159 | return -EINVAL; 160 | } 161 | 162 | dev->hello.version = ntrdma_version_choose(dev, in, &local); 163 | if (dev->hello.version == NTRDMA_VER_NONE) { 164 | ntrdma_err(dev, "version is not aggreed\n"); 165 | return -EINVAL; 166 | } 167 | ntrdma_dbg(dev, "Agree on version %d", dev->hello.version); 168 | 169 | /* protocol validation */ 170 | iowrite32(NTRDMA_V1_MAGIC, &out->version_magic); 171 | iowrite32(NTRDMA_V1_P2_MAGIC, &out->phase_magic); 172 | 173 | /* virtual doorbells */ 174 | ntc_export_buf_make_desc(&vbell_ntc_buf_desc, &dev->vbell.buf); 175 | memcpy_toio(&out->vbell_ntc_buf_desc, &vbell_ntc_buf_desc, 176 | sizeof(vbell_ntc_buf_desc)); 177 | 178 | iowrite32(dev->vbell.count, &out->vbell_count); 179 | iowrite32(0, &out->vbell_reserved); 180 | 181 | /* command rings */ 182 | ntrdma_dev_cmd_hello_info(dev, &out->cmd_info); 183 | 184 | /* ethernet rings */ 185 | return ntrdma_dev_eth_hello_info(dev, &out->eth_info); 186 | } 187 | 188 | static int ntrdma_dev_hello_phase2(struct ntrdma_dev *dev, 189 | const void *in_buf, size_t in_size, 190 | void __iomem *out_buf, size_t out_size) 191 | { 192 | const struct ntrdma_hello_phase2 *in; 193 | struct ntrdma_hello_phase3 __iomem *out; 194 | int rc; 195 | 196 | if (in_size < sizeof(*in) || out_size < sizeof(*out)) { 197 | ntrdma_err(dev, 198 | "in_size %zu *in size %ld, out_size %zu *out size %ld", 199 | in_size, sizeof(*in), out_size, sizeof(*out)); 200 | return -EINVAL; 201 | } 202 | 203 | in = in_buf; 204 | out = out_buf; 205 | 206 | /* protocol validation */ 207 | if (in->version_magic != NTRDMA_V1_MAGIC) { 208 | ntrdma_err(dev, "version magic 0x%x wrong (0x%x)", 209 | in->version_magic, NTRDMA_V1_MAGIC); 210 | return -EINVAL; 211 | } 212 | if (in->phase_magic != NTRDMA_V1_P2_MAGIC) { 213 | ntrdma_err(dev, "pase magic 0x%x wrong (0x%x)", 214 | in->phase_magic, NTRDMA_V1_P2_MAGIC); 215 | return -EINVAL; 216 | } 217 | 218 | iowrite32(NTRDMA_V1_MAGIC, &out->version_magic); 219 | iowrite32(NTRDMA_V1_P3_MAGIC, &out->phase_magic); 220 | 221 | ntrdma_dbg(dev, "vbell_count %d\n", in->vbell_count); 222 | if (in->vbell_count > NTRDMA_DEV_VBELL_COUNT) { 223 | ntrdma_err(dev, "vbell_count %d\n", in->vbell_count); 224 | return -EINVAL; 225 | } 226 | if (in->vbell_count > in->vbell_ntc_buf_desc.size / sizeof(u32)) { 227 | ntrdma_err(dev, "vbell_count %d vbell_ntc_buf_desc.size %lld\n", 228 | in->vbell_count, in->vbell_ntc_buf_desc.size); 229 | return -EINVAL; 230 | } 231 | 232 | rc = ntrdma_dev_vbell_enable(dev, &in->vbell_ntc_buf_desc, 233 | in->vbell_count); 234 | if (rc) { 235 | ntrdma_err(dev, "failed to enable vbell rc %d\n", rc); 236 | return rc; 237 | } 238 | /* command rings */ 239 | rc = ntrdma_dev_cmd_hello_prep(dev, &in->cmd_info, &out->cmd_prep); 240 | if (rc) { 241 | ntrdma_err(dev, "failed to cmd prep rc %d\n", rc); 242 | return rc; 243 | } 244 | 245 | /* ethernet rings */ 246 | rc = ntrdma_dev_eth_hello_prep(dev, &in->eth_info, &out->eth_prep); 247 | if (rc) { 248 | ntrdma_err(dev, "failed to eth prep rc %d\n", rc); 249 | return rc; 250 | } 251 | return NOT_DONE; 252 | } 253 | 254 | static int ntrdma_dev_hello_phase3(struct ntrdma_dev *dev, 255 | const void *in_buf, size_t in_size, 256 | void __iomem *out_buf, size_t out_size) 257 | { 258 | const struct ntrdma_hello_phase3 *in; 259 | int rc; 260 | 261 | if (in_size < sizeof(*in)) { 262 | ntrdma_err(dev, "in_size %zu smaller then needed %ld", 263 | in_size, sizeof(*in)); 264 | return -EINVAL; 265 | } 266 | 267 | in = in_buf; 268 | 269 | /* protocol validation */ 270 | if (in->version_magic != NTRDMA_V1_MAGIC || 271 | in->phase_magic != NTRDMA_V1_P3_MAGIC) { 272 | ntrdma_err(dev, "couldn't verify magic %u phase magic %u\n", 273 | in->version_magic, in->phase_magic); 274 | return -EINVAL; 275 | } 276 | 277 | /* command rings */ 278 | rc = ntrdma_dev_cmd_hello_done(dev, &in->cmd_prep); 279 | if (unlikely(rc)) 280 | return rc; 281 | 282 | /* ethernet rings */ 283 | rc = ntrdma_dev_eth_hello_done(dev, &in->eth_prep); 284 | if (unlikely(rc)) 285 | return rc; 286 | ntrdma_cq_arm_resync(dev); 287 | 288 | return DONE; 289 | } 290 | 291 | int ntrdma_dev_hello(struct ntrdma_dev *dev, int phase) 292 | { 293 | const void *in_buf; 294 | void __iomem *out_buf; 295 | int in_size = dev->hello.local_buf_size/2; 296 | int out_size = dev->hello.peer_buf_size/2; 297 | 298 | /* note: using double-buffer here, dividing the local and peer buffers 299 | * for two buffers each: 300 | * "odd" phases will use the "even" part of input buffers & "odd" output 301 | * "even" phases will use the opposite of the above 302 | */ 303 | if (phase & 1) { 304 | in_buf = dev->hello.local_buf; 305 | out_buf = dev->hello.peer_buf + out_size; 306 | } else { 307 | in_buf = dev->hello.local_buf + in_size; 308 | out_buf = dev->hello.peer_buf; 309 | } 310 | 311 | ntrdma_dbg(dev, "hello phase %d\n", phase); 312 | 313 | switch (phase) { 314 | case 0: 315 | return ntrdma_dev_hello_phase0(dev, in_buf, in_size, 316 | out_buf, out_size); 317 | case 1: 318 | return ntrdma_dev_hello_phase1(dev, in_buf, in_size, 319 | out_buf, out_size); 320 | case 2: 321 | return ntrdma_dev_hello_phase2(dev, in_buf, in_size, 322 | out_buf, out_size); 323 | case 3: 324 | return ntrdma_dev_hello_phase3(dev, in_buf, in_size, 325 | out_buf, out_size); 326 | } 327 | 328 | ntrdma_err(dev, "Failed\n"); 329 | return -EINVAL; 330 | } 331 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_hello.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_HELLO_H 34 | #define NTRDMA_HELLO_H 35 | 36 | #include "ntrdma.h" 37 | 38 | #define MAX_VBELL_COUNT 1024 39 | #define NTRDMA_VER_NONE 0 40 | #define NTRDMA_VER_FIRST 1 41 | #define NTRDMA_V1_MAGIC 0x3ce4dbd8 42 | #define NTRDMA_V1_P2_MAGIC 0x1a1530f1 43 | #define NTRDMA_V1_P3_MAGIC 0xe09005ed 44 | 45 | struct ntrdma_cmd_hello_info { 46 | struct ntc_remote_buf_desc send_rsp_buf_desc; 47 | u64 send_cons_shift; 48 | u32 send_cap; 49 | u32 send_idx; 50 | u32 send_vbell_idx; 51 | u32 recv_vbell_idx; 52 | }; 53 | 54 | struct ntrdma_cmd_hello_prep { 55 | struct ntc_remote_buf_desc recv_buf_desc; 56 | u64 recv_prod_shift; 57 | }; 58 | 59 | void ntrdma_dev_cmd_hello_info(struct ntrdma_dev *dev, 60 | struct ntrdma_cmd_hello_info __iomem *info); 61 | int ntrdma_dev_cmd_hello_prep(struct ntrdma_dev *dev, 62 | const struct ntrdma_cmd_hello_info *peer_info, 63 | struct ntrdma_cmd_hello_prep __iomem *prep); 64 | int ntrdma_dev_cmd_hello_done(struct ntrdma_dev *dev, 65 | const struct ntrdma_cmd_hello_prep *peer_prep); 66 | 67 | struct ntrdma_eth_hello_info { 68 | u32 rx_cap; 69 | u32 rx_idx; 70 | struct ntc_remote_buf_desc rx_cqe_buf_desc; 71 | struct ntc_remote_buf_desc rx_cons_buf_desc; 72 | u32 vbell_idx; 73 | }; 74 | 75 | struct ntrdma_eth_hello_prep { 76 | struct ntc_remote_buf_desc tx_wqe_buf_desc; 77 | struct ntc_remote_buf_desc tx_prod_buf_desc; 78 | }; 79 | 80 | #ifdef NTRDMA_FULL_ETH 81 | int ntrdma_dev_eth_hello_info(struct ntrdma_dev *dev, 82 | struct ntrdma_eth_hello_info __iomem *info); 83 | int ntrdma_dev_eth_hello_prep(struct ntrdma_dev *dev, 84 | const struct ntrdma_eth_hello_info *peer_info, 85 | struct ntrdma_eth_hello_prep __iomem *prep); 86 | int ntrdma_dev_eth_hello_done(struct ntrdma_dev *dev, 87 | const struct ntrdma_eth_hello_prep *peer_prep); 88 | #else 89 | static inline 90 | int ntrdma_dev_eth_hello_info(struct ntrdma_dev *dev, 91 | struct ntrdma_eth_hello_info __iomem *info) 92 | { 93 | return 0; 94 | } 95 | static inline 96 | int ntrdma_dev_eth_hello_prep(struct ntrdma_dev *dev, 97 | const struct ntrdma_eth_hello_info *peer_info, 98 | struct ntrdma_eth_hello_prep __iomem *prep) 99 | { 100 | return 0; 101 | } 102 | 103 | static inline 104 | int ntrdma_dev_eth_hello_done(struct ntrdma_dev *dev, 105 | const struct ntrdma_eth_hello_prep *peer_prep) 106 | { 107 | return 0; 108 | } 109 | #endif 110 | #endif 111 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_ib.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | #define NTRDMA_PKEY_DEFAULT 0xffff 33 | #define NTRDMA_GIB_TBL_LEN 1 34 | #define NTRDMA_PKEY_TBL_LEN 2 35 | 36 | #define DELL_VENDOR_ID 0x1028 37 | #define NOT_SUPPORTED 0 38 | /* TODO: remove this macro when adding to enum rdma_driver_id 39 | * in include/uapi/rdma/rdma_user_ioctl_cmds.h 40 | */ 41 | #define NTRDMA_IB_PERF_PRINT_ABOVE_MILI_SECS_LATENCY_CONSTANT 400 42 | #define NTRDMA_IB_PERF_INIT unsigned long ___ts_ = 0; \ 43 | unsigned long ___tt_ = 0; \ 44 | unsigned int ___perf_ = 0 45 | #define NTRDMA_IB_PERF_START {___ts_ = jiffies; } 46 | #define NTRDMA_IB_PERF_END do {\ 47 | ___tt_ = jiffies - ___ts_; \ 48 | ___perf_ = jiffies_to_msecs(___tt_);\ 49 | if (time_after(jiffies, ___ts_ + \ 50 | NTRDMA_IB_PERF_PRINT_ABOVE_MILI_SECS_LATENCY_CONSTANT)) { \ 51 | TRACE(\ 52 | "performance time of method %s: %u miliseconds", \ 53 | __func__, \ 54 | ___perf_); \ 55 | } \ 56 | } while (0) 57 | 58 | #define qp_enum_to_string(QP_ENUM) \ 59 | QP_ENUM == IB_QPS_RESET ? "IB_QPS_RESET" :\ 60 | QP_ENUM == IB_QPS_INIT ? "IB_QPS_INIT" :\ 61 | QP_ENUM == IB_QPS_RTR ? "IB_QPS_RTR" :\ 62 | QP_ENUM == IB_QPS_RTS ? "IB_QPS_RTS" :\ 63 | QP_ENUM == IB_QPS_SQD ? "IB_QPS_SQD" :\ 64 | QP_ENUM == IB_QPS_SQE ? "IB_QPS_SQE" :\ 65 | QP_ENUM == IB_QPS_ERR ? "IB_QPS_ERR" :\ 66 | "Undefined state number"\ 67 | 68 | static int ntrdma_qp_file_release(struct inode *inode, struct file *filp); 69 | static long ntrdma_qp_file_ioctl(struct file *filp, unsigned int cmd, 70 | unsigned long arg); 71 | static int ntrdma_cq_file_release(struct inode *inode, struct file *filp); 72 | static long ntrdma_cq_file_ioctl(struct file *filp, unsigned int cmd, 73 | unsigned long arg); 74 | 75 | static struct kmem_cache *qp_slab; 76 | static struct kmem_cache *cq_slab; 77 | 78 | struct ntrdma_ucontext { 79 | struct ib_ucontext ibucontext; 80 | }; 81 | 82 | static const struct file_operations ntrdma_qp_fops = { 83 | .owner = THIS_MODULE, 84 | .release = ntrdma_qp_file_release, 85 | .unlocked_ioctl = ntrdma_qp_file_ioctl, 86 | .compat_ioctl = ntrdma_qp_file_ioctl, 87 | }; 88 | 89 | static const struct file_operations ntrdma_cq_fops = { 90 | .owner = THIS_MODULE, 91 | .release = ntrdma_cq_file_release, 92 | .unlocked_ioctl = ntrdma_cq_file_ioctl, 93 | .compat_ioctl = ntrdma_cq_file_ioctl, 94 | }; 95 | 96 | struct ntrdma_ah { 97 | struct ib_ah ibah; 98 | struct rdma_ah_attr attr; 99 | }; 100 | 101 | static inline void ntrdma_create_ah_common(struct ntrdma_ah *ah, 102 | struct rdma_ah_attr *ah_attr) 103 | { 104 | ah->attr = *ah_attr; 105 | } 106 | 107 | static inline int ntrdma_destroy_qp_common(struct ib_qp *ibqp) 108 | { 109 | struct ntrdma_qp *qp = ntrdma_ib_qp(ibqp); 110 | struct ntrdma_dev *dev = ntrdma_qp_dev(qp); 111 | struct ntrdma_qp_cmd_cb qpcb; 112 | unsigned long t0, t1, t2, t3, t4; 113 | int key; 114 | 115 | NTRDMA_IB_PERF_INIT; 116 | NTRDMA_IB_PERF_START; 117 | 118 | key = qp ? qp->res.key : -1; 119 | ntrdma_dbg(dev, "QP %p (QP %d)\n", qp, key); 120 | 121 | if (unlikely(qp->send_cmpl != qp->send_post)) { 122 | ntrdma_dbg(dev, 123 | "Destroy QP %p (%d) while send cmpl %d send post %d send prod %d send cap %d\n", 124 | qp, qp->res.key, qp->send_cmpl, 125 | qp->send_post, qp->send_prod, qp->send_cap); 126 | } 127 | t0 = jiffies; 128 | ntrdma_debugfs_qp_del(qp); 129 | t1 = jiffies; 130 | memset(&qpcb, 0, sizeof(qpcb)); 131 | 132 | init_completion(&qpcb.cb.cmds_done); 133 | ntrdma_res_del(&qp->res, &qpcb.cb, &dev->res.qp_vec); 134 | t2 = jiffies; 135 | ntc_dma_flush(qp->dma_chan); 136 | t3 = jiffies; 137 | ntrdma_cm_qp_shutdown(qp); 138 | ntrdma_qp_put(qp); 139 | t4 = jiffies; 140 | NTRDMA_IB_PERF_END; 141 | if (t4 - t0 > 100) 142 | ntrdma_info(dev, 143 | "#NTRDMAPERF - put %ld, flush %ld, res_del %ld, debugfs %ld\n", 144 | t4 - t3, t3 - t2, t2 - t1, t1 - t0); 145 | return 0; 146 | } 147 | 148 | static inline int ntrdma_dereg_mr_common(struct ib_mr *ibmr) 149 | { 150 | struct ntrdma_mr *mr = ntrdma_ib_mr(ibmr); 151 | struct ntrdma_dev *dev = ntrdma_mr_dev(mr); 152 | struct ntrdma_mr_cmd_cb mrcb; 153 | struct completion done; 154 | unsigned long t0, t1, t2, t3, t4; 155 | 156 | NTRDMA_IB_PERF_INIT; 157 | NTRDMA_IB_PERF_START; 158 | 159 | ntrdma_dbg(dev, "dereg MR %p (key %d)\n", mr, mr->res.key); 160 | t0 = jiffies; 161 | ntrdma_debugfs_mr_del(mr); 162 | t1 = jiffies; 163 | 164 | memset(&mrcb, 0, sizeof(mrcb)); 165 | init_completion(&mrcb.cb.cmds_done); 166 | ntrdma_res_del(&mr->res, &mrcb.cb, &dev->res.mr_vec); 167 | t2 = jiffies; 168 | 169 | init_completion(&done); 170 | mr->done = &done; 171 | 172 | ntrdma_mr_put(mr); 173 | 174 | wait_for_completion(&done); 175 | t3 = jiffies; 176 | ntc_flush_dma_channels(dev->ntc); 177 | t4 = jiffies; 178 | 179 | 180 | NTRDMA_IB_PERF_END; 181 | if (t4 - t0 > 100) 182 | ntrdma_info(dev, 183 | "#NTRDMAPERF - flush %ld, put %ld, res_del %ld, debugfs %ld\n", 184 | t4 - t3, t3 - t2, t2 - t1, t1 - t0); 185 | return 0; 186 | } 187 | 188 | struct net_device *ntrdma_get_netdev(struct ib_device *ibdev, u8 port_num); 189 | static int ntrdma_get_port_immutable(struct ib_device *ibdev, u8 port, 190 | struct ib_port_immutable *imm); 191 | static int ntrdma_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index, 192 | u16 *pkey); 193 | static int ntrdma_query_gid(struct ib_device *ibdev, u8 port_num, int index, 194 | union ib_gid *ibgid); 195 | static struct ib_mr *ntrdma_get_dma_mr(struct ib_pd *ibpd, 196 | int mr_access_flags); 197 | static int ntrdma_query_device(struct ib_device *ibdev, 198 | struct ib_device_attr *ibattr, struct ib_udata *ibudata); 199 | static int ntrdma_query_port(struct ib_device *ibdev, u8 port, 200 | struct ib_port_attr *ibattr); 201 | 202 | static void ntrdma_cq_release(struct kref *kref); 203 | static void ntrdma_pd_release(struct kref *kref); 204 | 205 | static inline struct ntrdma_cq *ntrdma_destroy_cq_common(struct ib_cq *ibcq) 206 | { 207 | struct ntrdma_cq *cq = ntrdma_ib_cq(ibcq); 208 | struct ntrdma_dev *dev = ntrdma_cq_dev(cq); 209 | 210 | spin_lock_bh(&cq->arm_lock); 211 | cq->ibcq_valid = false; 212 | spin_unlock_bh(&cq->arm_lock); 213 | 214 | /* 215 | * Remove from list before killing vbell, 216 | * so that killed vbell does not go off from ntrdma_cq_arm_resync(). 217 | */ 218 | mutex_lock(&dev->res.lock); 219 | list_del(&cq->obj.dev_entry); 220 | mutex_unlock(&dev->res.lock); 221 | 222 | ntrdma_cq_vbell_kill(cq); 223 | return cq; 224 | } 225 | 226 | static int ntrdma_poll_cq(struct ib_cq *ibcq, int howmany, struct ib_wc *ibwc); 227 | static int ntrdma_req_notify_cq(struct ib_cq *ibcq, 228 | enum ib_cq_notify_flags flags); 229 | static struct ib_qp *ntrdma_create_qp(struct ib_pd *ibpd, 230 | struct ib_qp_init_attr *ibqp_attr, struct ib_udata *ibudata); 231 | static int ntrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *ibqp_attr, 232 | int ibqp_mask, struct ib_qp_init_attr *ibqp_init_attr); 233 | int ntrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *ibqp_attr, 234 | int ibqp_mask, struct ib_udata *ibudata); 235 | static int ntrdma_post_send(struct ib_qp *ibqp, 236 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) 237 | struct ib_send_wr *ibwr, 238 | struct ib_send_wr **bad); 239 | #else 240 | const struct ib_send_wr *ibwr, 241 | const struct ib_send_wr **bad); 242 | #endif 243 | static int ntrdma_post_recv(struct ib_qp *ibqp, 244 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) 245 | struct ib_recv_wr *ibwr, 246 | struct ib_recv_wr **bad); 247 | #else 248 | const struct ib_recv_wr *ibwr, 249 | const struct ib_recv_wr **bad); 250 | #endif 251 | static struct ib_mr *ntrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, 252 | u64 length, u64 virt_addr, int mr_access_flags, 253 | struct ib_udata *ibudata); 254 | enum rdma_link_layer ntrdma_get_link_layer(struct ib_device *device, 255 | u8 port_num); 256 | 257 | static inline struct ntrdma_pd *ntrdma_dealloc_pd_common(struct ib_pd *ibpd) 258 | { 259 | struct ntrdma_pd *pd = ntrdma_ib_pd(ibpd); 260 | struct ntrdma_dev *dev = ntrdma_pd_dev(pd); 261 | 262 | mutex_lock(&dev->res.lock); 263 | list_del(&pd->obj.dev_entry); 264 | mutex_unlock(&dev->res.lock); 265 | 266 | return pd; 267 | } 268 | 269 | int ntrdma_alloc_pd_common(struct ib_pd *ibpd, struct ib_device *ibdev, 270 | struct ib_ucontext *ibuctx, struct ib_udata *ibudata, 271 | struct ntrdma_pd **pd); 272 | static int ntrdma_create_cq_common(struct ib_cq *ibcq, struct ib_device *ibdev, 273 | const struct ib_cq_init_attr *ibattr, struct ib_ucontext *ibuctx, 274 | struct ib_udata *ibudata, struct ntrdma_cq **cq); 275 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_ib_5_3.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | #include "ntrdma_ib.h" 33 | 34 | /* The RDMA_DRIVER_NTRDMA value depends on the rdma-core library and not on kernel version 35 | * Old 2.2 had value of 17, but it conflicts with new values so from version 2.71 it moved 36 | * to 19 */ 37 | #define RDMA_DRIVER_NTRDMA 19 38 | 39 | /* not implemented / not required? */ 40 | /* if required, one needs to implement: 41 | * Perform path query to the Subnet Administrator (SA) 42 | Out of band connection to the remote node, for example: using socket 43 | * Using well-known values, for example: this can be done in a static 44 | subnet (which all of the addresses are predefined) or using 45 | multicast groups 46 | */ 47 | static int ntrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr, 48 | u32 flags, struct ib_udata *udata) 49 | { 50 | struct ntrdma_ah *ah = container_of(ibah, struct ntrdma_ah, ibah); 51 | 52 | ntrdma_create_ah_common(ah, ah_attr); 53 | return 0; 54 | } 55 | 56 | /* not implemented / not required? */ 57 | static void ntrdma_destroy_ah(struct ib_ah *ibah, u32 flags) 58 | { 59 | } 60 | 61 | static inline void ntrdma_cq_release_cache_free(struct ntrdma_cq *cq) 62 | { 63 | kmem_cache_free(cq_slab, cq); 64 | } 65 | 66 | static inline void ntrdma_pd_release_cache_free(struct ntrdma_pd *pd) 67 | { 68 | } 69 | 70 | static int ntrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 71 | { 72 | return ntrdma_destroy_qp_common(ibqp); 73 | } 74 | 75 | struct ib_umem *ntrdma_ib_umem_get(struct ib_udata *ibudata, 76 | struct ib_ucontext *context, unsigned long addr, size_t size, 77 | int access, int dmasync) 78 | { 79 | return ib_umem_get(ibudata, addr, size, access, dmasync); 80 | } 81 | 82 | static int ntrdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 83 | { 84 | return ntrdma_dereg_mr_common(ibmr); 85 | } 86 | 87 | static int ntrdma_alloc_ucontext(struct ib_ucontext *ctx, 88 | struct ib_udata *udata) 89 | { 90 | return 0; 91 | } 92 | 93 | static void ntrdma_dealloc_ucontext(struct ib_ucontext *ibuctx) 94 | { 95 | } 96 | 97 | int ntrdma_add_gid(const struct ib_gid_attr *attr, void **context) 98 | { 99 | return 0; 100 | } 101 | 102 | int ntrdma_del_gid(const struct ib_gid_attr *attr, void **context) 103 | { 104 | return 0; 105 | } 106 | 107 | int ntrdma_process_mad(struct ib_device *device, 108 | int process_mad_flags, 109 | u8 port_num, 110 | const struct ib_wc *in_wc, 111 | const struct ib_grh *in_grh, 112 | const struct ib_mad *in_mad, 113 | struct ib_mad *out_mad, 114 | size_t *out_mad_size, 115 | u16 *out_mad_pkey_index) 116 | { 117 | TRACE("RDMA CM MAD received: class %d\n", in_mad->mad_hdr.mgmt_class); 118 | return IB_MAD_RESULT_SUCCESS; 119 | } 120 | 121 | void ntrdma_disassociate_ucontext(struct ib_ucontext *ibcontext) 122 | { 123 | } 124 | 125 | static void ntrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 126 | { 127 | struct ntrdma_cq *cq; 128 | 129 | NTRDMA_IB_PERF_INIT; 130 | NTRDMA_IB_PERF_START; 131 | 132 | cq = ntrdma_destroy_cq_common(ibcq); 133 | 134 | ntrdma_cq_put(cq); 135 | 136 | NTRDMA_IB_PERF_END; 137 | } 138 | 139 | void ntrdma_pd_get(struct ntrdma_pd *pd) 140 | { 141 | } 142 | 143 | void ntrdma_pd_put(struct ntrdma_pd *pd) 144 | { 145 | } 146 | 147 | static void ntrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 148 | { 149 | struct ntrdma_pd *pd; 150 | 151 | NTRDMA_IB_PERF_INIT; 152 | NTRDMA_IB_PERF_START; 153 | 154 | pd = ntrdma_dealloc_pd_common(ibpd); 155 | ntrdma_pd_release(&pd->obj.kref); 156 | 157 | NTRDMA_IB_PERF_END; 158 | } 159 | 160 | static inline struct ntrdma_pd *ntrdma_new_pd(struct ib_pd *ibpd, 161 | struct ntrdma_dev *dev) 162 | { 163 | struct ntrdma_pd *pd = container_of(ibpd, struct ntrdma_pd, ibpd); 164 | 165 | return pd; 166 | } 167 | 168 | static int ntrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *ibudata) 169 | { 170 | struct ntrdma_pd *pd; 171 | 172 | return ntrdma_alloc_pd_common(ibpd, ibpd->device, NULL, ibudata, &pd); 173 | } 174 | 175 | static inline void ntrdma_dev_ib_deinit_common(struct ntrdma_dev *dev) 176 | { 177 | kmem_cache_destroy(dev->cmid_node_slab); 178 | } 179 | 180 | static inline struct ntrdma_cq *ntrdma_alloc_cq(struct ib_cq *ibcq, 181 | struct ntrdma_dev *dev) 182 | { 183 | struct ntrdma_ib_cq *ntrdma_ib_cq = 184 | container_of(ibcq, struct ntrdma_ib_cq, ibcq); 185 | struct ntrdma_cq *cq = 186 | kmem_cache_alloc_node(cq_slab, GFP_KERNEL, dev->node); 187 | 188 | if (ntrdma_ib_cq) { 189 | ntrdma_ib_cq->cq = cq; 190 | } 191 | if (cq) { 192 | memset(cq, 0, sizeof(*cq)); 193 | cq->ibcq = ntrdma_ib_cq; 194 | } 195 | 196 | return cq; 197 | } 198 | 199 | static int ntrdma_create_cq(struct ib_cq *ibcq, 200 | const struct ib_cq_init_attr *ibattr, 201 | struct ib_udata *ibudata) 202 | { 203 | struct ntrdma_cq *cq = NULL; 204 | 205 | return ntrdma_create_cq_common(ibcq, ibcq->device, ibattr, NULL, ibudata, 206 | &cq); 207 | } 208 | 209 | static const struct ib_device_ops ntrdma_dev_ops = { 210 | .owner = THIS_MODULE, 211 | .driver_id = RDMA_DRIVER_NTRDMA, 212 | .uverbs_abi_ver = 1, 213 | 214 | /* not implemented / not required */ 215 | .get_netdev = ntrdma_get_netdev, 216 | .get_port_immutable = ntrdma_get_port_immutable, 217 | .query_pkey = ntrdma_query_pkey, 218 | .query_gid = ntrdma_query_gid, 219 | .create_ah = ntrdma_create_ah, 220 | .destroy_ah = ntrdma_destroy_ah, 221 | .get_dma_mr = ntrdma_get_dma_mr, 222 | 223 | /* userspace context */ 224 | .alloc_ucontext = ntrdma_alloc_ucontext, 225 | .dealloc_ucontext = ntrdma_dealloc_ucontext, 226 | .disassociate_ucontext = ntrdma_disassociate_ucontext, 227 | 228 | /* device and port queries */ 229 | .query_device = ntrdma_query_device, 230 | .query_port = ntrdma_query_port, 231 | 232 | /* completion queue */ 233 | .create_cq = ntrdma_create_cq, 234 | .destroy_cq = ntrdma_destroy_cq, 235 | .poll_cq = ntrdma_poll_cq, 236 | .req_notify_cq = ntrdma_req_notify_cq, 237 | 238 | /* protection domain */ 239 | .alloc_pd = ntrdma_alloc_pd, 240 | .dealloc_pd = ntrdma_dealloc_pd, 241 | 242 | /* memory region */ 243 | .reg_user_mr = ntrdma_reg_user_mr, 244 | .dereg_mr = ntrdma_dereg_mr, 245 | 246 | /* queue pair */ 247 | .create_qp = ntrdma_create_qp, 248 | .query_qp = ntrdma_query_qp, 249 | .modify_qp = ntrdma_modify_qp, 250 | .destroy_qp = ntrdma_destroy_qp, 251 | .post_send = ntrdma_post_send, 252 | .post_recv = ntrdma_post_recv, 253 | .get_link_layer = ntrdma_get_link_layer, 254 | .add_gid = ntrdma_add_gid, 255 | .del_gid = ntrdma_del_gid, 256 | .process_mad = ntrdma_process_mad, 257 | 258 | INIT_RDMA_OBJ_SIZE(ib_ah, ntrdma_ah, ibah), 259 | INIT_RDMA_OBJ_SIZE(ib_pd, ntrdma_pd, ibpd), 260 | INIT_RDMA_OBJ_SIZE(ib_cq, ntrdma_ib_cq, ibcq), 261 | INIT_RDMA_OBJ_SIZE(ib_ucontext, ntrdma_ucontext, ibucontext), 262 | }; 263 | 264 | static inline int ntrdma_ib_register_device(struct ib_device *ibdev) 265 | { 266 | return ib_register_device(ibdev, "ntrdma_%d"); 267 | } 268 | 269 | static inline int ntrdma_set_ib_ops(struct ntrdma_dev *dev, 270 | struct ib_device *ibdev) 271 | { 272 | ib_set_device_ops(ibdev, &ntrdma_dev_ops); 273 | ntrdma_cm_init(ibdev); 274 | return 0; 275 | } 276 | 277 | void ntrdma_ib_module_deinit(void) 278 | { 279 | ntrdma_deinit_slab(&qp_slab); 280 | ntrdma_deinit_slab(&cq_slab); 281 | } 282 | 283 | static inline bool ntrdma_slab_init(void) 284 | { 285 | return ((qp_slab = KMEM_CACHE(ntrdma_qp, 0)) && 286 | (cq_slab = KMEM_CACHE(ntrdma_cq, 0))); 287 | } 288 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_ioctl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014-2019 Dell EMC, Inc. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef __NTRDMA_IOCTL_H__ 34 | #define __NTRDMA_IOCTL_H__ 35 | 36 | #ifdef __KERNEL__ 37 | 38 | #include 39 | #include 40 | 41 | typedef u64 ntrdma_u64; 42 | typedef u32 ntrdma_u32; 43 | typedef u16 ntrdma_u16; 44 | typedef u8 ntrdma_u8; 45 | typedef struct ib_sge ntrdma_sge; 46 | 47 | #else 48 | 49 | #include 50 | #include 51 | 52 | typedef uint64_t ntrdma_u64; 53 | typedef uint32_t ntrdma_u32; 54 | typedef uint16_t ntrdma_u16; 55 | typedef uint8_t ntrdma_u8; 56 | typedef struct ibv_sge ntrdma_sge; 57 | 58 | #endif 59 | 60 | #define NTRDMA_IOCTL_IF_HELPER_IMPL(SDESC, ...) \ 61 | typedef char ntrdma_ioctl_if_desc[sizeof(SDESC)]; \ 62 | \ 63 | static inline \ 64 | void ntrdma_ioctl_if_init_desc(ntrdma_ioctl_if_desc *desc) \ 65 | { \ 66 | memcpy(*desc, SDESC, sizeof(SDESC)); \ 67 | } \ 68 | \ 69 | static inline \ 70 | bool ntrdma_ioctl_if_check_desc(ntrdma_ioctl_if_desc *desc) \ 71 | { \ 72 | return !memcmp(*desc, SDESC, sizeof(SDESC)); \ 73 | } \ 74 | \ 75 | __VA_ARGS__ 76 | 77 | #define NTRDMA_IOCTL_IF_STRINGIFY_IMPL(X) #X 78 | #define NTRDMA_IOCTL_IF_STRINGIFY(X) NTRDMA_IOCTL_IF_STRINGIFY_IMPL(X) 79 | 80 | #define NTRDMA_IOCTL_IF_HELPER(DESC) \ 81 | NTRDMA_IOCTL_IF_HELPER_IMPL(NTRDMA_IOCTL_IF_STRINGIFY((DESC)), DESC) 82 | 83 | #define NTRDMA_IOCTL_IF \ 84 | \ 85 | struct ntrdma_create_qp_ext { \ 86 | ntrdma_ioctl_if_desc desc; \ 87 | ntrdma_u64 send_page_ptr; \ 88 | }; \ 89 | \ 90 | struct ntrdma_create_qp_resp_ext { \ 91 | int qpfd; \ 92 | }; \ 93 | \ 94 | struct ntrdma_create_cq_ext { \ 95 | ntrdma_ioctl_if_desc desc; \ 96 | ntrdma_u64 poll_page_ptr; \ 97 | }; \ 98 | \ 99 | struct ntrdma_create_cq_resp_ext { \ 100 | int cqfd; \ 101 | }; \ 102 | \ 103 | typedef enum { \ 104 | NTRDMA_SMALL_ENUM_MIN = 0, \ 105 | NTRDMA_SMALL_ENUM_MAX = 1 << 7, \ 106 | } ntrdma_small_enum; \ 107 | \ 108 | struct ntrdma_send_wqe { \ 109 | ntrdma_u64 ulp_handle; \ 110 | ntrdma_u16 op_code; \ 111 | ntrdma_u16 op_status; \ 112 | ntrdma_u32 recv_key; \ 113 | ntrdma_sge rdma_sge; \ 114 | ntrdma_u32 imm_data; \ 115 | ntrdma_u32 flags; \ 116 | union { \ 117 | ntrdma_u32 sg_count; \ 118 | ntrdma_u32 inline_len; \ 119 | }; \ 120 | }; \ 121 | \ 122 | struct ntrdma_snd_hdr { \ 123 | ntrdma_u32 wqe_counter; \ 124 | ntrdma_u32 first_wqe_size; \ 125 | }; \ 126 | \ 127 | struct ntrdma_poll_hdr { \ 128 | ntrdma_u32 wc_counter; \ 129 | }; \ 130 | \ 131 | struct ntrdma_ibv_wc { \ 132 | ntrdma_u64 wr_id; \ 133 | ntrdma_small_enum status; \ 134 | ntrdma_small_enum opcode; \ 135 | ntrdma_u32 vendor_err; \ 136 | ntrdma_u32 byte_len; \ 137 | union { \ 138 | ntrdma_u32 imm_data; \ 139 | ntrdma_u32 invalidated_rkey; \ 140 | }; \ 141 | ntrdma_u32 qp_num; \ 142 | ntrdma_u32 src_qp; \ 143 | unsigned int wc_flags; \ 144 | ntrdma_u16 pkey_index; \ 145 | ntrdma_u16 slid; \ 146 | ntrdma_u8 sl; \ 147 | ntrdma_u8 dlid_path_bits; \ 148 | }; \ 149 | \ 150 | enum { \ 151 | NTRDMA_IOCTL_SEND = _IO('N', 0x30), \ 152 | NTRDMA_IOCTL_POLL = _IO('N', 0x31), \ 153 | }; 154 | 155 | NTRDMA_IOCTL_IF_HELPER(NTRDMA_IOCTL_IF); 156 | 157 | #endif 158 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_main.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #include "ntrdma.h" 40 | #include "ntrdma_file.h" 41 | #include "ntrdma_dev.h" 42 | 43 | #define STRINGIFY(x) #x 44 | #define TOSTRING(x) STRINGIFY(x) 45 | 46 | #define DRIVER_NAME "ntrdma" 47 | #define DRIVER_VERSION "0.2" 48 | #define DRIVER_RELDATE "20 October 2015" 49 | 50 | #define MAX_LEN 2 51 | 52 | MODULE_AUTHOR("Allen Hubbe"); 53 | MODULE_DESCRIPTION("RDMA Driver for PCIe NTB and DMA"); 54 | MODULE_LICENSE("GPL v2"); 55 | MODULE_VERSION(DRIVER_VERSION); 56 | #ifdef NTRDMA_GIT_HASH 57 | MODULE_INFO(githash, TOSTRING(NTRDMA_GIT_HASH)); 58 | #endif 59 | 60 | 61 | static ssize_t link_show(struct device *dev, 62 | struct device_attribute *attr, 63 | char *buf) 64 | { 65 | struct ib_device *ibdev = container_of(dev, struct ib_device, dev); 66 | struct ntrdma_dev *ntrdma_dev = ntrdma_ib_dev(ibdev); 67 | struct ntc_dev *ntc = ntrdma_dev->ntc; 68 | 69 | return snprintf(buf, MAX_LEN, "%d\n", ntc_is_link_up(ntc)); 70 | } 71 | 72 | static ssize_t link_store(struct device *dev, 73 | struct device_attribute *attr, 74 | const char *buf, size_t count) 75 | { 76 | int rc = 0; 77 | 78 | struct ib_device *ibdev = container_of(dev, struct ib_device, dev); 79 | struct ntrdma_dev *ntrdma_dev = ntrdma_ib_dev(ibdev); 80 | struct ntc_dev *ntc = ntrdma_dev->ntc; 81 | int val; 82 | 83 | if (!ntc) { 84 | pr_err("%s: ERROR sysfs: failed to store sysfs link file, ntc is NULL\n", __func__); 85 | return -ENODEV; 86 | } 87 | 88 | if (sscanf(buf, "%d", &val) != 1) { 89 | pr_err("%s: ERROR sysfs: wrong param %s\n", __func__, buf); 90 | return -EINVAL; 91 | } 92 | 93 | if ((val == 0) && ntc_is_link_up(ntc)) { 94 | pr_debug("%s: sysfs: changed link state to %d\n", __func__, val); 95 | rc = ntc_link_disable(ntc); 96 | } else if ((val == 1) && !ntc_is_link_up(ntc)) { 97 | pr_debug("%s: sysfs: changed link state to %d\n", __func__, val); 98 | rc = ntc_link_enable(ntc); 99 | } else { 100 | pr_err("%s: ERROR sysfs: link already %d\n", __func__, val); 101 | if ((val != 0) && (val != 1)) 102 | return -EINVAL; 103 | } 104 | 105 | if (rc) 106 | pr_err("%s: sysfs: could not set link state (%d) by ntc, rc %d\n", 107 | __func__, val, rc); 108 | 109 | return count; 110 | } 111 | 112 | static struct device_attribute attr = { 113 | .attr = { 114 | .name = "link", 115 | .mode = 0660,/*S_IWUSR | S_IRUGO,*/ 116 | }, 117 | .show = link_show, 118 | .store = link_store, 119 | }; 120 | 121 | static int ntrdma_probe(struct ntc_driver *self, 122 | struct ntc_dev *ntc) 123 | { 124 | struct ntrdma_dev *dev; 125 | int rc; 126 | 127 | pr_devel("probe ntc %s\n", dev_name(&ntc->dev)); 128 | #ifdef NTRDMA_GIT_HASH 129 | pr_info("Probe ntrdma - git hash %s", TOSTRING(NTRDMA_GIT_HASH)); 130 | #endif 131 | ntc_link_disable(ntc); 132 | 133 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0) 134 | dev = (void *)ib_alloc_device(ntrdma_dev, ibdev); 135 | #else 136 | dev = (void *)ib_alloc_device(sizeof(*dev)); 137 | #endif 138 | if (!dev) 139 | return -ENOMEM; 140 | 141 | rc = ntrdma_dev_init(dev, ntc); 142 | if (rc) 143 | goto err_init; 144 | 145 | ntc_link_enable(ntc); 146 | 147 | rc = device_create_file((struct device *)&dev->ibdev.dev, &attr); 148 | if (rc) 149 | pr_err("failed to create sysfs entry rc = %d\n", rc); 150 | 151 | 152 | return 0; 153 | 154 | err_init: 155 | ib_dealloc_device((void *)dev); 156 | return rc; 157 | } 158 | 159 | static void ntrdma_remove(struct ntc_driver *self, struct ntc_dev *ntc) 160 | { 161 | struct ntrdma_dev *dev = ntc_get_ctx(ntc); 162 | pr_devel("remove ntc %s\n", dev_name(&ntc->dev)); 163 | device_remove_file((struct device *)&dev->ibdev.dev, &attr); 164 | ntrdma_dev_ib_deinit(dev); 165 | ntc_link_disable(ntc); 166 | ntc_link_reset(dev->ntc, SYNC_RESET); 167 | /* Prevent callbacks from the lower layer */ 168 | ntc_clear_ctx(dev->ntc); 169 | ntrdma_dev_deinit(dev); 170 | ib_dealloc_device((void *)dev); 171 | } 172 | 173 | struct ntc_driver ntrdma_driver = { 174 | .drv = { 175 | .name = KBUILD_MODNAME, 176 | }, 177 | .ops = { 178 | .probe = ntrdma_probe, 179 | .remove = ntrdma_remove, 180 | }, 181 | }; 182 | 183 | static void ntrdma_deinit(void) 184 | { 185 | ntrdma_debugfs_deinit(); 186 | ntrdma_eth_module_deinit(); 187 | ntrdma_ib_module_deinit(); 188 | ntrdma_qp_module_deinit(); 189 | ntrdma_vbell_module_deinit(); 190 | } 191 | 192 | static int __init ntrdma_init(void) 193 | { 194 | int rc; 195 | 196 | pr_info("NTRDMA module init\n"); 197 | 198 | rc = ntrdma_vbell_module_init(); 199 | if (rc < 0) 200 | goto err; 201 | 202 | rc = ntrdma_qp_module_init(); 203 | if (rc < 0) 204 | goto err; 205 | 206 | rc = ntrdma_ib_module_init(); 207 | if (rc < 0) 208 | goto err; 209 | 210 | rc = ntrdma_eth_module_init(); 211 | if (rc < 0) 212 | goto err; 213 | 214 | rc = ntrdma_debugfs_init(); 215 | if (rc < 0) 216 | goto err; 217 | 218 | rc = ntc_register_driver(&ntrdma_driver); 219 | if (rc < 0) 220 | goto err; 221 | 222 | rc = ntrdma_file_register(); 223 | if (rc < 0) 224 | goto err; 225 | 226 | return 0; 227 | 228 | err: 229 | ntrdma_deinit(); 230 | return rc; 231 | } 232 | module_init(ntrdma_init); 233 | 234 | static __exit void ntrdma_exit(void) 235 | { 236 | ntrdma_file_unregister(); 237 | ntc_unregister_driver(&ntrdma_driver); 238 | ntrdma_deinit(); 239 | pr_info("NTRDMA module exit\n"); 240 | } 241 | module_exit(ntrdma_exit); 242 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_mr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include "ntrdma_dev.h" 34 | #include "ntrdma_cmd.h" 35 | #include "ntrdma_pd.h" 36 | #include "ntrdma_mr.h" 37 | #include "ntrdma_res.h" 38 | 39 | #define ntrdma_cmd_cb_mrcb(__cb) \ 40 | container_of(__cb, struct ntrdma_mr_cmd_cb, cb) 41 | 42 | static int ntrdma_mr_enable_prep(struct ntrdma_cmd_cb *cb, 43 | union ntrdma_cmd *cmd); 44 | static void ntrdma_mr_enable_cmpl(struct ntrdma_cmd_cb *cb, 45 | const union ntrdma_rsp *rsp); 46 | static int ntrdma_mr_disable_prep(struct ntrdma_cmd_cb *cb, 47 | union ntrdma_cmd *cmd); 48 | static void ntrdma_mr_disable_cmpl(struct ntrdma_cmd_cb *cb, 49 | const union ntrdma_rsp *rsp); 50 | 51 | static int ntrdma_mr_enable_cb(struct ntrdma_res *res, 52 | struct ntrdma_cmd_cb *cb); 53 | static int ntrdma_mr_disable_cb(struct ntrdma_res *res, 54 | struct ntrdma_cmd_cb *cb); 55 | 56 | static void ntrdma_rmr_free(struct ntrdma_rres *rres); 57 | 58 | int ntrdma_mr_init(struct ntrdma_mr *mr, struct ntrdma_dev *dev) 59 | { 60 | int count; 61 | int rc; 62 | 63 | if (mr->ib_umem) { 64 | count = ntc_umem_sgl(dev->ntc, mr->ib_umem, 65 | mr->sg_list, mr->sg_count, mr->access); 66 | if (count != mr->sg_count) { 67 | ntrdma_err(dev, "mr %p count %d != sg_count %d", 68 | mr, count, mr->sg_count); 69 | rc = -EFAULT; 70 | goto err; 71 | } 72 | } else 73 | count = mr->sg_count; 74 | 75 | ntrdma_res_init(&mr->res, dev, 76 | ntrdma_mr_enable_cb, ntrdma_mr_disable_cb); 77 | 78 | rc = ntrdma_kvec_reserve_key(&dev->res.mr_vec, dev->node); 79 | if (rc < 0) { 80 | ntrdma_err(dev, "mr %p failed to reserve key rc = %d", mr, rc); 81 | goto err; 82 | } 83 | mr->res.key = rc; 84 | 85 | return 0; 86 | 87 | err: 88 | ntc_mr_buf_clear_sgl(mr->sg_list, count); 89 | 90 | return rc; 91 | } 92 | 93 | void ntrdma_mr_deinit(struct ntrdma_mr *mr, struct ntrdma_dev *dev) 94 | { 95 | ntrdma_kvec_dispose_key(dev->node, &dev->res.mr_vec, mr->res.key); 96 | ntc_mr_buf_clear_sgl(mr->sg_list, mr->sg_count); 97 | } 98 | 99 | static int ntrdma_mr_enable_cb(struct ntrdma_res *res, 100 | struct ntrdma_cmd_cb *cb) 101 | { 102 | struct ntrdma_dev *dev = ntrdma_res_dev(res); 103 | struct ntrdma_mr *mr = ntrdma_res_mr(res); 104 | struct ntrdma_mr_cmd_cb *mrcb; 105 | u32 count; 106 | 107 | mrcb = container_of(cb, struct ntrdma_mr_cmd_cb, cb); 108 | 109 | count = mr->sg_count; 110 | if (count > NTRDMA_CMD_MR_CREATE_SG_CAP) 111 | count = NTRDMA_CMD_MR_CREATE_SG_CAP; 112 | 113 | mrcb->cb.cmd_prep = ntrdma_mr_enable_prep; 114 | mrcb->cb.rsp_cmpl = ntrdma_mr_enable_cmpl; 115 | mrcb->mr = mr; 116 | mrcb->sg_pos = 0; 117 | mrcb->sg_count = count; 118 | 119 | return ntrdma_dev_cmd_add(dev, &mrcb->cb); 120 | } 121 | 122 | void ntrdma_mr_enable(struct ntrdma_mr *mr) 123 | { 124 | reinit_completion(&mr->enable_mrcb.cb.cmds_done); 125 | ntrdma_mr_enable_cb(&mr->res, &mr->enable_mrcb.cb); 126 | } 127 | 128 | static int ntrdma_mr_enable_prep(struct ntrdma_cmd_cb *cb, 129 | union ntrdma_cmd *cmd) 130 | { 131 | struct ntrdma_mr_cmd_cb *mrcb = ntrdma_cmd_cb_mrcb(cb); 132 | struct ntrdma_mr *mr = mrcb->mr; 133 | int i; 134 | 135 | TRACE("mr_enable prep: key=%d sg_pos=%d sg_count=%d", 136 | mr->res.key, mrcb->sg_pos, mrcb->sg_count); 137 | 138 | if (mrcb->sg_pos == 0) { 139 | cmd->mr_create.hdr.op = NTRDMA_CMD_MR_CREATE; 140 | cmd->mr_create.mr_key = mr->res.key; 141 | cmd->mr_create.pd_key = mr->pd_key; 142 | cmd->mr_create.access = mr->access; 143 | cmd->mr_create.mr_addr = mr->addr; 144 | cmd->mr_create.mr_len = mr->len; 145 | 146 | cmd->mr_create.sg_cap = mr->sg_count; 147 | cmd->mr_create.sg_count = mrcb->sg_count; 148 | 149 | for (i = 0; i < mrcb->sg_count; ++i) 150 | ntc_mr_buf_make_desc(&cmd->mr_create.sg_desc_list[i], 151 | &mr->sg_list[i]); 152 | } else { 153 | cmd->mr_append.hdr.op = NTRDMA_CMD_MR_APPEND; 154 | cmd->mr_append.mr_key = mr->res.key; 155 | 156 | cmd->mr_append.sg_pos = mrcb->sg_pos; 157 | cmd->mr_append.sg_count = mrcb->sg_count; 158 | 159 | for (i = 0; i < mrcb->sg_count; ++i) 160 | ntc_mr_buf_make_desc(&cmd->mr_append.sg_desc_list[i], 161 | &mr->sg_list[mrcb->sg_pos + i]); 162 | } 163 | 164 | return 0; 165 | } 166 | 167 | static void ntrdma_mr_enable_cmpl(struct ntrdma_cmd_cb *cb, 168 | const union ntrdma_rsp *rsp) 169 | { 170 | struct ntrdma_mr_cmd_cb *mrcb = ntrdma_cmd_cb_mrcb(cb); 171 | struct ntrdma_mr *mr = mrcb->mr; 172 | struct ntrdma_dev *dev = ntrdma_res_dev(&mr->res); 173 | u32 end, count; 174 | 175 | TRACE("mr_enable cmpl: %d\n", mr->res.key); 176 | 177 | cb->ret = READ_ONCE(rsp->hdr.status); 178 | if (unlikely(cb->ret)) { 179 | ntrdma_err(dev, "mr %p rsp status %d", mr, cb->ret); 180 | goto out; 181 | } 182 | 183 | end = mrcb->sg_pos + mrcb->sg_count; 184 | if (end != mr->sg_count) { 185 | /* Continue to send mr enable segments */ 186 | count = mr->sg_count - end; 187 | if (count > NTRDMA_CMD_MR_APPEND_SG_CAP) 188 | count = NTRDMA_CMD_MR_APPEND_SG_CAP; 189 | 190 | mrcb->sg_pos = end; 191 | mrcb->sg_count = count; 192 | 193 | ntrdma_dev_cmd_add_unsafe(dev, &mrcb->cb); 194 | 195 | return; 196 | } 197 | 198 | out: 199 | complete_all(&cb->cmds_done); 200 | } 201 | 202 | static int ntrdma_mr_disable_cb(struct ntrdma_res *res, 203 | struct ntrdma_cmd_cb *cb) 204 | { 205 | struct ntrdma_dev *dev = ntrdma_res_dev(res); 206 | struct ntrdma_mr *mr = ntrdma_res_mr(res); 207 | struct ntrdma_mr_cmd_cb *mrcb; 208 | 209 | mrcb = container_of(cb, struct ntrdma_mr_cmd_cb, cb); 210 | 211 | mrcb->cb.cmd_prep = ntrdma_mr_disable_prep; 212 | mrcb->cb.rsp_cmpl = ntrdma_mr_disable_cmpl; 213 | mrcb->mr = mr; 214 | mrcb->sg_pos = 0; 215 | mrcb->sg_count = 0; 216 | 217 | return ntrdma_dev_cmd_add(dev, &mrcb->cb); 218 | } 219 | 220 | static int ntrdma_mr_disable_prep(struct ntrdma_cmd_cb *cb, 221 | union ntrdma_cmd *cmd) 222 | { 223 | struct ntrdma_mr_cmd_cb *mrcb = ntrdma_cmd_cb_mrcb(cb); 224 | struct ntrdma_mr *mr = mrcb->mr; 225 | 226 | cmd->mr_delete.hdr.op = NTRDMA_CMD_MR_DELETE; 227 | cmd->mr_delete.mr_key = mr->res.key; 228 | 229 | return 0; 230 | } 231 | 232 | static void ntrdma_mr_disable_cmpl(struct ntrdma_cmd_cb *cb, 233 | const union ntrdma_rsp *rsp) 234 | { 235 | struct ntrdma_mr_cmd_cb *mrcb = ntrdma_cmd_cb_mrcb(cb); 236 | struct ntrdma_mr *mr = mrcb->mr; 237 | struct ntrdma_dev *dev = ntrdma_res_dev(&mr->res); 238 | 239 | cb->ret = READ_ONCE(rsp->hdr.status); 240 | if (unlikely(cb->ret)) 241 | ntrdma_err(dev, "mr %p status %d", mr, cb->ret); 242 | 243 | complete_all(&cb->cmds_done); 244 | return; 245 | } 246 | 247 | void ntrdma_rmr_init(struct ntrdma_rmr *rmr, 248 | struct ntrdma_dev *dev, 249 | u32 pd_key, u32 access, 250 | u64 addr, u64 len, 251 | u32 sg_count, u32 key) 252 | { 253 | rmr->done = NULL; 254 | rmr->pd_key = pd_key; 255 | rmr->access = access; 256 | 257 | rmr->addr = addr; 258 | rmr->len = len; 259 | 260 | rmr->sg_count = sg_count; 261 | 262 | memset(rmr->sg_list, 0, 263 | sg_count * sizeof(*rmr->sg_list)); 264 | 265 | ntrdma_rres_init(&rmr->rres, dev, &dev->rres.rmr_vec, 266 | ntrdma_rmr_free, key); 267 | } 268 | 269 | static void ntrdma_rmr_release(struct kref *kref) 270 | { 271 | struct ntrdma_obj *obj = container_of(kref, struct ntrdma_obj, kref); 272 | struct ntrdma_rres *rres = container_of(obj, struct ntrdma_rres, obj); 273 | struct ntrdma_rmr *rmr = container_of(rres, struct ntrdma_rmr, rres); 274 | struct ntrdma_dev *dev = ntrdma_rres_dev(rres); 275 | struct completion *done = rmr->done; 276 | int i; 277 | 278 | ntrdma_debugfs_rmr_del(rmr); 279 | 280 | for (i = 0; i < rmr->sg_count; i++) 281 | ntc_remote_buf_unmap(&rmr->sg_list[i], dev->ntc); 282 | 283 | kfree(rmr); 284 | 285 | if (done) 286 | complete_all(done); 287 | } 288 | 289 | void ntrdma_rmr_put(struct ntrdma_rmr *rmr) 290 | { 291 | ntrdma_rres_put(&rmr->rres, ntrdma_rmr_release); 292 | } 293 | 294 | static void ntrdma_rmr_free(struct ntrdma_rres *rres) 295 | { 296 | struct ntrdma_rmr *rmr = ntrdma_rres_rmr(rres); 297 | 298 | ntrdma_rmr_put(rmr); 299 | /* SYNC ref == 0 ?*/ 300 | } 301 | 302 | struct ntrdma_mr *ntrdma_dev_mr_look(struct ntrdma_dev *dev, u32 key) 303 | { 304 | struct ntrdma_res *res; 305 | 306 | res = ntrdma_res_look(&dev->res.mr_vec, key); 307 | if (!res) 308 | return NULL; 309 | 310 | return ntrdma_res_mr(res); 311 | } 312 | 313 | struct ntrdma_rmr *ntrdma_dev_rmr_look(struct ntrdma_dev *dev, u32 key) 314 | { 315 | struct ntrdma_rres *rres; 316 | 317 | rres = ntrdma_rres_look(&dev->rres.rmr_vec, key); 318 | if (!rres) 319 | return NULL; 320 | 321 | return ntrdma_rres_rmr(rres); 322 | } 323 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_mr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_MR_H 34 | #define NTRDMA_MR_H 35 | 36 | #include "ntc.h" 37 | 38 | #include 39 | 40 | #include "ntrdma_cmd.h" 41 | #include "ntrdma_res.h" 42 | #include "ntrdma_sg.h" 43 | 44 | struct ntrdma_dev; 45 | struct ntrdma_mr; 46 | 47 | struct ntrdma_mr_cmd_cb { 48 | struct ntrdma_cmd_cb cb; 49 | struct ntrdma_mr *mr; 50 | u32 sg_pos, sg_count; 51 | }; 52 | 53 | /* Memory Region */ 54 | struct ntrdma_mr { 55 | /* Ofed mr structure */ 56 | struct ib_mr ibmr; 57 | 58 | /* debugfs */ 59 | struct dentry *debug; 60 | 61 | /* Ntrdma resource bookkeeping structure */ 62 | struct ntrdma_res res; 63 | 64 | struct ib_umem *ib_umem; 65 | struct ntrdma_mr_cmd_cb enable_mrcb; 66 | struct completion *done; 67 | 68 | u32 pd_key; 69 | u32 access; 70 | 71 | u64 addr; 72 | u64 len; 73 | 74 | u32 sg_count; 75 | struct ntc_mr_buf sg_list[]; 76 | }; 77 | 78 | #define ntrdma_mr_dev(__mr) ntrdma_res_dev(&(__mr)->res) 79 | #define ntrdma_res_mr(__res) \ 80 | container_of(__res, struct ntrdma_mr, res) 81 | #define ntrdma_ib_mr(__ib) \ 82 | container_of(__ib, struct ntrdma_mr, ibmr) 83 | 84 | void ntrdma_mr_enable(struct ntrdma_mr *mr); 85 | 86 | /* Remote Memory Region */ 87 | struct ntrdma_rmr { 88 | /* debugfs */ 89 | struct dentry *debug; 90 | 91 | /* Ntrdma remote resource bookkeeping structure */ 92 | struct ntrdma_rres rres; 93 | struct completion *done; 94 | 95 | u32 pd_key; 96 | u32 access; 97 | 98 | u64 addr; 99 | u64 len; 100 | 101 | u32 sg_count; 102 | struct ntc_remote_buf sg_list[]; 103 | }; 104 | 105 | #define ntrdma_rmr_dev(__rmr) ntrdma_rres_dev(&(__rmr)->rres) 106 | #define ntrdma_rres_rmr(__rres) \ 107 | container_of(__rres, struct ntrdma_rmr, rres) 108 | 109 | int ntrdma_mr_init(struct ntrdma_mr *mr, struct ntrdma_dev *dev); 110 | void ntrdma_mr_deinit(struct ntrdma_mr *mr, struct ntrdma_dev *dev); 111 | 112 | static inline void ntrdma_mr_get(struct ntrdma_mr *mr) 113 | { 114 | ntrdma_res_get(&mr->res); 115 | } 116 | 117 | void ntrdma_mr_put(struct ntrdma_mr *mr); 118 | 119 | void ntrdma_rmr_init(struct ntrdma_rmr *rmr, 120 | struct ntrdma_dev *dev, 121 | u32 pd_key, u32 access, 122 | u64 addr, u64 len, 123 | u32 sg_count, u32 key); 124 | 125 | static inline int ntrdma_rmr_add(struct ntrdma_rmr *rmr) 126 | { 127 | ntrdma_debugfs_rmr_add(rmr); 128 | return ntrdma_rres_add(&rmr->rres); 129 | } 130 | 131 | static inline void ntrdma_rmr_get(struct ntrdma_rmr *rmr) 132 | { 133 | ntrdma_rres_get(&rmr->rres); 134 | } 135 | 136 | void ntrdma_rmr_put(struct ntrdma_rmr *rmr); 137 | 138 | struct ntrdma_mr *ntrdma_dev_mr_look(struct ntrdma_dev *dev, u32 key); 139 | struct ntrdma_rmr *ntrdma_dev_rmr_look(struct ntrdma_dev *dev, u32 key); 140 | 141 | 142 | /* from intel spec, canonical address is when bits 48-63 are identical and equal 143 | * to the 47th bit (starting count from 0). 144 | * When 0 its a user-space and 1 is for kernel space. 145 | * on aslr enabled systems, we can get 1's for user space addresses as well. 146 | */ 147 | #define MASK 0xffff800000000000 148 | static inline bool is_canonical(u64 addr) 149 | { 150 | return ((addr & MASK) == 0 || (addr & MASK) == MASK); 151 | } 152 | #endif 153 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_obj.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_OBJ_H 34 | #define NTRDMA_OBJ_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #include "ntrdma.h" 42 | 43 | struct ntrdma_obj { 44 | /* The ntrdma device to which this object belongs */ 45 | struct ntrdma_dev *dev; 46 | /* The entry in the device list for this type of object */ 47 | struct list_head dev_entry; 48 | struct kref kref; 49 | }; 50 | 51 | #define ntrdma_obj_dev(obj) ((obj)->dev) 52 | #define ntrdma_obj_dev_entry(obj) ((obj)->dev_entry) 53 | 54 | /* Claim a reference to the object */ 55 | static inline void ntrdma_obj_get(struct ntrdma_obj *obj) 56 | { 57 | int ret = kref_get_unless_zero(&obj->kref); 58 | 59 | WARN(!ret, "Should not happen ref is zero for obj %p", obj); 60 | } 61 | 62 | /* Relinquish a reference to the object */ 63 | static inline void ntrdma_obj_put(struct ntrdma_obj *obj, 64 | void (*obj_release)(struct kref *kref)) 65 | { 66 | kref_put(&obj->kref, obj_release); 67 | } 68 | 69 | static inline void ntrdma_obj_init(struct ntrdma_obj *obj, 70 | struct ntrdma_dev *dev) 71 | { 72 | obj->dev = dev; 73 | kref_init(&obj->kref); 74 | } 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_pd.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include 34 | 35 | #include "ntrdma_dev.h" 36 | #include "ntrdma_cmd.h" 37 | #include "ntrdma_res.h" 38 | #include "ntrdma_pd.h" 39 | 40 | void ntrdma_pd_init(struct ntrdma_pd *pd, struct ntrdma_dev *dev, u32 key) 41 | { 42 | pd->key = key; 43 | 44 | ntrdma_obj_init(&pd->obj, dev); 45 | } 46 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_pd.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_PD_H 34 | #define NTRDMA_PD_H 35 | 36 | #include "ntrdma_obj.h" 37 | 38 | /* Protection Domain */ 39 | struct ntrdma_pd { 40 | /* Ofed pd structure */ 41 | struct ib_pd ibpd; 42 | 43 | /* Ntrdma pd is a local-only object */ 44 | struct ntrdma_obj obj; /* member of dev->pd_list */ 45 | 46 | /* Protection domain key */ 47 | u32 key; 48 | }; 49 | 50 | #define ntrdma_pd_dev(__pd) ntrdma_obj_dev(&(__pd)->obj) 51 | #define ntrdma_ib_pd(__ib) \ 52 | container_of(__ib, struct ntrdma_pd, ibpd) 53 | 54 | void ntrdma_pd_init(struct ntrdma_pd *pd, struct ntrdma_dev *dev, u32 key); 55 | 56 | void ntrdma_pd_get(struct ntrdma_pd *pd); 57 | void ntrdma_pd_put(struct ntrdma_pd *pd); 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_res.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_RES_H 34 | #define NTRDMA_RES_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #include "ntrdma_obj.h" 42 | #include "ntrdma_dev.h" 43 | 44 | #define NTRDMA_RES_VEC_INIT_CAP 512 45 | #define NTRDMA_QP_VEC_PREALLOCATED 2 46 | #define NTRDMA_MR_VEC_PREALLOCATED 0 47 | 48 | struct ntrdma_dev; 49 | struct ntrdma_cmd_cb; 50 | struct ntrdma_vec; 51 | struct ntrdma_kvec; 52 | 53 | struct ntrdma_res *ntrdma_res_look(struct ntrdma_kvec *vec, u32 key); 54 | 55 | void ntrdma_dev_rres_reset(struct ntrdma_dev *dev); 56 | 57 | struct ntrdma_rres *ntrdma_rres_look(struct ntrdma_vec *vec, u32 key); 58 | 59 | /* Local rdma resource */ 60 | struct ntrdma_res { 61 | /* The resource is an ntrdma object */ 62 | struct ntrdma_obj obj; 63 | 64 | /* Initiate commands to create the remote resource */ 65 | int (*enable)(struct ntrdma_res *res, 66 | struct ntrdma_cmd_cb *cb); 67 | /* Initiate commands to delete the remote resource */ 68 | int (*disable)(struct ntrdma_res *res, 69 | struct ntrdma_cmd_cb *cb); 70 | 71 | /* The key identifies this resource */ 72 | u32 key; 73 | 74 | /* Synchronize operations affecting the local resource */ 75 | struct mutex lock; 76 | unsigned long timeout; 77 | }; 78 | 79 | #define ntrdma_res_dev(res) ntrdma_obj_dev(&(res)->obj) 80 | 81 | void ntrdma_res_init(struct ntrdma_res *res, 82 | struct ntrdma_dev *dev, 83 | int (*enable)(struct ntrdma_res *res, 84 | struct ntrdma_cmd_cb *cb), 85 | int (*disable)(struct ntrdma_res *res, 86 | struct ntrdma_cmd_cb *cb)); 87 | 88 | int ntrdma_res_add(struct ntrdma_res *res, struct ntrdma_cmd_cb *cb, 89 | struct list_head *res_list, struct ntrdma_kvec *res_vec); 90 | void ntrdma_res_del(struct ntrdma_res *res, struct ntrdma_cmd_cb *cb, 91 | struct ntrdma_kvec *res_vec); 92 | 93 | static inline void ntrdma_res_lock(struct ntrdma_res *res) 94 | { 95 | mutex_lock(&res->lock); 96 | } 97 | 98 | 99 | static inline void ntrdma_res_unlock(struct ntrdma_res *res) 100 | { 101 | mutex_unlock(&res->lock); 102 | } 103 | 104 | inline int ntrdma_res_wait_cmds(struct ntrdma_dev *dev, 105 | struct ntrdma_cmd_cb *cb, 106 | unsigned long timeout); 107 | 108 | static inline void ntrdma_res_get(struct ntrdma_res *res) 109 | { 110 | ntrdma_obj_get(&res->obj); 111 | } 112 | 113 | void ntrdma_res_put(struct ntrdma_res *res, 114 | void (*obj_release)(struct kref *kref)); 115 | 116 | /* Remote rdma resource */ 117 | struct ntrdma_rres { 118 | /* The resource is an ntrdma object */ 119 | struct ntrdma_obj obj; /* Entry protected by dev->rres_lock. */ 120 | 121 | bool in_rres_list; /* Protected by dev->rres_lock. */ 122 | 123 | /* The vector to which this remote resource is added */ 124 | struct ntrdma_vec *vec; 125 | 126 | /* Free this remote resource on reset */ 127 | void (*free)(struct ntrdma_rres *rres); 128 | 129 | /* The key identifies this remote resource */ 130 | u32 key; 131 | }; 132 | 133 | #define ntrdma_rres_dev(rres) ntrdma_obj_dev(&(rres)->obj) 134 | #define ntrdma_rres_dbg(__rres, __args...) \ 135 | ntrdma_dbg(ntrdma_rres_dev(__rres), ## __args) 136 | 137 | void ntrdma_rres_init(struct ntrdma_rres *rres, 138 | struct ntrdma_dev *dev, 139 | struct ntrdma_vec *vec, 140 | void (*free)(struct ntrdma_rres *rres), 141 | u32 key); 142 | 143 | int ntrdma_rres_add(struct ntrdma_rres *rres); 144 | void ntrdma_rres_remove(struct ntrdma_rres *rres); 145 | void ntrdma_rres_remove_unsafe(struct ntrdma_rres *rres); 146 | 147 | static inline void ntrdma_rres_get(struct ntrdma_rres *rres) 148 | { 149 | ntrdma_obj_get(&rres->obj); 150 | } 151 | 152 | static inline void ntrdma_rres_put(struct ntrdma_rres *rres, 153 | void (*release)(struct kref *)) 154 | { 155 | ntrdma_obj_put(&rres->obj, release); 156 | } 157 | 158 | static inline bool ntrdma_list_is_entry_poisoned(struct list_head *entry) 159 | { 160 | if (entry->next == LIST_POISON1 && entry->prev == LIST_POISON2) 161 | return true; 162 | return false; 163 | } 164 | 165 | #endif 166 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_ring.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_RING_H 34 | #define NTRDMA_RING_H 35 | 36 | #include "linux/types.h" 37 | 38 | /* Initialize a range of indices for production */ 39 | static inline bool ntrdma_ring_produce(u32 prod, u32 cons, u32 cap, 40 | u32 *prod_idx, u32 *prod_end, 41 | u32 *base) 42 | { 43 | if (prod < cap) { 44 | *base = 0; 45 | *prod_idx = prod; 46 | if (cons < cap) 47 | *prod_end = cap; 48 | else 49 | *prod_end = cons - cap; 50 | } else { 51 | *base = cap; 52 | *prod_idx = prod - cap; 53 | if (cons < cap) 54 | *prod_end = cons; 55 | else 56 | *prod_end = cap; 57 | } 58 | 59 | WARN((prod >= (u32)(cap << 1)) || 60 | (cons >= (u32)(cap << 1)), 61 | "prod %u and cons %u must be < cap %u", 62 | prod, cons, cap); 63 | WARN(((int)prod - (int)cons > (int)cap) || 64 | (((int)prod - (int)cons < 0) && ((int)cons - (int)prod < (int)cap)), 65 | "the distance between prod %u and cons %u must be < cap %u", 66 | prod, cons, cap); 67 | 68 | return *prod_idx != *prod_end; 69 | } 70 | 71 | /* Initialize a range of indices for consumption */ 72 | static inline void ntrdma_ring_consume(u32 prod, u32 cons, u32 cap, 73 | u32 *cons_idx, u32 *cons_end, 74 | u32 *base) 75 | { 76 | if (cons < cap) { 77 | *base = 0; 78 | *cons_idx = cons; 79 | if (prod < cap) 80 | *cons_end = prod; 81 | else 82 | *cons_end = cap; 83 | } else { 84 | *base = cap; 85 | *cons_idx = cons - cap; 86 | if (prod < cap) 87 | *cons_end = cap; 88 | else 89 | *cons_end = prod - cap; 90 | } 91 | } 92 | 93 | /* Compute a new index after producing or consuming */ 94 | static inline u32 ntrdma_ring_update(u32 idx, u32 base, u32 cap) 95 | { 96 | if (idx == cap) 97 | return cap - base; 98 | 99 | return base + idx; 100 | } 101 | 102 | /* Calculate the number of entries produced */ 103 | static inline u32 ntrdma_ring_count(u32 prod, u32 cons, u32 cap) 104 | { 105 | if (prod < cons) 106 | prod += cap << 1; 107 | 108 | return prod - cons; 109 | } 110 | 111 | /* Calculate the number of entries available */ 112 | static inline u32 ntrdma_ring_space(u32 prod, u32 cons, u32 cap) 113 | { 114 | if (prod < cons) 115 | prod += cap << 1; 116 | 117 | return cap + cons - prod; 118 | } 119 | 120 | /* Calculate the number of contiguous entries produced */ 121 | static inline u32 ntrdma_ring_count_ctg(u32 prod, u32 cons, u32 cap) 122 | { 123 | int idx, end, base; 124 | 125 | ntrdma_ring_consume(prod, cons, cap, &idx, &end, &base); 126 | 127 | return end - idx; 128 | } 129 | 130 | /* Calculate the number of contiguous entries available */ 131 | static inline u32 ntrdma_ring_space_ctg(u32 prod, u32 cons, u32 cap) 132 | { 133 | int idx, end, base; 134 | 135 | ntrdma_ring_produce(prod, cons, cap, &idx, &end, &base); 136 | 137 | return end - idx; 138 | } 139 | 140 | /* The ring is valid if prod is no more than cap entries ahead of cons */ 141 | static inline bool ntrdma_ring_valid(u32 prod, u32 cons, u32 cap) 142 | { 143 | if ((cap << 1) <= prod) 144 | return 0; 145 | 146 | if ((cap << 1) <= cons) 147 | return 0; 148 | 149 | return ntrdma_ring_count(prod, cons, cap) <= cap; 150 | } 151 | 152 | #endif 153 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_sg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_SG_H 34 | #define NTRDMA_SG_H 35 | 36 | #include "ntc.h" 37 | #include 38 | #include 39 | 40 | #include "ntrdma.h" 41 | 42 | struct ntrdma_wr_rcv_sge_shadow { 43 | u32 local_key; 44 | union { 45 | /* local_key != NTRDMA_RESERVED_DMA_LEKY */ 46 | u64 local_addr; 47 | /* local_key == NTRDMA_RESERVED_DMA_LEKY */ 48 | struct ntc_local_buf rcv_dma_buf; 49 | }; 50 | struct ntc_export_buf exp_buf; 51 | }; 52 | 53 | struct ntrdma_wr_rcv_sge { 54 | struct ntrdma_wr_rcv_sge_shadow *shadow; 55 | union { 56 | /* shadow == NULL */ 57 | /* sge.lkey != NTRDMA_RESERVED_DMA_LEKY */ 58 | struct ib_sge sge; 59 | /* shadow != NULL: the desc of shadow->exp_buf */ 60 | struct ntc_remote_buf_desc exp_buf_desc; 61 | }; 62 | }; 63 | 64 | static inline bool ntrdma_ib_sge_reserved(const struct ib_sge *sge) 65 | { 66 | return sge->lkey == NTRDMA_RESERVED_DMA_LEKY; 67 | } 68 | 69 | static inline u64 ntrdma_wr_rcv_sge_len(const struct ntrdma_wr_rcv_sge *sge) 70 | { 71 | if (!sge->shadow) 72 | return sge->sge.length; 73 | else 74 | return sge->exp_buf_desc.size; 75 | } 76 | 77 | static inline 78 | u32 ntrdma_wr_rcv_sge_remote_key(const struct ntrdma_wr_rcv_sge *sge) 79 | { 80 | if (!sge->shadow) 81 | return sge->sge.lkey; 82 | else 83 | return NTRDMA_RESERVED_DMA_LEKY; 84 | } 85 | 86 | struct ntrdma_wr_rcv_sge_shadow * 87 | ntrdma_zalloc_sge_shadow(gfp_t gfp, struct ntrdma_dev *dev); 88 | void ntrdma_free_sge_shadow(struct ntrdma_wr_rcv_sge_shadow *shadow); 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_util.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include 34 | 35 | #include "ntrdma_util.h" 36 | 37 | #define BITS_TO_LONGS_SIZE(bits) \ 38 | (BITS_TO_LONGS(bits) * sizeof(long)) 39 | 40 | #define MAX_VEC_CAP 0x800000U 41 | #define MAX_KVEC_CAP 0x800000U 42 | 43 | int ntrdma_vec_init(struct ntrdma_vec *vec, u32 cap, int node) 44 | { 45 | struct ntrdma_rcu_vec *rvec; 46 | 47 | if (cap == 0) 48 | return -EINVAL; 49 | 50 | if (cap >= MAX_VEC_CAP) 51 | return -ENOMEM; 52 | rvec = kzalloc_node(sizeof(*vec->rvec), GFP_KERNEL, node); 53 | if (!rvec) { 54 | pr_err("Failed to alloc rvec, size %zd\n", sizeof(*vec->rvec)); 55 | vec->rvec = NULL; 56 | return -ENOMEM; 57 | } 58 | 59 | rvec->look = kzalloc_node(cap * sizeof(*vec->rvec->look), 60 | GFP_KERNEL, node); 61 | if (!rvec->look) { 62 | pr_err("Failed to alloc rvec->look, cap %d, size %zd\n", cap, sizeof(*vec->rvec->look)); 63 | kfree(rvec); 64 | vec->rvec = NULL; 65 | return -ENOMEM; 66 | } 67 | 68 | rvec->cap = cap; 69 | rcu_assign_pointer(vec->rvec, rvec); 70 | 71 | mutex_init(&vec->lock); 72 | 73 | return 0; 74 | } 75 | 76 | void ntrdma_vec_deinit(struct ntrdma_vec *vec) 77 | { 78 | rcu_barrier(); 79 | kfree(vec->rvec->look); 80 | kfree(vec->rvec); 81 | } 82 | 83 | int ntrdma_vec_copy_assign(struct ntrdma_vec *vec, u32 cap, int node, u32 key, 84 | void *val) 85 | { 86 | struct ntrdma_rcu_vec *rvec; 87 | 88 | if (cap == 0) 89 | return -EINVAL; 90 | 91 | if (cap >= MAX_VEC_CAP) 92 | return -ENOMEM; 93 | 94 | rvec = kzalloc_node(sizeof(*vec->rvec), GFP_KERNEL, node); 95 | if (!rvec) { 96 | return -ENOMEM; 97 | } 98 | 99 | rvec->look = kzalloc_node(cap * sizeof(*rvec->look), GFP_KERNEL, node); 100 | 101 | if (!rvec->look) { 102 | kfree(rvec); 103 | return -ENOMEM; 104 | } 105 | 106 | memcpy(rvec->look, vec->rvec->look, 107 | vec->rvec->cap * sizeof(*vec->rvec->look)); 108 | rvec->look[key] = val; 109 | rvec->cap = cap; 110 | 111 | rcu_assign_pointer(vec->rvec, rvec); 112 | 113 | return 0; 114 | } 115 | 116 | int ntrdma_kvec_init(struct ntrdma_kvec *vec, u32 cap, u32 num_reserved_keys, 117 | int node) 118 | { 119 | struct ntrdma_rcu_kvec *rkvec; 120 | u32 i; 121 | 122 | if (cap == 0) 123 | return -EINVAL; 124 | 125 | if (cap <= num_reserved_keys) 126 | return -EINVAL; 127 | 128 | if (cap >= MAX_KVEC_CAP) 129 | return -ENOMEM; 130 | 131 | rkvec = kzalloc_node(sizeof(*vec->rkvec), GFP_KERNEL, node); 132 | if (!rkvec) { 133 | pr_err("Failed to alloc rkvec, size %zd\n", sizeof(*vec->rkvec)); 134 | vec->rkvec = NULL; 135 | return -ENOMEM; 136 | } 137 | 138 | rkvec->keys = kzalloc_node(BITS_TO_LONGS_SIZE(cap), GFP_KERNEL, node); 139 | if (!rkvec->keys) { 140 | pr_err("Failed to alloc keys, size %zd\n", BITS_TO_LONGS_SIZE(cap)); 141 | kfree(rkvec); 142 | vec->rkvec = NULL; 143 | return -ENOMEM; 144 | } 145 | 146 | rkvec->look = kzalloc_node(cap * sizeof(*rkvec->look), GFP_KERNEL, node); 147 | if (!rkvec->look) { 148 | pr_err("Failed to alloc look, size %zd\n", cap * sizeof(*rkvec->look)); 149 | kfree(rkvec->keys); 150 | kfree(rkvec); 151 | vec->rkvec = NULL; 152 | return -ENOMEM; 153 | } 154 | 155 | for (i = 0; i < num_reserved_keys; i++) 156 | __set_bit(i, rkvec->keys); 157 | 158 | rkvec->cap = cap; 159 | rkvec->num_reserved_keys = num_reserved_keys; 160 | rkvec->next_key = num_reserved_keys; 161 | 162 | rcu_assign_pointer(vec->rkvec, rkvec); 163 | mutex_init(&vec->lock); 164 | 165 | return 0; 166 | } 167 | 168 | void ntrdma_kvec_deinit(struct ntrdma_kvec *vec) 169 | { 170 | rcu_barrier(); 171 | kfree(vec->rkvec->look); 172 | kfree(vec->rkvec->keys); 173 | kfree(vec->rkvec); 174 | } 175 | 176 | int ntrdma_kvec_new_copy(struct ntrdma_kvec *vec, u32 cap, 177 | int node, struct ntrdma_rcu_kvec **ret_vec) 178 | { 179 | struct ntrdma_rcu_kvec *rkvec; 180 | 181 | if (cap == 0) 182 | return -EINVAL; 183 | 184 | if (cap >= MAX_KVEC_CAP) 185 | return -ENOMEM; 186 | 187 | rkvec = kzalloc_node(sizeof(*vec->rkvec), GFP_KERNEL, node); 188 | if (!rkvec) { 189 | pr_err("Failed to alloc rkvec, size %zd\n", sizeof(*vec->rkvec)); 190 | return -ENOMEM; 191 | } 192 | 193 | rkvec->keys = kzalloc_node(BITS_TO_LONGS_SIZE(cap), GFP_KERNEL, node); 194 | if (!rkvec->keys) { 195 | pr_err("Failed to alloc keys, size %zd\n", BITS_TO_LONGS_SIZE(cap)); 196 | kfree(rkvec); 197 | return -ENOMEM; 198 | } 199 | 200 | rkvec->look = kzalloc_node(cap * sizeof(*rkvec->look), GFP_KERNEL, node); 201 | if (!rkvec->look) { 202 | pr_err("Failed to alloc look, size %zd\n", cap * sizeof(*rkvec->look)); 203 | kfree(rkvec->keys); 204 | kfree(rkvec); 205 | return -ENOMEM; 206 | } 207 | 208 | memcpy(rkvec->keys, vec->rkvec->keys, BITS_TO_LONGS_SIZE(vec->rkvec->cap)); 209 | memcpy(rkvec->look, vec->rkvec->look, vec->rkvec->cap * sizeof(*rkvec->look)); 210 | rkvec->cap = cap; 211 | rkvec->next_key = vec->rkvec->next_key; 212 | rkvec->num_reserved_keys = vec->rkvec->num_reserved_keys; 213 | 214 | *ret_vec = rkvec; 215 | return 0; 216 | } 217 | 218 | int ntrdma_kvec_reserve_key(struct ntrdma_kvec *vec, int node) 219 | { 220 | u32 key; 221 | u32 cap; 222 | int rc; 223 | struct ntrdma_rcu_kvec *old_rkvec; 224 | struct ntrdma_rcu_kvec *new_rkvec; 225 | 226 | mutex_lock(&vec->lock); 227 | 228 | old_rkvec = vec->rkvec; 229 | key = find_next_zero_bit(old_rkvec->keys, old_rkvec->cap, 230 | old_rkvec->next_key); 231 | cap = old_rkvec->cap; 232 | if (key >= old_rkvec->cap) { 233 | cap = key << 1; 234 | } 235 | 236 | rc = ntrdma_kvec_new_copy(vec, cap, node, &new_rkvec); 237 | if (rc < 0) { 238 | key = rc; 239 | goto out; 240 | } 241 | 242 | __set_bit(key, new_rkvec->keys); 243 | new_rkvec->next_key = (key + 1 == new_rkvec->cap) ? new_rkvec->num_reserved_keys : key + 1; 244 | 245 | rcu_assign_pointer(vec->rkvec, new_rkvec); 246 | call_rcu(&old_rkvec->rcu, remove_rcu_kvec); 247 | out: 248 | mutex_unlock(&vec->lock); 249 | 250 | return key; 251 | } 252 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_UTIL_H 34 | #define NTRDMA_UTIL_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | /* Resizable vector */ 43 | struct ntrdma_rcu_vec { 44 | u32 cap; 45 | void **look; 46 | struct rcu_head rcu; 47 | }; 48 | 49 | struct ntrdma_vec { 50 | struct ntrdma_rcu_vec *rvec; 51 | struct mutex lock; 52 | }; 53 | 54 | /* Allocate an empty vector with a capacity */ 55 | int ntrdma_vec_init(struct ntrdma_vec *vec, u32 cap, int node); 56 | /* Destroy a vector */ 57 | void ntrdma_vec_deinit(struct ntrdma_vec *vec); 58 | 59 | int ntrdma_vec_copy_assign(struct ntrdma_vec *vec, u32 cap, int node, u32 key, 60 | void *val); 61 | static void remove_rcu_vec(struct rcu_head *rhp) 62 | { 63 | struct ntrdma_rcu_vec *p = container_of(rhp, struct ntrdma_rcu_vec, rcu); 64 | 65 | kfree(p->look); 66 | kfree(p); 67 | } 68 | static inline int ntrdma_vec_set(struct ntrdma_vec *vec, u32 key, void *value, 69 | int node) 70 | { 71 | int rc; 72 | struct ntrdma_rcu_vec *rvec; 73 | u32 new_cap; 74 | 75 | mutex_lock(&vec->lock); 76 | 77 | new_cap = vec->rvec->cap; 78 | if (key >= vec->rvec->cap) 79 | new_cap = roundup_pow_of_two(key + 1); 80 | rvec = vec->rvec; 81 | rc = ntrdma_vec_copy_assign(vec, new_cap, node, key, value); 82 | if (rc < 0) { 83 | mutex_unlock(&vec->lock); 84 | pr_err( 85 | "rc %d for ntrdma_vec_copy_assign, key %d, vec %p, val %p node %d, cap %d", 86 | rc, key, vec, value, node, new_cap); 87 | return rc; 88 | } 89 | 90 | call_rcu(&rvec->rcu, remove_rcu_vec); 91 | mutex_unlock(&vec->lock); 92 | 93 | return 0; 94 | } 95 | 96 | /* Resizable vector with key reservation */ 97 | struct ntrdma_rcu_kvec { 98 | /* Capacity of the vec */ 99 | u32 cap; 100 | /* Preallocated and never deallocated */ 101 | u32 num_reserved_keys; 102 | /* Next key to check when reserving */ 103 | u32 next_key; 104 | /* Bitset of available/used keys, cap bits long */ 105 | unsigned long *keys; 106 | /* Key indexed lookup of elements, cap elements long */ 107 | void **look; 108 | struct rcu_head rcu; 109 | }; 110 | 111 | struct ntrdma_kvec { 112 | struct ntrdma_rcu_kvec *rkvec; 113 | struct mutex lock; 114 | }; 115 | 116 | /* Allocate an empty vector with a capacity */ 117 | int ntrdma_kvec_init(struct ntrdma_kvec *vec, u32 cap, u32 num_reserved_keys, 118 | int node); 119 | /* Destroy a vector */ 120 | void ntrdma_kvec_deinit(struct ntrdma_kvec *vec); 121 | /* Reserve the next available key */ 122 | int ntrdma_kvec_reserve_key(struct ntrdma_kvec *vec, int node); 123 | int ntrdma_kvec_new_copy(struct ntrdma_kvec *vec, u32 cap, 124 | int node, struct ntrdma_rcu_kvec **ret_vac); 125 | static void remove_rcu_kvec(struct rcu_head *rhp) 126 | { 127 | struct ntrdma_rcu_kvec *p = container_of(rhp, struct ntrdma_rcu_kvec, rcu); 128 | 129 | kfree(p->look); 130 | kfree(p); 131 | } 132 | 133 | /* Dispose a key that no longer needs to be reserved */ 134 | static inline void ntrdma_kvec_dispose_key(int node, 135 | struct ntrdma_kvec *vec, u32 key) 136 | { 137 | struct ntrdma_rcu_kvec *old_rkvec; 138 | struct ntrdma_rcu_kvec *new_rkvec; 139 | int rc; 140 | 141 | mutex_lock(&vec->lock); 142 | 143 | old_rkvec = vec->rkvec; 144 | if (key < old_rkvec->num_reserved_keys) 145 | goto out; 146 | 147 | rc = ntrdma_kvec_new_copy(vec, old_rkvec->cap, node, &new_rkvec); 148 | if (rc < 0) 149 | goto out; 150 | 151 | 152 | __clear_bit(key, new_rkvec->keys); 153 | rcu_assign_pointer(vec->rkvec, new_rkvec); 154 | call_rcu(&old_rkvec->rcu, remove_rcu_kvec); 155 | out: 156 | mutex_unlock(&vec->lock); 157 | } 158 | 159 | static inline 160 | void ntrdma_kvec_set_key(int node, struct ntrdma_kvec *vec, 161 | u32 key, void *value) 162 | { 163 | struct ntrdma_rcu_kvec *old_rkvec; 164 | struct ntrdma_rcu_kvec *new_rkvec; 165 | int rc; 166 | 167 | mutex_lock(&vec->lock); 168 | 169 | old_rkvec = vec->rkvec; 170 | rc = ntrdma_kvec_new_copy(vec, old_rkvec->cap, node, &new_rkvec); 171 | if (rc < 0) 172 | goto out; 173 | 174 | new_rkvec->look[key] = value; 175 | 176 | rcu_assign_pointer(vec->rkvec, new_rkvec); 177 | call_rcu(&old_rkvec->rcu, remove_rcu_kvec); 178 | out: 179 | mutex_unlock(&vec->lock); 180 | } 181 | 182 | static inline void ntrdma_deinit_slab(struct kmem_cache **pslab) 183 | { 184 | struct kmem_cache *slab = *pslab; 185 | 186 | if (!slab) 187 | return; 188 | 189 | *pslab = NULL; 190 | 191 | kmem_cache_destroy(slab); 192 | } 193 | 194 | #endif 195 | 196 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_vbell.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include 34 | 35 | #include "ntrdma_dev.h" 36 | #include "ntrdma_vbell.h" 37 | 38 | static void ntrdma_dev_vbell_work_cb(unsigned long ptrhld); 39 | 40 | static struct workqueue_struct *ntrdma_workq; 41 | 42 | int ntrdma_dev_vbell_init(struct ntrdma_dev *dev, 43 | u32 vbell_count, u32 vbell_start) 44 | { 45 | int rc, i; 46 | 47 | dev->vbell.enable = 0; 48 | spin_lock_init(&dev->vbell.lock); 49 | 50 | for (i = 0; i < NTB_MAX_IRQS; i++) { 51 | dev->vbell.work_data[i].dev = dev; 52 | dev->vbell.work_data[i].vec = i; 53 | tasklet_init(&dev->vbell.work[i], 54 | ntrdma_dev_vbell_work_cb, 55 | to_ptrhld(&dev->vbell.work_data[i])); 56 | } 57 | 58 | dev->vbell.count = vbell_count; 59 | dev->vbell.start = vbell_start; 60 | atomic_set(&dev->vbell.next, 0); 61 | 62 | dev->vbell.vec = kmalloc_node(vbell_count * sizeof(*dev->vbell.vec), 63 | GFP_KERNEL, dev->node); 64 | if (!dev->vbell.vec) { 65 | ntrdma_err(dev, "failed to alloc vbell vec"); 66 | rc = -ENOMEM; 67 | goto err_vec; 68 | } 69 | 70 | rc = ntc_export_buf_zalloc(&dev->vbell.buf, dev->ntc, 71 | vbell_count * sizeof(u32), 72 | GFP_KERNEL); 73 | if (rc < 0) { 74 | ntrdma_err(dev, "failed to alloc vbell buffer"); 75 | goto err_buf; 76 | } 77 | 78 | for (i = 0; i < vbell_count; ++i) 79 | ntrdma_vbell_head_init(&dev->vbell.vec[i]); 80 | 81 | for (i = 0; i < NTRDMA_DEV_VBELL_COUNT; ++i) 82 | ntrdma_peer_vbell_init(&dev->vbell.peer[i]); 83 | 84 | dev->vbell.peer_count = 0; 85 | 86 | return 0; 87 | 88 | err_buf: 89 | kfree(dev->vbell.vec); 90 | err_vec: 91 | for (i = 0; i < NTB_MAX_IRQS; i++) 92 | tasklet_kill(&dev->vbell.work[i]); 93 | return rc; 94 | } 95 | 96 | void ntrdma_dev_vbell_deinit(struct ntrdma_dev *dev) 97 | { 98 | int i; 99 | 100 | for (i = 0; i < NTB_MAX_IRQS; i++) 101 | tasklet_kill(&dev->vbell.work[i]); 102 | 103 | ntc_export_buf_free(&dev->vbell.buf); 104 | kfree(dev->vbell.vec); 105 | } 106 | 107 | u32 ntrdma_dev_vbell_next(struct ntrdma_dev *dev) 108 | { 109 | u32 idx = atomic_inc_return(&dev->vbell.next) - 1; 110 | 111 | return dev->vbell.start + (idx % (dev->vbell.count - dev->vbell.start)); 112 | } 113 | 114 | int ntrdma_dev_vbell_enable(struct ntrdma_dev *dev, 115 | const struct ntc_remote_buf_desc *vbell_ntc_buf_desc, 116 | u32 peer_vbell_count) 117 | { 118 | int rc; 119 | struct ntc_remote_buf peer_vbell_buf; 120 | int i; 121 | 122 | if (unlikely(dev->vbell.enable)) { 123 | spin_lock_bh(&dev->vbell.lock); 124 | if (likely(dev->vbell.enable)) { 125 | spin_unlock_bh(&dev->vbell.lock); 126 | return 0; 127 | } 128 | spin_unlock_bh(&dev->vbell.lock); 129 | } 130 | 131 | rc = ntc_remote_buf_map(&peer_vbell_buf, dev->ntc, 132 | vbell_ntc_buf_desc); 133 | if (rc < 0) { 134 | ntrdma_err(dev, "Failed to map peer vbell buff"); 135 | return rc; 136 | } 137 | 138 | spin_lock_bh(&dev->vbell.lock); 139 | 140 | if (unlikely(dev->vbell.enable)) { 141 | ntc_remote_buf_unmap(&peer_vbell_buf, dev->ntc); 142 | spin_unlock_bh(&dev->vbell.lock); 143 | return 0; 144 | } 145 | 146 | dev->vbell.enable = 1; 147 | 148 | for (i = 0; i < peer_vbell_count; ++i) 149 | ntrdma_peer_vbell_enable(&dev->vbell.peer[i]); 150 | 151 | dev->vbell.peer_buf = peer_vbell_buf; 152 | dev->vbell.peer_count = peer_vbell_count; 153 | 154 | for (i = 0; i < dev->vbell.count; ++i) 155 | ntrdma_vbell_head_enable(&dev->vbell.vec[i]); 156 | 157 | spin_unlock_bh(&dev->vbell.lock); 158 | 159 | return 0; 160 | } 161 | 162 | void ntrdma_dev_vbell_disable(struct ntrdma_dev *dev) 163 | { 164 | int i; 165 | 166 | spin_lock_bh(&dev->vbell.lock); 167 | 168 | if (unlikely(!dev->vbell.enable)) { 169 | spin_unlock_bh(&dev->vbell.lock); 170 | return; 171 | } 172 | 173 | dev->vbell.enable = 0; 174 | 175 | for (i = 0; i < dev->vbell.count; ++i) 176 | ntrdma_vbell_head_disable(&dev->vbell.vec[i]); 177 | 178 | for (i = 0; i < dev->vbell.peer_count; ++i) 179 | ntrdma_peer_vbell_disable(&dev->vbell.peer[i]); 180 | 181 | ntc_remote_buf_unmap(&dev->vbell.peer_buf, dev->ntc); 182 | 183 | ntc_export_buf_reinit_by_zeroes(&dev->vbell.buf, 0, 184 | dev->vbell.count * sizeof(u32)); 185 | 186 | dev->vbell.peer_count = 0; 187 | 188 | spin_unlock_bh(&dev->vbell.lock); 189 | } 190 | 191 | void ntrdma_dev_vbell_reset(struct ntrdma_dev *dev) 192 | { 193 | ntrdma_dbg(dev, "vbell reset not implemented\n"); 194 | } 195 | 196 | static void ntrdma_dev_vbell_work(struct ntrdma_dev *dev, int vec) 197 | { 198 | struct ntrdma_vbell_head *head; 199 | const u32 *vbell_buf; 200 | u32 vbell_val; 201 | int i; 202 | 203 | 204 | vbell_buf = ntc_export_buf_const_deref(&dev->vbell.buf, 205 | 0, dev->vbell.count * sizeof(u32)); 206 | for (i = 0; i < dev->vbell.count; ++i) { 207 | head = &dev->vbell.vec[i]; 208 | vbell_val = READ_ONCE(vbell_buf[i]); 209 | if (READ_ONCE(head->seq) == vbell_val) 210 | continue; 211 | ntrdma_vbell_head_fire(head, vbell_val); 212 | } 213 | 214 | if (ntc_clear_signal(dev->ntc, vec)) 215 | tasklet_schedule(&dev->vbell.work[vec]); 216 | } 217 | 218 | static void ntrdma_dev_vbell_work_cb(unsigned long ptrhld) 219 | { 220 | struct vbell_work_data_s *vbell_work_data = of_ptrhld(ptrhld); 221 | 222 | ntrdma_dev_vbell_work(vbell_work_data->dev, vbell_work_data->vec); 223 | } 224 | 225 | int ntrdma_dev_vbell_peer(struct ntrdma_dev *dev, 226 | struct ntc_dma_chan *chan, u32 idx) 227 | { 228 | struct ntrdma_peer_vbell *peer_vbell = &dev->vbell.peer[idx]; 229 | int rc = 0; 230 | 231 | TRACE_VDBG("vbell peer idx %d\n", idx); 232 | 233 | spin_lock_bh(&peer_vbell->lock); 234 | 235 | if (unlikely(!peer_vbell->enabled)) { 236 | ntrdma_err(dev, "peer vbell disabled"); 237 | rc = -EINVAL; 238 | goto exit_unlock; 239 | } 240 | 241 | rc = ntc_request_imm32(chan, 242 | &dev->vbell.peer_buf, idx * sizeof(u32), 243 | ++peer_vbell->seq, 244 | true, NULL, NULL); 245 | 246 | if (unlikely(rc < 0)) 247 | ntrdma_err(dev, "ntc_request_imm32 failed. rc=%d", rc); 248 | 249 | exit_unlock: 250 | spin_unlock_bh(&peer_vbell->lock); 251 | 252 | return rc; 253 | } 254 | 255 | int ntrdma_dev_vbell_peer_direct(struct ntrdma_dev *dev, u32 idx) 256 | { 257 | struct ntrdma_peer_vbell *peer_vbell = &dev->vbell.peer[idx]; 258 | int rc = 0; 259 | 260 | TRACE_DEBUG("vbell peer idx %d\n", idx); 261 | 262 | spin_lock_bh(&peer_vbell->lock); 263 | 264 | if (unlikely(!peer_vbell->enabled)) { 265 | ntrdma_err(dev, "peer vbell disabled"); 266 | rc = -EINVAL; 267 | goto exit_unlock; 268 | } 269 | 270 | rc = ntc_imm32(&dev->vbell.peer_buf, idx * sizeof(u32), 271 | ++peer_vbell->seq); 272 | 273 | if (unlikely(rc < 0)) 274 | ntrdma_err(dev, "ntc_imm32 failed. rc=%d", rc); 275 | 276 | exit_unlock: 277 | spin_unlock_bh(&peer_vbell->lock); 278 | 279 | return rc; 280 | } 281 | 282 | static void vbell_tasklet_cb(void *cb_ctx) 283 | { 284 | struct tasklet_struct *tasklet = cb_ctx; 285 | 286 | tasklet_schedule(tasklet); 287 | } 288 | 289 | void ntrdma_tasklet_vbell_init(struct ntrdma_dev *dev, 290 | struct ntrdma_vbell *vbell, u32 idx, 291 | struct tasklet_struct *tasklet) 292 | { 293 | ntrdma_vbell_init(dev, vbell, idx, vbell_tasklet_cb, tasklet); 294 | } 295 | 296 | static void vbell_work_cb(void *cb_ctx) 297 | { 298 | struct work_struct *work = cb_ctx; 299 | 300 | queue_work(ntrdma_workq, work); 301 | } 302 | 303 | void ntrdma_work_vbell_init(struct ntrdma_dev *dev, 304 | struct ntrdma_vbell *vbell, u32 idx, 305 | struct work_struct *work) 306 | { 307 | ntrdma_vbell_init(dev, vbell, idx, vbell_work_cb, work); 308 | } 309 | 310 | static void vbell_napi_cb(void *cb_ctx) 311 | { 312 | struct napi_struct *napi = cb_ctx; 313 | 314 | napi_schedule(napi); 315 | } 316 | 317 | void ntrdma_napi_vbell_init(struct ntrdma_dev *dev, 318 | struct ntrdma_vbell *vbell, u32 idx, 319 | struct napi_struct *napi) 320 | { 321 | ntrdma_vbell_init(dev, vbell, idx, vbell_napi_cb, napi); 322 | } 323 | 324 | void ntrdma_tasklet_vbell_kill(struct ntrdma_vbell *vbell) 325 | { 326 | struct tasklet_struct *tasklet = vbell->cb_ctx; 327 | 328 | ntrdma_vbell_del(vbell); 329 | tasklet_kill(tasklet); 330 | } 331 | 332 | void ntrdma_work_vbell_flush(struct ntrdma_vbell *vbell) 333 | { 334 | struct work_struct *work = vbell->cb_ctx; 335 | 336 | ntrdma_vbell_disable(vbell); 337 | flush_work(work); 338 | flush_work(work); 339 | } 340 | 341 | void ntrdma_work_vbell_kill(struct ntrdma_vbell *vbell) 342 | { 343 | struct work_struct *work = vbell->cb_ctx; 344 | 345 | ntrdma_vbell_del(vbell); 346 | flush_work(work); 347 | flush_work(work); 348 | } 349 | 350 | void ntrdma_napi_vbell_kill(struct ntrdma_vbell *vbell) 351 | { 352 | struct napi_struct *napi = vbell->cb_ctx; 353 | 354 | ntrdma_vbell_del(vbell); 355 | netif_napi_del(napi); 356 | } 357 | 358 | int __init ntrdma_vbell_module_init(void) 359 | { 360 | ntrdma_workq = 361 | alloc_workqueue("ntrdma-vbell", WQ_UNBOUND | WQ_MEM_RECLAIM | 362 | WQ_SYSFS, 0); 363 | if (!ntrdma_workq) { 364 | pr_err("%s failed to alloc work queue\n", __func__); 365 | return -ENOMEM; 366 | } 367 | 368 | return 0; 369 | } 370 | 371 | void ntrdma_vbell_module_deinit(void) 372 | { 373 | if (ntrdma_workq) { 374 | destroy_workqueue(ntrdma_workq); 375 | ntrdma_workq = NULL; 376 | } 377 | } 378 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_vbell.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_VBELL_H 34 | #define NTRDMA_VBELL_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | 43 | struct ntrdma_dev; 44 | 45 | struct ntrdma_vbell { 46 | struct ntrdma_dev *dev; 47 | u32 idx; /* Index in the device. */ 48 | struct list_head entry; /* Protected by head->lock. */ 49 | void (*cb_fn)(void *cb_ctx); /* Called under head->lock. */ 50 | void *cb_ctx; 51 | u32 seq; /* Protected by head->lock. */ 52 | bool arm; /* Protected by head->lock. */ 53 | bool enabled;/* Protected by head->lock. */ 54 | bool alive; /* Protected by head->lock. */ 55 | }; 56 | 57 | struct ntrdma_vbell_head { 58 | spinlock_t lock; 59 | struct list_head list; /* Protected by lock. */ 60 | u32 seq; /* Write protected by lock. */ 61 | bool enabled;/* Protected by lock. */ 62 | }; 63 | 64 | struct ntrdma_peer_vbell { 65 | spinlock_t lock; 66 | u32 seq; /* Protected by lock. */ 67 | bool enabled;/* Protected by lock. */ 68 | }; 69 | 70 | int ntrdma_dev_vbell_init(struct ntrdma_dev *dev, 71 | u32 vbell_count, 72 | u32 vbell_start); 73 | 74 | void ntrdma_dev_vbell_deinit(struct ntrdma_dev *dev); 75 | 76 | u32 ntrdma_dev_vbell_next(struct ntrdma_dev *dev); 77 | 78 | int ntrdma_dev_vbell_enable(struct ntrdma_dev *dev, 79 | const struct ntc_remote_buf_desc *vbell_ntc_buf_desc, 80 | u32 peer_vbell_count); 81 | 82 | void ntrdma_dev_vbell_disable(struct ntrdma_dev *dev); 83 | 84 | int ntrdma_dev_vbell_peer(struct ntrdma_dev *dev, 85 | struct ntc_dma_chan *chan, u32 idx); 86 | int ntrdma_dev_vbell_peer_direct(struct ntrdma_dev *dev, u32 idx); 87 | 88 | static inline void ntrdma_vbell_init(struct ntrdma_dev *dev, 89 | struct ntrdma_vbell *vbell, u32 idx, 90 | void (*cb_fn)(void *cb_ctx), void *cb_ctx) 91 | { 92 | vbell->dev = dev; 93 | vbell->idx = idx; 94 | vbell->cb_fn = cb_fn; 95 | vbell->cb_ctx = cb_ctx; 96 | vbell->seq = 0; 97 | vbell->arm = false; 98 | vbell->enabled = true; 99 | vbell->alive = true; 100 | } 101 | 102 | void ntrdma_tasklet_vbell_init(struct ntrdma_dev *dev, 103 | struct ntrdma_vbell *vbell, u32 idx, 104 | struct tasklet_struct *tasklet); 105 | 106 | void ntrdma_work_vbell_init(struct ntrdma_dev *dev, 107 | struct ntrdma_vbell *vbell, u32 idx, 108 | struct work_struct *work); 109 | 110 | void ntrdma_napi_vbell_init(struct ntrdma_dev *dev, 111 | struct ntrdma_vbell *vbell, u32 idx, 112 | struct napi_struct *napi); 113 | 114 | void ntrdma_tasklet_vbell_kill(struct ntrdma_vbell *vbell); 115 | void ntrdma_work_vbell_flush(struct ntrdma_vbell *vbell); 116 | void ntrdma_work_vbell_kill(struct ntrdma_vbell *vbell); 117 | void ntrdma_napi_vbell_kill(struct ntrdma_vbell *vbell); 118 | 119 | static inline void ntrdma_vbell_head_init(struct ntrdma_vbell_head *head) 120 | { 121 | INIT_LIST_HEAD(&head->list); 122 | head->seq = 0; 123 | spin_lock_init(&head->lock); 124 | head->enabled = 0; 125 | } 126 | 127 | static inline void ntrdma_peer_vbell_init(struct ntrdma_peer_vbell *peer_vbell) 128 | { 129 | peer_vbell->seq = 0; 130 | spin_lock_init(&peer_vbell->lock); 131 | peer_vbell->enabled = 0; 132 | } 133 | 134 | static inline void ntrdma_peer_vbell_enable(struct ntrdma_peer_vbell *peer_vbell) 135 | { 136 | spin_lock_bh(&peer_vbell->lock); 137 | peer_vbell->enabled = 1; 138 | spin_unlock_bh(&peer_vbell->lock); 139 | } 140 | 141 | static inline void ntrdma_peer_vbell_disable(struct ntrdma_peer_vbell *peer_vbell) 142 | { 143 | spin_lock_bh(&peer_vbell->lock); 144 | peer_vbell->enabled = 0; 145 | spin_unlock_bh(&peer_vbell->lock); 146 | } 147 | 148 | #endif 149 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_wr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_WR_H 34 | #define NTRDMA_WR_H 35 | 36 | #include "ntrdma_ioctl.h" 37 | #include "ntrdma_sg.h" 38 | 39 | /* work and completion queue entry op_code values */ 40 | #define NTRDMA_WR_RDMA_WRITE 0 /* IB_WR_RDMA_WRITE post rdma write */ 41 | #define NTRDMA_WR_SEND_RDMA 1 /* IB_WR_RDMA_WRITE_WITH_IMM post write with imm */ 42 | #define NTRDMA_WR_SEND 2 /* IB_WR_SEND post send */ 43 | #define NTRDMA_WR_SEND_IMM 3 /* IB_WR_SEND_WITH_IMM post send with imm */ 44 | #define NTRDMA_WR_RDMA_READ 4 /* IB_WR_RDMA_READ post rdma read */ 45 | #define NTRDMA_SEND_WR_MAX_SUPPORTED IB_WR_RDMA_READ 46 | #define NTRDMA_WR_RECV 5 /* post recv and consumed by send */ 47 | #define NTRDMA_WR_RECV_INV 6 /* recv consumed by send with inv */ 48 | #define NTRDMA_WR_RECV_IMM 7 /* recv consumed by send with imm */ 49 | #define NTRDMA_WR_RECV_RDMA 8 /* recv consumed by write with imm */ 50 | #define NTRDMA_WR_SEND_INV 9 /* post send with inv */ 51 | 52 | /* completion queue entry op_status values */ 53 | #define NTRDMA_WC_SUCCESS 0 54 | #define NTRDMA_WC_ERR_ABORTED 1 55 | #define NTRDMA_WC_ERR_CORRUPT 2 56 | #define NTRDMA_WC_ERR_OPCODE 3 57 | #define NTRDMA_WC_ERR_RECV_NOT_READY 4 58 | #define NTRDMA_WC_ERR_RECV_MISSING 5 59 | #define NTRDMA_WC_ERR_RECV_OVERFLOW 6 60 | #define NTRDMA_WC_ERR_RDMA_KEY 7 61 | #define NTRDMA_WC_ERR_RDMA_RANGE 8 62 | #define NTRDMA_WC_ERR_RDMA_ACCESS 9 63 | #define NTRDMA_WC_ERR_LOC_PORT 10 64 | 65 | 66 | struct ntrdma_recv_wqe { 67 | u64 ulp_handle; 68 | u16 op_code; 69 | u16 op_status; 70 | u32 sg_count; 71 | struct ntrdma_wr_rcv_sge rcv_sg_list[]; 72 | }; 73 | 74 | struct ntrdma_cqe { 75 | u64 ulp_handle; 76 | u16 op_code; 77 | u16 op_status; 78 | u32 rdma_len; 79 | u32 imm_data; 80 | int flags; 81 | }; 82 | 83 | 84 | static inline size_t ntrdma_wqe_snd_sg_list_size(u32 sg_cap) 85 | { 86 | return sg_cap * sizeof(struct ib_sge); 87 | } 88 | 89 | static inline size_t ntrdma_wqe_snd_inline_size(u32 inline_cap) 90 | { 91 | return inline_cap; 92 | } 93 | 94 | static inline bool ntrdma_send_wqe_is_inline(const struct ntrdma_send_wqe *wqe) 95 | { 96 | return (wqe->op_code == IB_WR_RDMA_WRITE) && 97 | (wqe->flags & IB_SEND_INLINE); 98 | } 99 | 100 | static inline struct ib_sge * 101 | snd_sg_list(int index, struct ntrdma_send_wqe *wqe) 102 | { 103 | void *past_wqe = wqe + 1; 104 | struct ib_sge *array = past_wqe; 105 | 106 | return &array[index]; 107 | } 108 | 109 | static inline const struct ib_sge * 110 | const_snd_sg_list(int index, const struct ntrdma_send_wqe *wqe) 111 | { 112 | const void *past_wqe = wqe + 1; 113 | const struct ib_sge *array = past_wqe; 114 | 115 | return &array[index]; 116 | } 117 | 118 | static inline u8 *snd_inline_data(struct ntrdma_send_wqe *wqe) 119 | { 120 | void *past_wqe = wqe + 1; 121 | 122 | return past_wqe; 123 | } 124 | 125 | static inline size_t ntrdma_wqe_rcv_sg_list_size(u32 sg_cap) 126 | { 127 | return sg_cap * sizeof(struct ntrdma_wr_rcv_sge); 128 | } 129 | 130 | static inline size_t ntrdma_send_wqe_size(u32 sg_cap, u32 inline_cap) 131 | { 132 | return sizeof(struct ntrdma_send_wqe) + 133 | max(ntrdma_wqe_snd_sg_list_size(sg_cap), 134 | ntrdma_wqe_snd_inline_size(inline_cap)); 135 | } 136 | 137 | static inline size_t ntrdma_recv_wqe_size(u32 sg_cap) 138 | { 139 | return sizeof(struct ntrdma_recv_wqe) + 140 | ntrdma_wqe_rcv_sg_list_size(sg_cap); 141 | } 142 | 143 | static inline bool ntrdma_wr_status_no_recv(u16 op_status) 144 | { 145 | switch (op_status) { 146 | case NTRDMA_WC_ERR_RECV_NOT_READY: 147 | case NTRDMA_WC_ERR_RECV_MISSING: 148 | return true; 149 | } 150 | return false; 151 | } 152 | 153 | static inline bool ntrdma_wr_code_is_recv(u16 op_code) 154 | { 155 | switch (op_code) { 156 | case NTRDMA_WR_RECV: 157 | case NTRDMA_WR_RECV_INV: 158 | case NTRDMA_WR_RECV_IMM: 159 | case NTRDMA_WR_RECV_RDMA: 160 | return true; 161 | } 162 | 163 | return false; 164 | } 165 | 166 | static inline bool ntrdma_wr_code_is_send(u16 op_code) 167 | { 168 | switch (op_code) { 169 | case IB_WR_SEND: 170 | case NTRDMA_WR_SEND_INV: 171 | case IB_WR_SEND_WITH_IMM: 172 | case IB_WR_RDMA_WRITE_WITH_IMM: 173 | return true; 174 | } 175 | 176 | return false; 177 | } 178 | 179 | static inline bool ntrdma_wr_code_is_rdma(u16 op_code) 180 | { 181 | switch (op_code) { 182 | case IB_WR_RDMA_WRITE_WITH_IMM: 183 | case IB_WR_RDMA_WRITE: 184 | case IB_WR_RDMA_READ: 185 | return true; 186 | } 187 | return false; 188 | } 189 | 190 | static inline bool ntrdma_wr_code_push_data(u16 op_code) 191 | { 192 | switch (op_code) { 193 | case IB_WR_SEND: 194 | case NTRDMA_WR_SEND_INV: 195 | case IB_WR_SEND_WITH_IMM: 196 | case IB_WR_RDMA_WRITE_WITH_IMM: 197 | case IB_WR_RDMA_WRITE: 198 | return true; 199 | } 200 | return false; 201 | } 202 | 203 | static inline bool ntrdma_wr_code_pull_data(u16 op_code) 204 | { 205 | return op_code == IB_WR_RDMA_READ; 206 | } 207 | 208 | int ntrdma_recv_wqe_sync(struct ntrdma_dev *dev, struct ntrdma_recv_wqe *wqe); 209 | void ntrdma_recv_wqe_cleanup(struct ntrdma_recv_wqe *wqe); 210 | 211 | #endif 212 | -------------------------------------------------------------------------------- /drivers/infiniband/hw/ntrdma/ntrdma_zip.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTRDMA_ZIP_H 34 | #define NTRDMA_ZIP_H 35 | 36 | #include "ntrdma_sg.h" 37 | 38 | struct ntrdma_pd; 39 | struct ntrdma_rpd; 40 | 41 | /* Append dma requests from snd entries in pd to rcv entries in rpd */ 42 | /* Called on snd side. */ 43 | int ntrdma_zip_rdma(struct ntrdma_dev *dev, struct ntc_dma_chan *chan, 44 | u32 *rdma_len, 45 | const struct ntrdma_wr_rcv_sge *rcv_sg_list, 46 | const struct ib_sge *snd_sg_list, 47 | u32 rcv_sg_count, u32 snd_sg_count, u32 rcv_start_offset, 48 | u64 wrid, int dma_wait); 49 | 50 | int ntrdma_zip_rdma_imm(struct ntrdma_dev *dev, struct ntc_dma_chan *chan, 51 | const struct ntrdma_wr_rcv_sge *rcv_sg_list, 52 | const void *snd_data, 53 | u32 rcv_sg_count, u32 snd_data_size, 54 | u32 rcv_start_offset, u64 wrid); 55 | 56 | /* Sync incoming data to dst entries in pd for the cpu */ 57 | /* Called on rcv side. */ 58 | int ntrdma_zip_sync(struct ntrdma_dev *dev, 59 | struct ntrdma_wr_rcv_sge *lrcv_sg_list, u32 lrcv_sg_count); 60 | 61 | int ntrdma_zip_memcpy(struct ntrdma_dev *dev, u32 target_key, u64 target_addr, 62 | void *src, u64 size); 63 | 64 | #endif 65 | 66 | -------------------------------------------------------------------------------- /drivers/ntc/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_NTC) += ntc_ntb.o 2 | CFLAGS_ntc_ntb_msi.o = -I$(src) 3 | 4 | ntc_ntb-y += ntc.o 5 | ntc_ntb-y += ntc_ntb_msi.o 6 | -------------------------------------------------------------------------------- /drivers/ntc/ntc-trace.h: -------------------------------------------------------------------------------- 1 | #undef TRACE_SYSTEM 2 | #define TRACE_SYSTEM ntc 3 | 4 | #if !defined(_NTC_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) 5 | #define _NTC_TRACE_H 6 | 7 | #include 8 | 9 | DECLARE_EVENT_CLASS(dma_comp_template, 10 | TP_PROTO(u64 wrid, u64 addr, size_t len, u32 result, u32 residue), 11 | TP_ARGS(wrid, addr, len, result, residue), 12 | TP_STRUCT__entry( 13 | __field(u64, wrid) 14 | __field(u64, addr) 15 | __field(size_t, len) 16 | __field(u32, result) 17 | __field(u32, residue) 18 | ), 19 | TP_fast_assign( 20 | __entry->wrid = wrid; 21 | __entry->addr = addr; 22 | __entry->len = len; 23 | __entry->result = result; 24 | __entry->residue = residue; 25 | ), 26 | TP_printk( 27 | "Completion of wrid %#llx addr %#llx len %zu, result %d, residue %d", 28 | __entry->wrid, __entry->addr, __entry->len, 29 | __entry->result, __entry->residue) 30 | ); 31 | 32 | /*trace_post_send*/ 33 | DEFINE_EVENT(dma_comp_template, dma_completion, 34 | TP_PROTO(u64 wrid, u64 addr, size_t len, u32 result, u32 residue), 35 | TP_ARGS(wrid, addr, len, result, residue)); 36 | #endif /* _NTC_TRACE_H */ 37 | 38 | /* This part must be outside protection */ 39 | #undef TRACE_INCLUDE_PATH 40 | #define TRACE_INCLUDE_PATH . 41 | #define TRACE_INCLUDE_FILE ntc-trace 42 | #include 43 | -------------------------------------------------------------------------------- /drivers/ntc/ntc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #include 34 | #include 35 | #include 36 | 37 | #define NTC_COUNTERS 38 | #include "ntc.h" 39 | #include 40 | #include 41 | 42 | #include "ntc_internal.h" 43 | 44 | #define DRIVER_NAME "ntc" 45 | #define DRIVER_DESCRIPTION "NTC Driver Framework" 46 | 47 | #define DRIVER_LICENSE "Dual BSD/GPL" 48 | #define DRIVER_VERSION "0.2" 49 | #define DRIVER_RELDATE "2 October 2015" 50 | #define DRIVER_AUTHOR "Allen Hubbe " 51 | 52 | MODULE_LICENSE(DRIVER_LICENSE); 53 | MODULE_VERSION(DRIVER_VERSION); 54 | MODULE_AUTHOR(DRIVER_AUTHOR); 55 | MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 56 | 57 | DEFINE_PER_CPU(struct ntc_dev_counters, ntc_dev_cnt); 58 | EXPORT_PER_CPU_SYMBOL(ntc_dev_cnt); 59 | 60 | void inc_dma_reject_counter(void) 61 | { 62 | this_cpu_inc(ntc_dev_cnt.dma_reject_count); 63 | } 64 | EXPORT_SYMBOL(inc_dma_reject_counter); 65 | 66 | void ntc_flush_dma_channels(struct ntc_dev *ntc) 67 | { 68 | int i; 69 | 70 | for (i = 0; i < ARRAY_SIZE(ntc->dma_chan); i++) 71 | if (ntc->dma_chan[i].chan) { 72 | ntc_vdbg(ntc, "flushing DMA channel %d", i); 73 | ntc_dma_flush(&ntc->dma_chan[i]); 74 | } else 75 | break; 76 | 77 | ntc_vdbg(ntc, "All DMA channels flushed"); 78 | } 79 | EXPORT_SYMBOL(ntc_flush_dma_channels); 80 | 81 | int ntc_umem_sgl(struct ntc_dev *ntc, struct ib_umem *ib_umem, 82 | struct ntc_mr_buf *sgl, int count, int mr_access_flags) 83 | { 84 | struct scatterlist *sg, *next; 85 | dma_addr_t dma_addr; 86 | size_t dma_len, offset, total_len; 87 | int i, n; 88 | 89 | offset = ib_umem_offset(ib_umem); 90 | total_len = 0; 91 | n = 0; 92 | for_each_sg(ib_umem->sg_head.sgl, sg, ib_umem->sg_head.nents, i) { 93 | /* dma_addr is start DMA addr of the contiguous range */ 94 | dma_addr = sg_dma_address(sg); 95 | /* dma_len accumulates the length of the contiguous range */ 96 | dma_len = sg_dma_len(sg); 97 | 98 | TRACE("ntc_umem_sgl: dma_addr %#llx access %d", 99 | dma_addr, mr_access_flags); 100 | 101 | for (; i + 1 < ib_umem->sg_head.nents; ++i) { 102 | next = sg_next(sg); 103 | if (!next) 104 | break; 105 | if (sg_dma_address(next) != dma_addr + dma_len) 106 | break; 107 | dma_len += sg_dma_len(next); 108 | sg = next; 109 | } 110 | 111 | if (dma_len <= offset) { 112 | offset -= dma_len; 113 | continue; 114 | } 115 | 116 | if (offset) { 117 | dma_addr += offset; 118 | dma_len -= offset; 119 | offset = 0; 120 | } 121 | 122 | total_len += dma_len; 123 | if (total_len > ib_umem->length) { 124 | dma_len -= total_len - ib_umem->length; 125 | total_len = ib_umem->length; 126 | } 127 | 128 | TRACE("ntc_umem_sgl: dma_len %#llx", (u64)dma_len); 129 | if (sgl && (n < count)) { 130 | if (ntc_mr_buf_map_dma(&sgl[n], ntc, dma_len, 131 | dma_addr, mr_access_flags) < 0) 132 | break; 133 | } 134 | 135 | ++n; 136 | 137 | if (total_len == ib_umem->length) 138 | break; 139 | } 140 | 141 | return n; 142 | } 143 | EXPORT_SYMBOL(ntc_umem_sgl); 144 | 145 | static int ntc_probe(struct device *dev) 146 | { 147 | struct ntc_dev *ntc = ntc_of_dev(dev); 148 | struct ntc_driver *driver = ntc_of_driver(dev->driver); 149 | int rc; 150 | 151 | ntc_vdbg(ntc, "probe"); 152 | 153 | get_device(dev); 154 | rc = driver->ops.probe(driver, ntc); 155 | if (rc) 156 | put_device(dev); 157 | 158 | ntc_vdbg(ntc, "probe return %d", rc); 159 | 160 | return rc; 161 | } 162 | 163 | static int ntc_remove(struct device *dev) 164 | { 165 | struct ntc_dev *ntc = ntc_of_dev(dev); 166 | struct ntc_driver *driver; 167 | 168 | ntc_vdbg(ntc, "remove"); 169 | 170 | if (dev->driver) { 171 | driver = ntc_of_driver(dev->driver); 172 | 173 | driver->ops.remove(driver, ntc); 174 | put_device(dev); 175 | } 176 | 177 | return 0; 178 | } 179 | 180 | static struct bus_type ntc_bus = { 181 | .name = "ntc", 182 | .probe = ntc_probe, 183 | .remove = ntc_remove, 184 | }; 185 | 186 | struct bus_type *ntc_bus_ptr(void) 187 | { 188 | return &ntc_bus; 189 | } 190 | EXPORT_SYMBOL(ntc_bus_ptr); 191 | 192 | static 193 | struct ntc_dma_chan *ntc_req_rr(struct ntc_dev *ntc, 194 | enum ntc_dma_chan_type type) 195 | { 196 | int i; 197 | 198 | i = smp_processor_id() % get_num_dma_chan(); 199 | if (!ntc->dma_chan[i].chan) { 200 | i = 0; 201 | } 202 | 203 | return &ntc->dma_chan[i]; 204 | } 205 | 206 | void ntc_init_dma_chan(struct ntc_dma_chan **dma_chan, 207 | struct ntc_dev *ntc, enum ntc_dma_chan_type type) 208 | { 209 | *dma_chan = ntc_req_rr(ntc, type); 210 | } 211 | EXPORT_SYMBOL(ntc_init_dma_chan); 212 | 213 | static int __init ntc_driver_init(void) 214 | { 215 | int rc; 216 | int i; 217 | int num_cpus; 218 | 219 | pr_info("%s: %s %s init\n", DRIVER_NAME, 220 | DRIVER_DESCRIPTION, DRIVER_VERSION); 221 | rc = bus_register(&ntc_bus); 222 | if (rc < 0) 223 | return rc; 224 | 225 | rc = ntc_init(); 226 | num_cpus = num_online_cpus(); 227 | for (i = 0; i < num_cpus; i++) 228 | memset(per_cpu_ptr(&ntc_dev_cnt, i), 0, 229 | sizeof(struct ntc_dev_counters)); 230 | if (rc < 0) 231 | bus_unregister(&ntc_bus); 232 | 233 | return rc; 234 | } 235 | module_init(ntc_driver_init); 236 | 237 | static void __exit ntc_driver_exit(void) 238 | { 239 | ntc_exit(); 240 | bus_unregister(&ntc_bus); 241 | pr_info("%s: %s %s exit\n", DRIVER_NAME, 242 | DRIVER_DESCRIPTION, DRIVER_VERSION); 243 | } 244 | module_exit(ntc_driver_exit); 245 | 246 | -------------------------------------------------------------------------------- /drivers/ntc/ntc.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | #pragma once 3 | #include "../../include/linux/ntc.h" 4 | -------------------------------------------------------------------------------- /drivers/ntc/ntc_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Dell Technologies. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef _NTC_INTERNAL_H_ 34 | #define _NTC_INTERNAL_H_ 35 | 36 | #include "ntc.h" 37 | 38 | /** 39 | * ntc_ctx_hello() - exchange data for upper layer protocol initialization 40 | * @ntc: Device context. 41 | * @phase: Hello phase number. 42 | * @in_buf: Hello input buffer. 43 | * @in_buf: Hello input buffer size. 44 | * @out_buf: Hello output buffer. 45 | * @out_buf: Hello output buffer size. 46 | * 47 | * In each phase, provide an input buffer with data from the peer, and an 48 | * output buffer with data for the peer. The inputs are from the peer previous 49 | * phase outputs, and the outputs are for the peer next phase inputs. Return 50 | * the size of a buffer for the peer next phase outputs, and the local phase 51 | * after next inputs. 52 | * 53 | * The zero'th phase has no inputs or outputs, and returns the size of the 54 | * buffer for the peer phase one outputs, which will be the local phase two 55 | * inputs. 56 | * 57 | * The first phase has only outputs, which will be the peer phase two inputs. 58 | * Phase one returns the size of the buffer for peer phase two outputs, which 59 | * will be the local phase three inputs. The second and later phases have both 60 | * inputs and outputs, until the final phases. 61 | * 62 | * If a phase returns zero, then the next phase has no outputs. The next phase 63 | * is the last phase. Likewise, the last phase must return zero, or an error 64 | * code, only. 65 | * 66 | * When the next phase (logically, after the last phase) has no inputs or 67 | * outputs, initialization is complete. After initialization is complete on 68 | * both sides of the channel, a link event will be indicated to the upper 69 | * layer, indicating that the link is up. 70 | * 71 | * The input buffer size is the capacity of the buffer, although the peer may 72 | * have written less data than the buffer size. Likewise, the output buffer 73 | * size is the capacity of the buffer on the peer. It is the responsibility of 74 | * the upper layer to detect and handle any cases where the data in the buffer 75 | * is less than the capacity of the buffer. 76 | * 77 | * Return: Input buffer size for phase after next, zero, or an error number. 78 | */ 79 | static inline int ntc_ctx_hello(struct ntc_dev *ntc, int phase, 80 | void *in_buf, size_t in_size, 81 | void *out_buf, size_t out_size) 82 | { 83 | const struct ntc_ctx_ops *ctx_ops; 84 | void *ctx; 85 | 86 | ntc_dbg(ntc, "hello phase %d", phase); 87 | 88 | ctx = ntc_get_ctx(ntc); 89 | rmb(); /* if ctx_ops is set, drvdata must be set */ 90 | ctx_ops = ntc->ctx_ops; 91 | 92 | if (ctx_ops && ctx_ops->hello) 93 | return ctx_ops->hello(ctx, phase, 94 | in_buf, in_size, 95 | out_buf, out_size); 96 | 97 | if (phase || in_size || out_size) 98 | return -EINVAL; 99 | 100 | return 0; 101 | } 102 | 103 | /** 104 | * ntc_ctx_enable() - notify driver that the link is active 105 | * @ntc: Device context. 106 | * 107 | * Notify the driver context that the link is active. The driver may begin 108 | * issuing requests. 109 | */ 110 | static inline int ntc_ctx_enable(struct ntc_dev *ntc) 111 | { 112 | const struct ntc_ctx_ops *ctx_ops; 113 | void *ctx; 114 | int ret = 0; 115 | 116 | ntc_dbg(ntc, "enable"); 117 | 118 | ctx = ntc_get_ctx(ntc); 119 | rmb(); /* if ctx_ops is set, drvdata must be set */ 120 | ctx_ops = ntc->ctx_ops; 121 | 122 | if (ctx_ops) 123 | ret = ctx_ops->enable(ctx); 124 | 125 | return ret; 126 | } 127 | 128 | /** 129 | * ntc_ctx_disable() - notify driver that the link is not active 130 | * @ntc: Device context. 131 | * 132 | * Notify the driver context that the link is not active. The driver should 133 | * stop issuing requests. See ntc_ctx_quiesce(). 134 | */ 135 | static inline void ntc_ctx_disable(struct ntc_dev *ntc) 136 | { 137 | const struct ntc_ctx_ops *ctx_ops; 138 | void *ctx; 139 | 140 | ntc_dbg(ntc, "disable"); 141 | 142 | ctx = ntc_get_ctx(ntc); 143 | rmb(); /* if ctx_ops is set, drvdata must be set */ 144 | ctx_ops = ntc->ctx_ops; 145 | 146 | if (ctx_ops) 147 | ctx_ops->disable(ctx); 148 | } 149 | 150 | /** 151 | * ntc_ctx_quiesce() - make sure driver is not issuing any new requests 152 | * @ntc: Device context. 153 | * 154 | * Make sure the driver context is not issuing any new requests. The driver 155 | * should block and wait for any asynchronous operations to stop. It is not 156 | * necessary to wait for previously issued requests to complete, just that no 157 | * new requests will be issued after this function returns. 158 | */ 159 | static inline void ntc_ctx_quiesce(struct ntc_dev *ntc) 160 | { 161 | const struct ntc_ctx_ops *ctx_ops; 162 | void *ctx; 163 | 164 | ntc_dbg(ntc, "quiesce"); 165 | 166 | ctx = ntc_get_ctx(ntc); 167 | rmb(); /* if ctx_ops is set, drvdata must be set */ 168 | ctx_ops = ntc->ctx_ops; 169 | 170 | if (ctx_ops && ctx_ops->quiesce) 171 | ctx_ops->quiesce(ctx); 172 | } 173 | 174 | /** 175 | * ntc_ctx_reset() - safely deallocate upper layer protocol structures 176 | * @ntc: Device context. 177 | * 178 | * Call the upper layer to deallocate data structures that the peer may have 179 | * been writing. When calling this function, the channel must guarantee to the 180 | * upper layer that the peer is no longer using any of the locally channel 181 | * mapped buffers. Either the channel has obtained acknowledgment from the 182 | * peer for a protocol reset, or the channel can prevent misbehavior from a 183 | * peer in an unknown state. 184 | * 185 | * It is important that the upper layer does not immediately deallocate 186 | * structures on a link event, because while the channel link may report to be 187 | * down, a hardware link may still be active. Immediately deallocating data 188 | * buffers could result in memory corruption, as it may be used by the peer 189 | * after it is freed locally. A coordinated protocol reset is therefore 190 | * necessary to prevent use after free conditions from occurring. 191 | */ 192 | static inline void ntc_ctx_reset(struct ntc_dev *ntc) 193 | { 194 | const struct ntc_ctx_ops *ctx_ops; 195 | void *ctx; 196 | 197 | ntc_dbg(ntc, "reset"); 198 | 199 | ctx = ntc_get_ctx(ntc); 200 | rmb(); /* if ctx_ops is set, drvdata must be set */ 201 | ctx_ops = ntc->ctx_ops; 202 | 203 | if (ctx_ops && ctx_ops->reset) 204 | ctx_ops->reset(ctx); 205 | } 206 | 207 | /** 208 | * ntc_ctx_signal() - notify driver context of a signal event 209 | * @ntc: Device context. 210 | * @vector: Interrupt vector number. 211 | * 212 | * Notify the driver context of a signal event triggered by the peer. 213 | */ 214 | static inline void ntc_ctx_signal(struct ntc_dev *ntc, int vec) 215 | { 216 | const struct ntc_ctx_ops *ctx_ops; 217 | void *ctx; 218 | 219 | ntc_vdbg(ntc, "signal"); 220 | 221 | ctx = ntc_get_ctx(ntc); 222 | rmb(); /* if ctx_ops is set, drvdata must be set */ 223 | ctx_ops = ntc->ctx_ops; 224 | 225 | if (ctx_ops && ctx_ops->signal) 226 | ctx_ops->signal(ctx, vec); 227 | } 228 | 229 | /** 230 | * ntc_register_device() - register a ntc device 231 | * @ntc: Device context. 232 | * 233 | * The device will be added to the list of ntc devices. If any drivers are 234 | * interested in ntc devices, each driver will be notified of the ntc device, 235 | * until at most one driver accepts the device. 236 | * 237 | * Return: Zero if the device is registered, otherwise an error number. 238 | */ 239 | static inline int ntc_register_device(struct ntc_dev *ntc) 240 | { 241 | pr_devel("register device %s\n", dev_name(&ntc->dev)); 242 | 243 | ntc->ctx_ops = NULL; 244 | 245 | ntc->dev.bus = ntc_bus_ptr(); 246 | 247 | return device_register(&ntc->dev); 248 | } 249 | 250 | /** 251 | * ntc_unregister_device() - unregister a ntc device 252 | * @ntc: Device context. 253 | * 254 | * The device will be removed from the list of ntc devices. If the ntc device 255 | * is associated with a driver, the driver will be notified to remove the 256 | * device. 257 | */ 258 | static inline void ntc_unregister_device(struct ntc_dev *ntc) 259 | { 260 | pr_devel("unregister device %s\n", dev_name(&ntc->dev)); 261 | 262 | device_unregister(&ntc->dev); 263 | 264 | ntc->dev.bus = NULL; 265 | } 266 | 267 | int __init ntc_init(void); 268 | void __exit ntc_exit(void); 269 | 270 | #endif 271 | -------------------------------------------------------------------------------- /include/linux/ntc_mm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Dell Technologies. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef _NTC_MM_H_ 34 | #define _NTC_MM_H_ 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include "ntc_trace.h" 41 | 42 | #define FREE_MAGIC (0xdeadc0de) 43 | #define FIXED_MAGIC (0xbed00bad) 44 | struct ntc_mm_free_entry { 45 | struct ntc_mm_free_entry *next; 46 | u32 magic; 47 | }; 48 | 49 | struct ntc_fixed_mm { 50 | struct ntc_mm_free_entry *free; 51 | spinlock_t lock; 52 | u32 magic; 53 | }; 54 | 55 | struct ntc_mm { 56 | void *memory; 57 | void *end; 58 | void *brk; 59 | struct idr fixed; 60 | spinlock_t lock; 61 | struct mutex idr_lock; 62 | }; 63 | 64 | static inline int _ntc_mm_chunk_size(int size) 65 | { 66 | int clz; 67 | 68 | BUILD_BUG_ON((sizeof(struct ntc_mm_free_entry) - 1) & 69 | sizeof(struct ntc_mm_free_entry)); 70 | 71 | if (unlikely(!size)) 72 | return 0; 73 | 74 | size = ALIGN(size, sizeof(struct ntc_mm_free_entry)); 75 | clz = __builtin_clz(size); /* The number of leading zeros in size. */ 76 | 77 | if (unlikely(clz < 2)) 78 | return 0; 79 | 80 | if (size << (clz + 1) != 0) 81 | return 1 << (8 * sizeof(int) - clz); 82 | else 83 | return 1 << (8 * sizeof(int) - clz - 1); 84 | } 85 | 86 | static inline void *_ntc_mm_round_up_ptr(void *ptr) 87 | { 88 | return (void *)ALIGN((long)ptr, sizeof(struct ntc_mm_free_entry)); 89 | } 90 | 91 | static inline void ntc_mm_fixed_init(struct ntc_fixed_mm *fixed) 92 | { 93 | fixed->free = NULL; 94 | fixed->magic = FIXED_MAGIC; 95 | spin_lock_init(&fixed->lock); 96 | } 97 | 98 | static inline void *ntc_mm_fixed_alloc(struct ntc_fixed_mm *fixed) 99 | { 100 | struct ntc_mm_free_entry *free; 101 | unsigned long irqflags; 102 | 103 | spin_lock_irqsave(&fixed->lock, irqflags); 104 | free = fixed->free; 105 | if (free) { 106 | BUG_ON(free->magic != FREE_MAGIC); 107 | fixed->free = free->next; 108 | } else 109 | free = ERR_PTR(-ENOMEM); 110 | spin_unlock_irqrestore(&fixed->lock, irqflags); 111 | 112 | return free; 113 | } 114 | 115 | static inline void ntc_mm_fixed_free(struct ntc_fixed_mm *fixed, void *ptr) 116 | { 117 | struct ntc_mm_free_entry *free = ptr; 118 | unsigned long irqflags; 119 | 120 | spin_lock_irqsave(&fixed->lock, irqflags); 121 | free->next = fixed->free; 122 | free->magic = FREE_MAGIC; 123 | fixed->free = free; 124 | spin_unlock_irqrestore(&fixed->lock, irqflags); 125 | } 126 | 127 | static inline int ntc_mm_init(struct ntc_mm *mm, void *memory, size_t size) 128 | { 129 | void *m; 130 | void *end; 131 | 132 | spin_lock_init(&mm->lock); 133 | 134 | m = _ntc_mm_round_up_ptr(memory); 135 | if (size) 136 | size -= (m - memory); 137 | memory = m; 138 | 139 | end = memory + size; 140 | if (end < memory) 141 | return -EINVAL; 142 | 143 | mm->memory = memory; 144 | mm->brk = memory; 145 | mm->end = end; 146 | 147 | mutex_init(&mm->idr_lock); 148 | idr_init(&mm->fixed); 149 | 150 | return 0; 151 | } 152 | 153 | static inline void ntc_mm_deinit(struct ntc_mm *mm) 154 | { 155 | struct ntc_fixed_mm *fixed; 156 | int size; 157 | 158 | idr_for_each_entry(&mm->fixed, fixed, size) 159 | kfree(fixed); 160 | 161 | idr_destroy(&mm->fixed); 162 | mutex_destroy(&mm->idr_lock); 163 | } 164 | 165 | static inline void *ntc_mm_sbrk(struct ntc_mm *mm, int inc) 166 | { 167 | void *result; 168 | void *brk; 169 | unsigned long irqflags; 170 | 171 | if (unlikely(inc < 0)) 172 | return ERR_PTR(-EINVAL); 173 | 174 | spin_lock_irqsave(&mm->lock, irqflags); 175 | 176 | result = mm->brk; 177 | brk = result + inc; 178 | 179 | if (unlikely(brk < result)) { 180 | result = ERR_PTR(-ENOMEM); 181 | goto out; 182 | } 183 | 184 | if (unlikely(brk > mm->end)) { 185 | result = ERR_PTR(-ENOMEM); 186 | goto out; 187 | } 188 | 189 | mm->brk = brk; 190 | 191 | out: 192 | spin_unlock_irqrestore(&mm->lock, irqflags); 193 | 194 | return result; 195 | } 196 | 197 | static inline struct ntc_fixed_mm *_ntc_mm_find_fixed(struct ntc_mm *mm, 198 | int size) 199 | { 200 | struct ntc_fixed_mm *rc; 201 | rcu_read_lock(); 202 | rc = idr_find(&mm->fixed, size); 203 | rcu_read_unlock(); 204 | return rc; 205 | } 206 | 207 | static inline struct ntc_fixed_mm * 208 | _ntc_mm_get_fixed(struct ntc_mm *mm, int size, gfp_t gfp) 209 | { 210 | struct ntc_fixed_mm *fixed; 211 | int rc; 212 | 213 | fixed = _ntc_mm_find_fixed(mm, size); 214 | if (likely(fixed)) { 215 | BUG_ON(fixed->magic != FIXED_MAGIC); 216 | return fixed; 217 | } 218 | 219 | if (size <= 0) 220 | return ERR_PTR(-EINVAL); 221 | 222 | fixed = kmalloc(sizeof(*fixed), gfp); 223 | if (!fixed) 224 | return ERR_PTR(-ENOMEM); 225 | gfp &= ~__GFP_ZERO; 226 | ntc_mm_fixed_init(fixed); 227 | 228 | mutex_lock(&mm->idr_lock); 229 | rc = idr_alloc(&mm->fixed, fixed, size, size + 1, gfp); 230 | mutex_unlock(&mm->idr_lock); 231 | if (rc < 0) { 232 | kfree(fixed); 233 | 234 | if (rc == -ENOSPC) { 235 | fixed = _ntc_mm_find_fixed(mm, size); 236 | if (fixed) { 237 | BUG_ON(fixed->magic != FIXED_MAGIC); 238 | return fixed; 239 | } else 240 | return ERR_PTR(-EFAULT); 241 | } 242 | 243 | return ERR_PTR(rc); 244 | } 245 | 246 | return fixed; 247 | } 248 | 249 | static inline int ntc_mm_preinit(struct ntc_mm *mm, int size, int num_els) 250 | { 251 | struct ntc_fixed_mm *fixed; 252 | void *ptr; 253 | int i; 254 | 255 | size = _ntc_mm_chunk_size(size); 256 | 257 | fixed = _ntc_mm_get_fixed(mm, size, GFP_KERNEL); 258 | if (IS_ERR(fixed)) 259 | return PTR_ERR(fixed); 260 | 261 | for (i = 0; i < num_els; i++) { 262 | ptr = ntc_mm_sbrk(mm, size); 263 | if (IS_ERR(ptr)) { 264 | if (i > 0) 265 | return i; 266 | else 267 | return PTR_ERR(ptr); 268 | } 269 | ntc_mm_fixed_free(fixed, ptr); 270 | } 271 | 272 | TRACE("mm %p: added %d buffers of size %d. brk is %ld", 273 | mm, num_els, size, mm->brk - mm->memory); 274 | pr_debug("mm %p: added %d buffers of size %d. brk is %ld", 275 | mm, num_els, size, mm->brk - mm->memory); 276 | 277 | return num_els; 278 | } 279 | 280 | static inline void *ntc_mm_alloc(const char *caller, int line, 281 | struct ntc_mm *mm, int size, gfp_t gfp) 282 | { 283 | struct ntc_fixed_mm *fixed; 284 | void *ptr; 285 | 286 | size = _ntc_mm_chunk_size(size); 287 | 288 | fixed = _ntc_mm_get_fixed(mm, size, gfp); 289 | if (unlikely(IS_ERR(fixed))) { 290 | TRACE("mm %p from %s:%d: failed to add %d bytes. No fixed mm", 291 | mm, caller, line, size); 292 | pr_debug("mm %p from %s:%d: failed to add %d bytes. No fixed mm", 293 | mm, caller, line, size); 294 | return fixed; 295 | } 296 | 297 | ptr = ntc_mm_fixed_alloc(fixed); 298 | if (likely(!IS_ERR(ptr))) { 299 | if (gfp & __GFP_ZERO) 300 | memset(ptr, 0, size); 301 | return ptr; 302 | } 303 | 304 | ptr = ntc_mm_sbrk(mm, size); 305 | if (unlikely(IS_ERR(ptr))) { 306 | TRACE("mm %p from %s:%d: failed to add %d bytes. brk is %ld", 307 | mm, caller, line, size, mm->brk - mm->memory); 308 | pr_debug("mm %p from %s:%d: failed to add %d bytes. brk is %ld", 309 | mm, caller, line, size, mm->brk - mm->memory); 310 | return ptr; 311 | } 312 | 313 | if (gfp & __GFP_ZERO) 314 | memset(ptr, 0, size); 315 | 316 | TRACE("mm %p from %s:%d: added buffer of size %d. brk is %ld", 317 | mm, caller, line, size, mm->brk - mm->memory); 318 | pr_debug("mm %p from %s:%d: added buffer of size %d. brk is %ld", 319 | mm, caller, line, size, mm->brk - mm->memory); 320 | 321 | return ptr; 322 | } 323 | 324 | static inline void ntc_mm_free(struct ntc_mm *mm, void *ptr, int size) 325 | { 326 | struct ntc_fixed_mm *fixed; 327 | 328 | size = _ntc_mm_chunk_size(size); 329 | 330 | fixed = _ntc_mm_find_fixed(mm, size); 331 | if (likely(fixed)) { 332 | BUG_ON(fixed->magic != FIXED_MAGIC); 333 | ntc_mm_fixed_free(fixed, ptr); 334 | } else 335 | WARN_ON(!fixed); 336 | } 337 | 338 | #endif 339 | -------------------------------------------------------------------------------- /include/linux/ntc_pfn.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 Dell Technologies, Inc. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree. 8 | */ 9 | 10 | #ifndef _NTC_PFN_H_ 11 | #define _NTC_PFN_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | /* 19 | * Return the PFN for the specified address in the vma. This only 20 | * works for a vma that is VM_PFNMAP, VM_IO and not hugetlb. 21 | */ 22 | static inline unsigned long ntc_follow_io_pfn(struct vm_area_struct *vma, 23 | unsigned long address, int write) 24 | { 25 | pgd_t *pgd; 26 | p4d_t *p4d; 27 | pud_t *pud; 28 | pmd_t *pmd; 29 | pte_t *ptep, pte; 30 | spinlock_t *ptl; 31 | unsigned long pfn; 32 | struct mm_struct *mm = vma->vm_mm; 33 | 34 | if (!(vma->vm_flags & VM_PFNMAP) || 35 | !(vma->vm_flags & VM_IO) || 36 | is_vm_hugetlb_page(vma)) 37 | return 0; 38 | 39 | pgd = pgd_offset(mm, address); 40 | if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 41 | return 0; 42 | 43 | p4d = p4d_offset(pgd, address); 44 | if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d))) 45 | return 0; 46 | 47 | pud = pud_offset(p4d, address); 48 | if (pud_none(*pud)) 49 | return 0; 50 | if (unlikely(pud_bad(*pud))) 51 | return 0; 52 | 53 | pmd = pmd_offset(pud, address); 54 | if (pmd_none(*pmd)) 55 | return 0; 56 | if (unlikely(pmd_bad(*pmd))) 57 | return 0; 58 | 59 | ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 60 | pte = *ptep; 61 | if (!pte_present(pte)) 62 | goto bad; 63 | if (write && !pte_write(pte)) 64 | goto bad; 65 | 66 | pfn = pte_pfn(pte); 67 | pte_unmap_unlock(ptep, ptl); 68 | return pfn; 69 | bad: 70 | pte_unmap_unlock(ptep, ptl); 71 | return 0; 72 | } 73 | 74 | static inline int ntc_get_io_pfn_segment_locked(struct mm_struct *mm, 75 | unsigned long addr, 76 | unsigned long len, 77 | int write, 78 | unsigned long *dma_out, 79 | unsigned long *len_out) 80 | { 81 | unsigned long end = addr + len; 82 | unsigned long start = addr & PAGE_MASK; 83 | unsigned long shift = addr & ~PAGE_MASK; 84 | unsigned long pfn; 85 | unsigned long start_pfn; 86 | struct vm_area_struct *vma; 87 | 88 | if (unlikely(end < addr)) 89 | return -EINVAL; 90 | 91 | vma = find_vma(mm, start); 92 | if (unlikely(!vma) || 93 | (start < vma->vm_start) || 94 | unlikely(start >= vma->vm_end)) 95 | return -EINVAL; 96 | 97 | pfn = ntc_follow_io_pfn(vma, start, write); 98 | if (!pfn) 99 | return -EINVAL; 100 | 101 | start_pfn = pfn; 102 | 103 | for (start = vma->vm_end; end > start; start = vma->vm_end) { 104 | cond_resched(); 105 | 106 | vma = find_vma(mm, start); 107 | if (unlikely(!vma) || 108 | (start < vma->vm_start) || 109 | unlikely(start >= vma->vm_end)) 110 | goto partial; 111 | 112 | pfn = ntc_follow_io_pfn(vma, start, write); 113 | if (unlikely((pfn != start_pfn + ((start - addr) >> PAGE_SHIFT)) 114 | || !pfn)) 115 | goto partial; 116 | } 117 | 118 | *dma_out = (start_pfn << PAGE_SHIFT) + shift; 119 | *len_out = len; 120 | 121 | return 0; 122 | 123 | partial: 124 | *dma_out = (start_pfn << PAGE_SHIFT) + shift; 125 | *len_out = start - addr; 126 | 127 | return 0; 128 | } 129 | 130 | static inline int ntc_get_io_pfn_segment(struct mm_struct *mm, 131 | unsigned long addr, 132 | unsigned long len, 133 | int write, 134 | unsigned long *pfn_out, 135 | unsigned long *len_out) 136 | { 137 | int rc; 138 | 139 | down_read(&mm->mmap_sem); 140 | 141 | rc = ntc_get_io_pfn_segment_locked(mm, addr, len, write, 142 | pfn_out, len_out); 143 | 144 | up_read(&mm->mmap_sem); 145 | 146 | return rc; 147 | } 148 | 149 | #endif 150 | -------------------------------------------------------------------------------- /include/linux/ntc_trace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, 2015 EMC Corporation. All rights reserved. 3 | * 4 | * This software is available to you under a choice of one of two 5 | * licenses. You may choose to be licensed under the terms of the GNU 6 | * General Public License (GPL) Version 2, available from the file 7 | * COPYING in the main directory of this source tree, or the 8 | * OpenIB.org BSD license below: 9 | * 10 | * Redistribution and use in source and binary forms, with or 11 | * without modification, are permitted provided that the following 12 | * conditions are met: 13 | * 14 | * - Redistributions of source code must retain the above 15 | * copyright notice, this list of conditions and the following 16 | * disclaimer. 17 | * 18 | * - Redistributions in binary form must reproduce the above 19 | * copyright notice, this list of conditions and the following 20 | * disclaimer in the documentation and/or other materials 21 | * provided with the distribution. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | * SOFTWARE. 31 | */ 32 | 33 | #ifndef NTC_TRACE_H 34 | #define NTC_TRACE_H 35 | 36 | #include 37 | #include 38 | #include 39 | #include 40 | 41 | #define TRACE_EN 42 | #define TRACE_DEBUG_ENABLE 43 | #define TRACE_DBG_ENABLE 44 | /* #define TRACE_VDBG_ENABLE */ 45 | /* #define VERBOSE_DEBUG */ 46 | 47 | 48 | #ifdef TRACE_EN 49 | #define TRACE(fmt, ...) do { \ 50 | char _______FMT[] = __stringify(fmt); \ 51 | int _______SIZE = sizeof(_______FMT); \ 52 | if ((_______SIZE >= 4) && \ 53 | (_______FMT[_______SIZE - 4] == '\\') && \ 54 | (_______FMT[_______SIZE - 3] == 'n')) \ 55 | trace_printk(fmt, ##__VA_ARGS__); \ 56 | else \ 57 | trace_printk(fmt "\n", ##__VA_ARGS__); \ 58 | } while (0) 59 | #else 60 | #define TRACE(...) do {} while (0) 61 | #endif 62 | 63 | #ifdef TRACE_DEBUG_ENABLE 64 | #define TRACE_DEBUG(...) TRACE(__VA_ARGS__) 65 | #else 66 | #define TRACE_DEBUG(...) do {} while (0) 67 | #endif 68 | 69 | #ifdef TRACE_DBG_ENABLE 70 | #define TRACE_DBG(...) TRACE(__VA_ARGS__) 71 | #else 72 | #define TRACE_DBG(...) do {} while (0) 73 | #endif 74 | 75 | #ifdef TRACE_VDBG_ENABLE 76 | #define TRACE_VDBG(...) TRACE(__VA_ARGS__) 77 | #else 78 | #define TRACE_VDBG(...) do {} while (0) 79 | #endif 80 | 81 | #define _ntc_out(__out, __trace, __ntc, __fmt, __args...) do { \ 82 | __trace("%s: %d: " __fmt, __func__, __LINE__, ##__args);\ 83 | __out(&(__ntc)->dev, "%s: %d: " __fmt, __func__, \ 84 | __LINE__, ## __args); \ 85 | } while (0) 86 | 87 | #define ntc_out(__out, __dev, __fmt, ...) do { \ 88 | char _______FMT[] = __stringify(__fmt); \ 89 | int _______SIZE = sizeof(_______FMT); \ 90 | if ((_______SIZE >= 4) && \ 91 | (_______FMT[_______SIZE - 4] == '\\') && \ 92 | (_______FMT[_______SIZE - 3] == 'n')) \ 93 | __out(__dev, __fmt, ##__VA_ARGS__); \ 94 | else \ 95 | __out(__dev, __fmt "\n", ##__VA_ARGS__); \ 96 | } while (0) 97 | 98 | #define _ntc_dbg(__ntc, __fmt, __args...) \ 99 | _ntc_out(dev_dbg, TRACE_DBG, __ntc, __fmt, ##__args) 100 | 101 | #define _ntc_err(__ntc, __fmt, __args...) \ 102 | _ntc_out(dev_err, TRACE, __ntc, "ERROR: " __fmt, ##__args) 103 | 104 | #define _ntc_info(__ntc, __fmt, __args...) \ 105 | _ntc_out(dev_info, TRACE, __ntc, __fmt, ##__args) 106 | 107 | #define _ntc_vdbg(__ntc, __fmt, __args...) \ 108 | _ntc_out(dev_vdbg, TRACE_VDBG, __ntc, __fmt, ##__args) 109 | 110 | #define ntc_dbg(__ntc, __fmt, ...) \ 111 | ntc_out(_ntc_dbg, __ntc, __fmt, ##__VA_ARGS__) 112 | 113 | #define ntc_err(__ntc, __fmt, ...) \ 114 | ntc_out(_ntc_err, __ntc, __fmt, ##__VA_ARGS__) 115 | 116 | #define ntc_info(__ntc, __fmt, ...) \ 117 | ntc_out(_ntc_info, __ntc, __fmt, ##__VA_ARGS__) 118 | 119 | #define ntc_vdbg(__ntc, __fmt, ...) \ 120 | ntc_out(_ntc_vdbg, __ntc, __fmt, ##__VA_ARGS__) 121 | 122 | struct ntc_perf_tracker { 123 | cycles_t total; 124 | cycles_t total_out; 125 | cycles_t last; 126 | u64 num_calls; 127 | }; 128 | 129 | struct ntc_perf_tracker_current { 130 | struct ntc_perf_tracker *tracker; 131 | cycles_t start; 132 | const char *prefix; 133 | u64 window; 134 | }; 135 | 136 | static inline cycles_t ntc_get_cycles(void) 137 | { 138 | asm volatile ("":::"memory"); 139 | return get_cycles(); 140 | } 141 | 142 | static inline void ntc_perf_finish_measure(struct ntc_perf_tracker_current *c) 143 | { 144 | struct ntc_perf_tracker *t = c->tracker; 145 | 146 | if (likely(t->last || t->num_calls)) 147 | t->total_out += c->start - t->last; 148 | 149 | t->num_calls++; 150 | t->last = ntc_get_cycles(); 151 | t->total += t->last - c->start; 152 | 153 | if (t->num_calls != c->window) 154 | return; 155 | 156 | pr_info("PERF: %s [%d]: %lld calls. %lld%% of time. %lld cyc average.", 157 | c->prefix, smp_processor_id(), t->num_calls, 158 | t->total * 100 / (t->total + t->total_out), 159 | t->total / t->num_calls); 160 | 161 | t->num_calls = 0; 162 | t->total_out = 0; 163 | t->total = 0; 164 | } 165 | 166 | #ifdef NTC_PERF_TRACK 167 | 168 | #define DEFINE_NTC_PERF_TRACKER(name, p, w) \ 169 | static DEFINE_PER_CPU(struct ntc_perf_tracker, name##_per_cpu); \ 170 | struct ntc_perf_tracker_current name = \ 171 | { .start = ntc_get_cycles(), .prefix = p, .window = w } 172 | 173 | #define NTC_PERF_MEASURE(name) do { \ 174 | name.tracker = get_cpu_ptr(&name##_per_cpu); \ 175 | ntc_perf_finish_measure(&name); \ 176 | put_cpu_ptr(&name##_per_cpu); \ 177 | } while (0) 178 | 179 | #else 180 | #define DEFINE_NTC_PERF_TRACKER(name, p, w) int name __attribute__ ((unused)) 181 | #define NTC_PERF_MEASURE(name) do {} while (0) 182 | #endif 183 | 184 | 185 | #define DEFINE_NTC_FUNC_PERF_TRACKER(name, w) \ 186 | DEFINE_NTC_PERF_TRACKER(name, __func__, w) 187 | 188 | #endif 189 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | ifndef KSRC 2 | KSRC = /lib/modules/$(shell uname -r)/build 3 | endif 4 | 5 | # Build the ntrdma kernel module 6 | LINUXCONFIG += DEBUG=1 7 | LINUXCONFIG += CONFIG_NTC=m 8 | LINUXCONFIG += CONFIG_NTRDMA=m 9 | 10 | MAKE_TARGETS = all modules modules_install clean help 11 | MAKE_OPTIONS = -C $(KSRC) M=$(CURDIR) $(LINUXCONFIG) 12 | 13 | install: modules_install 14 | 15 | $(MAKE_TARGETS): 16 | $(MAKE) $(MAKE_OPTIONS) $@ 17 | 18 | .PHONY: $(MAKE_TARGETS) 19 | 20 | -------------------------------------------------------------------------------- /patches/0001-iommu-vt-d-Implement-dma_-un-map_resource.patch: -------------------------------------------------------------------------------- 1 | From d54e3e619c66441ac0f446748ab07b00da1c475b Mon Sep 17 00:00:00 2001 2 | From: Logan Gunthorpe 3 | Date: Fri, 18 Jan 2019 17:05:59 -0700 4 | Subject: [PATCH] iommu/vt-d: Implement dma_[un]map_resource() 5 | 6 | Currently the Intel IOMMU uses the default dma_[un]map_resource() 7 | implementations does nothing and simply returns the physical address 8 | unmodified. 9 | 10 | However, this doesn't create the IOVA entries necessary for addresses 11 | mapped this way to work when the IOMMU is enabled. Thus, when the 12 | IOMMU is enabled, drivers relying on dma_map_resource() will trigger 13 | DMAR errors. We see this when running ntb_transport with the IOMMU 14 | enabled, DMA, and switchtec hardware. 15 | 16 | The implementation for intel_map_resource() is nearly identical to 17 | intel_map_page(), we just have to re-create __intel_map_single(). 18 | dma_unmap_resource() uses intel_unmap_page() directly as the 19 | functions are identical. 20 | 21 | Change-Id: Icfc6e4aa07099c02097b0e95ebeafff8e90cab33 22 | Signed-off-by: Logan Gunthorpe 23 | Cc: David Woodhouse 24 | Cc: Joerg Roedel 25 | --- 26 | drivers/iommu/intel-iommu.c | 20 ++++++++++++++++++-- 27 | 1 file changed, 18 insertions(+), 2 deletions(-) 28 | 29 | diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c 30 | index 83f3d48..973e043 100644 31 | --- a/drivers/iommu/intel-iommu.c 32 | +++ b/drivers/iommu/intel-iommu.c 33 | @@ -3647,13 +3647,27 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, 34 | return 0; 35 | } 36 | 37 | +static dma_addr_t __intel_map_page(struct device *dev, struct page *page, 38 | + unsigned long offset, size_t size, int dir, 39 | + u64 dma_mask) 40 | +{ 41 | + return __intel_map_single(dev, page_to_phys(page) + offset, size, dir, 42 | + dma_mask); 43 | +} 44 | + 45 | static dma_addr_t intel_map_page(struct device *dev, struct page *page, 46 | unsigned long offset, size_t size, 47 | enum dma_data_direction dir, 48 | unsigned long attrs) 49 | { 50 | - return __intel_map_single(dev, page_to_phys(page) + offset, size, 51 | - dir, *dev->dma_mask); 52 | + return __intel_map_page(dev, page, offset, size, dir, *dev->dma_mask); 53 | +} 54 | + 55 | +static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr, 56 | + size_t size, enum dma_data_direction dir, 57 | + unsigned long attrs) 58 | +{ 59 | + return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask); 60 | } 61 | 62 | static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size) 63 | @@ -3873,6 +3887,8 @@ static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr) 64 | .unmap_sg = intel_unmap_sg, 65 | .map_page = intel_map_page, 66 | .unmap_page = intel_unmap_page, 67 | + .map_resource = intel_map_resource, 68 | + .unmap_resource = intel_unmap_page, 69 | .mapping_error = intel_mapping_error, 70 | #ifdef CONFIG_X86 71 | .dma_supported = x86_dma_supported, 72 | -- 73 | 1.9.3 74 | 75 | -------------------------------------------------------------------------------- /patches/ntb_no_msix/0001-ntb_hw_intel-add-no_msix-parameter.patch: -------------------------------------------------------------------------------- 1 | From 95663e638c0b57b3a32be76ce38c5bc21a900a2f Mon Sep 17 00:00:00 2001 2 | Message-Id: <95663e638c0b57b3a32be76ce38c5bc21a900a2f.1442936761.git.Allen.Hubbe@emc.com> 3 | From: Allen Hubbe 4 | Date: Tue, 19 May 2015 17:16:31 -0400 5 | Subject: [PATCH] ntb_hw_intel: add no_msix parameter 6 | 7 | Signed-off-by: Allen Hubbe 8 | --- 9 | drivers/ntb/hw/intel/ntb_hw_intel.c | 7 +++++++ 10 | 1 file changed, 7 insertions(+) 11 | 12 | diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c 13 | index 87751cfd6f4f..f217f0bfb5fd 100644 14 | --- a/drivers/ntb/hw/intel/ntb_hw_intel.c 15 | +++ b/drivers/ntb/hw/intel/ntb_hw_intel.c 16 | @@ -91,6 +91,10 @@ static const struct ntb_dev_ops intel_ntb_ops; 17 | static const struct file_operations intel_ntb_debugfs_info; 18 | static struct dentry *debugfs_dir; 19 | 20 | +static int no_msix; 21 | +module_param(no_msix, int, 0644); 22 | +MODULE_PARM_DESC(no_msix, "Do not enable msix interrupts."); 23 | + 24 | static int b2b_mw_idx = -1; 25 | module_param(b2b_mw_idx, int, 0644); 26 | MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A " 27 | @@ -435,6 +439,9 @@ static int ndev_init_isr(struct intel_ntb_dev *ndev, 28 | 29 | /* Try to set up msix irq */ 30 | 31 | + if (no_msix) 32 | + goto err_msix_vec_alloc; 33 | + 34 | ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), 35 | GFP_KERNEL, node); 36 | if (!ndev->vec) 37 | -- 38 | 2.5.0.rc1 39 | 40 | -------------------------------------------------------------------------------- /scripts/getstats.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | 4 | # dma channel stats 5 | echo "dma channels stats (bytes per channel)" 6 | cat /sys/class/dma/dma*chan*/bytes_transferred | nl -w2 -s': ' | awk '$0="dma channel"$0' 7 | 8 | echo "======================================================================" 9 | echo "dma channels stats (memcpy ops per channel)" 10 | cat /sys/class/dma/dma*chan*/memcpy_count | nl -w2 -s': '| awk '$0="dma channel"$0' 11 | echo "======================================================================" 12 | 13 | export CPU_CYC_PER_USEC=$(dmesg | grep tsc | grep ca | grep -Eo '[+-]?[0-9]+([.][0-9]+)?' | sed -n '2p') 14 | 15 | if [ !$CPU_CYC_PER_USEC ]; then 16 | echo "NOTE: dmesg was probably wrapped around, setting to default freq $CPU_CYC_PER_USEC" 17 | export CPU_CYC_PER_USEC=2200 18 | fi 19 | 20 | echo "cpu freq is: $CPU_CYC_PER_USEC" 21 | 22 | USEC_IN_SEC=1000^2 23 | 24 | SHIFT_SAVE_BITS=1024 25 | 26 | for (( ; ; )) 27 | do 28 | a=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep post_send_bytes | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 29 | 30 | poll=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep poll_cq_count | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 31 | 32 | cycles1=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep accum_latency | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 33 | 34 | cqes1=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep cqes_polled | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 35 | 36 | sleep 1; 37 | 38 | b=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep post_send_bytes | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 39 | 40 | poll_next=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep poll_cq_count | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 41 | 42 | cycles2=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep accum_latency | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 43 | 44 | cqes2=$(cat /sys/kernel/debug/ntrdma/ntrdma_0/perf | grep cqes_polled | grep -o -E '[0-9]+' | awk '{s+=$1} END {printf "%.0f\n", s}') 45 | 46 | c=`expr $b - $a` 47 | if [ ${c} -ne 0 ] 48 | then 49 | bw=$(awk "BEGIN {printf \"%.3f\",${c}/1024^3}") 50 | echo "bw: ${bw} GB/s ${c} bytes/sec" 51 | fi 52 | 53 | cycles=`expr $cycles2 - $cycles1` 54 | cqes=`expr $cqes2 - $cqes1` 55 | 56 | if [[ ${cqes} -ne 0 && ${cycles} -ne 0 ]]; then lat=$(awk "BEGIN {printf \"%.3f\",${cycles}*${SHIFT_SAVE_BITS}/($cqes*$CPU_CYC_PER_USEC)}"); echo "latency: $lat usec"; fi 57 | 58 | diff=`expr $poll_next - $poll` 59 | if [[ ${diff} -ne 0 && ${cqes} -ne 0 ]] 60 | then 61 | poll_cyc=$(awk 'BEGIN {printf "%.3f\n", '$USEC_IN_SEC/$diff' }'); echo "user polling : $poll_cyc usec" 62 | fi 63 | 64 | 65 | done 66 | 67 | --------------------------------------------------------------------------------