├── Makefile ├── README ├── install.sh ├── vc_support.h ├── mapper.c ├── vc_support.c └── dmaer.c /Makefile: -------------------------------------------------------------------------------- 1 | ifneq ($(KERNELRELEASE),) 2 | obj-m := dmaer_master.o 3 | dmaer_master-objs := dmaer.o vc_support.o 4 | 5 | else 6 | KDIR ?= /lib/modules/`uname -r`/build 7 | 8 | default: 9 | $(MAKE) -C $(KDIR) M=$$PWD 10 | 11 | endif 12 | 13 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | A Raspberry Pi Linux kernel module that can do user virtual address to bus address translation along with kicking and waiting for DMA. 2 | Also can allocate kernel logical addresses and map them into user space to provide non-swappable memory. -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd "$(dirname "$0")" 3 | 4 | sudo rmmod dmaer_master 5 | sudo insmod dmaer_master.ko || exit 1 6 | sudo rm -f /dev/dmaer_4k 7 | major=$(awk '$2=="dmaer" {print $1}' /proc/devices) 8 | echo device $major 9 | sudo mknod /dev/dmaer_4k c $major 0 10 | -------------------------------------------------------------------------------- /vc_support.h: -------------------------------------------------------------------------------- 1 | #ifndef _VC_SUPPORT_H_ 2 | #define _VC_SUPPORT_H_ 3 | 4 | /* 5 | * vc_support.h 6 | * 7 | * Created on: 25 Nov 2012 8 | * Author: Simon 9 | */ 10 | 11 | enum { 12 | /* 13 | If a MEM_HANDLE_T is discardable, the memory manager may resize it to size 14 | 0 at any time when it is not locked or retained. 15 | */ 16 | MEM_FLAG_DISCARDABLE = 1 << 0, 17 | 18 | /* 19 | If a MEM_HANDLE_T is allocating (or normal), its block of memory will be 20 | accessed in an allocating fashion through the cache. 21 | */ 22 | MEM_FLAG_NORMAL = 0 << 2, 23 | MEM_FLAG_ALLOCATING = MEM_FLAG_NORMAL, 24 | 25 | /* 26 | If a MEM_HANDLE_T is direct, its block of memory will be accessed 27 | directly, bypassing the cache. 28 | */ 29 | MEM_FLAG_DIRECT = 1 << 2, 30 | 31 | /* 32 | If a MEM_HANDLE_T is coherent, its block of memory will be accessed in a 33 | non-allocating fashion through the cache. 34 | */ 35 | MEM_FLAG_COHERENT = 2 << 2, 36 | 37 | /* 38 | If a MEM_HANDLE_T is L1-nonallocating, its block of memory will be accessed by 39 | the VPU in a fashion which is allocating in L2, but only coherent in L1. 40 | */ 41 | MEM_FLAG_L1_NONALLOCATING = (MEM_FLAG_DIRECT | MEM_FLAG_COHERENT), 42 | 43 | /* 44 | If a MEM_HANDLE_T is zero'd, its contents are set to 0 rather than 45 | MEM_HANDLE_INVALID on allocation and resize up. 46 | */ 47 | MEM_FLAG_ZERO = 1 << 4, 48 | 49 | /* 50 | If a MEM_HANDLE_T is uninitialised, it will not be reset to a defined value 51 | (either zero, or all 1's) on allocation. 52 | */ 53 | MEM_FLAG_NO_INIT = 1 << 5, 54 | 55 | /* 56 | Hints. 57 | */ 58 | MEM_FLAG_HINT_PERMALOCK = 1 << 6, /* Likely to be locked for long periods of time. */ 59 | }; 60 | 61 | unsigned int QpuEnable(bool e); 62 | 63 | unsigned int AllocateVcMemory(unsigned int *pHandle, unsigned int size, unsigned int alignment, unsigned int flags); 64 | unsigned int ReleaseVcMemory(unsigned int handle); 65 | unsigned int LockVcMemory(unsigned int *pBusAddress, unsigned int handle); 66 | unsigned int UnlockVcMemory(unsigned int handle); 67 | 68 | unsigned int ExecuteVcCode(unsigned int code, 69 | unsigned int r0, unsigned int r1, unsigned int r2, unsigned int r3, unsigned int r4, unsigned int r5); 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /mapper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * mapper.c -- simple file that mmap()s a file region and prints it 3 | * 4 | * Copyright (C) 1998,2000,2001 Alessandro Rubini 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation; either version 2 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | /***** DEFINES ******/ 34 | /***** DEFINES ******/ 35 | //magic number defining the module 36 | #define DMA_MAGIC 0xdd 37 | 38 | //do user virtual to physical translation of the CB chain 39 | #define DMA_PREPARE _IOWR(DMA_MAGIC, 0, struct DmaControlBlock *) 40 | 41 | //kick the pre-prepared CB chain 42 | #define DMA_KICK _IOW(DMA_MAGIC, 1, struct DmaControlBlock *) 43 | 44 | //prepare it, kick it, wait for it 45 | #define DMA_PREPARE_KICK_WAIT _IOWR(DMA_MAGIC, 2, struct DmaControlBlock *) 46 | 47 | //prepare it, kick it, don't wait for it 48 | #define DMA_PREPARE_KICK _IOWR(DMA_MAGIC, 3, struct DmaControlBlock *) 49 | 50 | //not currently implemented 51 | #define DMA_WAIT_ONE _IO(DMA_MAGIC, 4, struct DmaControlBlock *) 52 | 53 | //wait on all kicked CB chains 54 | #define DMA_WAIT_ALL _IO(DMA_MAGIC, 5) 55 | 56 | //in order to discover the largest AXI burst that should be programmed into the transfer params 57 | #define DMA_MAX_BURST _IO(DMA_MAGIC, 6) 58 | 59 | //set the address range through which the user address is assumed to already by a physical address 60 | #define DMA_SET_MIN_PHYS _IOW(DMA_MAGIC, 7, unsigned long) 61 | #define DMA_SET_MAX_PHYS _IOW(DMA_MAGIC, 8, unsigned long) 62 | 63 | struct DmaControlBlock 64 | { 65 | unsigned int m_transferInfo; 66 | void *m_pSourceAddr; 67 | void *m_pDestAddr; 68 | unsigned int m_xferLen; 69 | unsigned int m_tdStride; 70 | struct DmaControlBlock *m_pNext; 71 | unsigned int m_blank1, m_blank2; 72 | }; 73 | 74 | #define MY_ASSERT(x) if (!(x)) { *(int *)0 = 0; } 75 | 76 | inline void CopyLinear(struct DmaControlBlock *pCB, 77 | void *pDestAddr, void *pSourceAddr, unsigned int length, unsigned int srcInc) 78 | { 79 | MY_ASSERT(pCB); 80 | MY_ASSERT(pDestAddr); 81 | MY_ASSERT(pSourceAddr); 82 | MY_ASSERT(length > 0 && length <= 0x3fffffff); 83 | MY_ASSERT(srcInc == 0 || srcInc == 1); 84 | 85 | if (srcInc) 86 | { 87 | unsigned long source_start = (unsigned long)pSourceAddr >> 12; 88 | unsigned long source_end = (unsigned long)(pSourceAddr + length - 1) >> 12; 89 | 90 | if (source_start != source_end) 91 | { 92 | fprintf(stderr, "linear source range straddles page boundary %p->%p, %lx->%lx\n", 93 | pSourceAddr, pSourceAddr + length, source_start, source_end); 94 | 95 | if (source_end - source_start > 1) 96 | fprintf(stderr, "\tstraddles %ld pages\n", source_end - source_start); 97 | MY_ASSERT(0); 98 | } 99 | } 100 | 101 | unsigned long dest_start = (unsigned long)pDestAddr >> 12; 102 | unsigned long dest_end = (unsigned long)(pDestAddr + length - 1) >> 12; 103 | 104 | if (dest_start != dest_end) 105 | { 106 | fprintf(stderr, "linear dest range straddles page boundary %p->%p, %lx->%lx\n", 107 | pDestAddr, pDestAddr + length, dest_start, dest_end); 108 | 109 | if (dest_end - dest_start > 1) 110 | fprintf(stderr, "\tstraddles %ld pages\n", dest_end - dest_start); 111 | MY_ASSERT(0); 112 | } 113 | 114 | pCB->m_transferInfo = (srcInc << 8) | (1 << 4) | (5 << 12) | (1 << 9) | (1 << 5); 115 | pCB->m_pSourceAddr = pSourceAddr; 116 | pCB->m_pDestAddr = pDestAddr; 117 | pCB->m_xferLen = length; 118 | pCB->m_tdStride = 0xffffffff; 119 | pCB->m_pNext = 0; 120 | pCB->m_blank1 = pCB->m_blank2 = 0; 121 | } 122 | 123 | int main(int argc, char **argv) 124 | { 125 | char *fname; 126 | FILE *f; 127 | void *address; 128 | int err; 129 | 130 | const unsigned int transfer_size = 32 * 1024 * 1024; 131 | 132 | srand(time(0)); 133 | 134 | 135 | 136 | fname=argv[1]; 137 | 138 | if (!(f=fopen(fname,"r+b"))) { 139 | fprintf(stderr, "%s: %s: %s\n", argv[0], fname, strerror(errno)); 140 | exit(1); 141 | } 142 | 143 | address=mmap(0, transfer_size * 2 + 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(f), 0); 144 | 145 | if (address == (void *)-1) { 146 | fprintf(stderr,"%s: mmap(): %s\n",argv[0],strerror(errno)); 147 | exit(1); 148 | } 149 | 150 | memset(address, 0xcd, transfer_size * 2 + 4096); 151 | 152 | unsigned char *pSrc = (unsigned char *)address + 4096; 153 | unsigned char *pDst = pSrc + transfer_size; 154 | int count; 155 | 156 | //if (0) 157 | /*fprintf(stderr, "0\n"); 158 | for (count = 0; count < 16; count++) 159 | { 160 | pSrc[count] = rand(); 161 | pDst[count] = rand(); 162 | 163 | fprintf(stderr, "%d: %d %d\n", count, pSrc[count], pDst[count]); 164 | } 165 | fprintf(stderr, "4096\n"); 166 | for (count = 4096; count < 4096+16; count++) 167 | { 168 | pSrc[count] = rand(); 169 | pDst[count] = rand(); 170 | 171 | fprintf(stderr, "%d: %d %d\n", count, pSrc[count], pDst[count]); 172 | }*/ 173 | 174 | struct DmaControlBlock *pHead = (struct DmaControlBlock *)address; 175 | 176 | for (count = 0; count < transfer_size / 4096; count++) 177 | { 178 | CopyLinear(&pHead[count], &pDst[count * 4096], &pSrc[count * 4096], 4096, 1); 179 | pHead[count].m_pNext = &pHead[count + 1]; 180 | } 181 | pHead[count - 1].m_pNext = 0; 182 | 183 | struct timeval start; 184 | gettimeofday(&start, 0); 185 | 186 | err = ioctl(fileno(f), DMA_PREPARE, address); 187 | if (err == -1) 188 | { 189 | fprintf(stderr, "dma prepare err %d\n", errno); 190 | MY_ASSERT(0); 191 | } 192 | 193 | struct timeval mid; 194 | gettimeofday(&mid, 0); 195 | 196 | err = ioctl(fileno(f), DMA_KICK, address); 197 | if (err == -1) 198 | { 199 | fprintf(stderr, "dma kick err %d\n", errno); 200 | MY_ASSERT(0); 201 | } 202 | 203 | // time_t end = clock(); 204 | 205 | ioctl(fileno(f), DMA_WAIT_ALL, address); 206 | 207 | struct timeval wait; 208 | gettimeofday(&wait, 0); 209 | 210 | fprintf(stderr, "prepare took %d us, kick and wait took %d us\n", 211 | (int)((mid.tv_sec-start.tv_sec)*1000000ULL+(mid.tv_usec-start.tv_usec)), 212 | (int)((wait.tv_sec-mid.tv_sec)*1000000ULL+(wait.tv_usec-mid.tv_usec))); 213 | fprintf(stderr, "prepare %.2f us/CB, k&w took %.2f us/CB\n", 214 | (double)((mid.tv_sec-start.tv_sec)*1000000ULL+(mid.tv_usec-start.tv_usec)) / (transfer_size / 4096), 215 | (double)((wait.tv_sec-mid.tv_sec)*1000000ULL+(wait.tv_usec-mid.tv_usec)) / (transfer_size / 4096)); 216 | fprintf(stderr, "actual %.2f MB/s\n", (double)transfer_size / (int)((wait.tv_sec-mid.tv_sec)*1000000ULL+(wait.tv_usec-mid.tv_usec))); 217 | fprintf(stderr, "aggregate %.2f MB/s\n", (double)transfer_size / (int)((wait.tv_sec-start.tv_sec)*1000000ULL+(wait.tv_usec-start.tv_usec))); 218 | 219 | 220 | 221 | /* fprintf(stderr, "0\n"); 222 | for (count = 0; count < 16; count++) 223 | { 224 | fprintf(stderr, "%d: %d %d\n", count, pSrc[count], pDst[count]); 225 | } 226 | fprintf(stderr, "4096\n"); 227 | for (count = 4096; count < 4096+16; count++) 228 | { 229 | fprintf(stderr, "%d: %d %d\n", count, pSrc[count], pDst[count]); 230 | }*/ 231 | 232 | 233 | fclose(f); 234 | return 0; 235 | } 236 | 237 | -------------------------------------------------------------------------------- /vc_support.c: -------------------------------------------------------------------------------- 1 | /* 2 | * vc_support.c 3 | * 4 | * Created on: 25 Nov 2012 5 | * Author: Simon 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | #ifdef ECLIPSE_IGNORE 12 | 13 | #define __user 14 | #define __init 15 | #define __exit 16 | #define __iomem 17 | #define KERN_DEBUG 18 | #define KERN_ERR 19 | #define KERN_WARNING 20 | #define KERN_INFO 21 | #define _IOWR(a, b, c) b 22 | #define _IOW(a, b, c) b 23 | #define _IO(a, b) b 24 | 25 | #endif 26 | 27 | /****** VC MAILBOX FUNCTIONALITY ******/ 28 | unsigned int QpuEnable(bool e) 29 | { 30 | struct vc_msg 31 | { 32 | unsigned int m_msgSize; 33 | unsigned int m_response; 34 | 35 | struct vc_tag 36 | { 37 | unsigned int m_tagId; 38 | unsigned int m_sendBufferSize; 39 | unsigned int m_sendDataSize; 40 | 41 | struct args 42 | { 43 | unsigned int m_enable; 44 | } m_args; 45 | } m_tag; 46 | unsigned int m_endTag; 47 | } msg; 48 | int s; 49 | 50 | msg.m_msgSize = sizeof(msg); 51 | msg.m_response = 0; 52 | msg.m_endTag = 0; 53 | 54 | //the enable/disable tag 55 | msg.m_tag.m_tagId = 0x30012; 56 | msg.m_tag.m_sendBufferSize = 4; 57 | msg.m_tag.m_sendDataSize = 4; 58 | 59 | //which option we want 60 | msg.m_tag.m_args.m_enable = e; 61 | 62 | //run the command 63 | s = bcm_mailbox_property(&msg, sizeof(msg)); 64 | 65 | if (s == 0 && msg.m_response == 0x80000000) 66 | { 67 | printk(KERN_DEBUG "qpu %s\n", e ? "ENABLED" : "DISABLED"); 68 | return 0; 69 | } 70 | else 71 | { 72 | printk(KERN_ERR "failed to toggle qpu state\n"); 73 | return 1; 74 | } 75 | } 76 | 77 | unsigned int AllocateVcMemory(unsigned int *pHandle, unsigned int size, unsigned int alignment, unsigned int flags) 78 | { 79 | struct vc_msg 80 | { 81 | unsigned int m_msgSize; 82 | unsigned int m_response; 83 | 84 | struct vc_tag 85 | { 86 | unsigned int m_tagId; 87 | unsigned int m_sendBufferSize; 88 | union { 89 | unsigned int m_sendDataSize; 90 | unsigned int m_recvDataSize; 91 | }; 92 | 93 | struct args 94 | { 95 | union { 96 | unsigned int m_size; 97 | unsigned int m_handle; 98 | }; 99 | unsigned int m_alignment; 100 | unsigned int m_flags; 101 | } m_args; 102 | } m_tag; 103 | 104 | unsigned int m_endTag; 105 | } msg; 106 | int s; 107 | 108 | msg.m_msgSize = sizeof(msg); 109 | msg.m_response = 0; 110 | msg.m_endTag = 0; 111 | 112 | //fill in the tag for the allocation command 113 | msg.m_tag.m_tagId = 0x3000c; 114 | msg.m_tag.m_sendBufferSize = 12; 115 | msg.m_tag.m_sendDataSize = 12; 116 | 117 | //fill in our args 118 | msg.m_tag.m_args.m_size = size; 119 | msg.m_tag.m_args.m_alignment = alignment; 120 | msg.m_tag.m_args.m_flags = flags; 121 | 122 | //run the command 123 | s = bcm_mailbox_property(&msg, sizeof(msg)); 124 | 125 | if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004) 126 | { 127 | *pHandle = msg.m_tag.m_args.m_handle; 128 | return 0; 129 | } 130 | else 131 | { 132 | printk(KERN_ERR "failed to allocate vc memory: s=%d response=%08x recv data size=%08x\n", 133 | s, msg.m_response, msg.m_tag.m_recvDataSize); 134 | return 1; 135 | } 136 | } 137 | 138 | unsigned int ReleaseVcMemory(unsigned int handle) 139 | { 140 | struct vc_msg 141 | { 142 | unsigned int m_msgSize; 143 | unsigned int m_response; 144 | 145 | struct vc_tag 146 | { 147 | unsigned int m_tagId; 148 | unsigned int m_sendBufferSize; 149 | union { 150 | unsigned int m_sendDataSize; 151 | unsigned int m_recvDataSize; 152 | }; 153 | 154 | struct args 155 | { 156 | union { 157 | unsigned int m_handle; 158 | unsigned int m_error; 159 | }; 160 | } m_args; 161 | } m_tag; 162 | 163 | unsigned int m_endTag; 164 | } msg; 165 | int s; 166 | 167 | msg.m_msgSize = sizeof(msg); 168 | msg.m_response = 0; 169 | msg.m_endTag = 0; 170 | 171 | //fill in the tag for the release command 172 | msg.m_tag.m_tagId = 0x3000f; 173 | msg.m_tag.m_sendBufferSize = 4; 174 | msg.m_tag.m_sendDataSize = 4; 175 | 176 | //pass across the handle 177 | msg.m_tag.m_args.m_handle = handle; 178 | 179 | s = bcm_mailbox_property(&msg, sizeof(msg)); 180 | 181 | if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004 && msg.m_tag.m_args.m_error == 0) 182 | return 0; 183 | else 184 | { 185 | printk(KERN_ERR "failed to release vc memory: s=%d response=%08x recv data size=%08x error=%08x\n", 186 | s, msg.m_response, msg.m_tag.m_recvDataSize, msg.m_tag.m_args.m_error); 187 | return 1; 188 | } 189 | } 190 | 191 | unsigned int LockVcMemory(unsigned int *pBusAddress, unsigned int handle) 192 | { 193 | struct vc_msg 194 | { 195 | unsigned int m_msgSize; 196 | unsigned int m_response; 197 | 198 | struct vc_tag 199 | { 200 | unsigned int m_tagId; 201 | unsigned int m_sendBufferSize; 202 | union { 203 | unsigned int m_sendDataSize; 204 | unsigned int m_recvDataSize; 205 | }; 206 | 207 | struct args 208 | { 209 | union { 210 | unsigned int m_handle; 211 | unsigned int m_busAddress; 212 | }; 213 | } m_args; 214 | } m_tag; 215 | 216 | unsigned int m_endTag; 217 | } msg; 218 | int s; 219 | 220 | msg.m_msgSize = sizeof(msg); 221 | msg.m_response = 0; 222 | msg.m_endTag = 0; 223 | 224 | //fill in the tag for the lock command 225 | msg.m_tag.m_tagId = 0x3000d; 226 | msg.m_tag.m_sendBufferSize = 4; 227 | msg.m_tag.m_sendDataSize = 4; 228 | 229 | //pass across the handle 230 | msg.m_tag.m_args.m_handle = handle; 231 | 232 | s = bcm_mailbox_property(&msg, sizeof(msg)); 233 | 234 | if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004) 235 | { 236 | //pick out the bus address 237 | *pBusAddress = msg.m_tag.m_args.m_busAddress; 238 | return 0; 239 | } 240 | else 241 | { 242 | printk(KERN_ERR "failed to lock vc memory: s=%d response=%08x recv data size=%08x\n", 243 | s, msg.m_response, msg.m_tag.m_recvDataSize); 244 | return 1; 245 | } 246 | } 247 | 248 | unsigned int UnlockVcMemory(unsigned int handle) 249 | { 250 | struct vc_msg 251 | { 252 | unsigned int m_msgSize; 253 | unsigned int m_response; 254 | 255 | struct vc_tag 256 | { 257 | unsigned int m_tagId; 258 | unsigned int m_sendBufferSize; 259 | union { 260 | unsigned int m_sendDataSize; 261 | unsigned int m_recvDataSize; 262 | }; 263 | 264 | struct args 265 | { 266 | union { 267 | unsigned int m_handle; 268 | unsigned int m_error; 269 | }; 270 | } m_args; 271 | } m_tag; 272 | 273 | unsigned int m_endTag; 274 | } msg; 275 | int s; 276 | 277 | msg.m_msgSize = sizeof(msg); 278 | msg.m_response = 0; 279 | msg.m_endTag = 0; 280 | 281 | //fill in the tag for the unlock command 282 | msg.m_tag.m_tagId = 0x3000e; 283 | msg.m_tag.m_sendBufferSize = 4; 284 | msg.m_tag.m_sendDataSize = 4; 285 | 286 | //pass across the handle 287 | msg.m_tag.m_args.m_handle = handle; 288 | 289 | s = bcm_mailbox_property(&msg, sizeof(msg)); 290 | 291 | //check the error code too 292 | if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004 && msg.m_tag.m_args.m_error == 0) 293 | return 0; 294 | else 295 | { 296 | printk(KERN_ERR "failed to unlock vc memory: s=%d response=%08x recv data size=%08x error%08x\n", 297 | s, msg.m_response, msg.m_tag.m_recvDataSize, msg.m_tag.m_args.m_error); 298 | return 1; 299 | } 300 | } 301 | 302 | unsigned int ExecuteVcCode(unsigned int code, 303 | unsigned int r0, unsigned int r1, unsigned int r2, unsigned int r3, unsigned int r4, unsigned int r5) 304 | { 305 | struct vc_msg 306 | { 307 | unsigned int m_msgSize; 308 | unsigned int m_response; 309 | 310 | struct vc_tag 311 | { 312 | unsigned int m_tagId; 313 | unsigned int m_sendBufferSize; 314 | union { 315 | unsigned int m_sendDataSize; 316 | unsigned int m_recvDataSize; 317 | }; 318 | 319 | struct args 320 | { 321 | union { 322 | unsigned int m_pCode; 323 | unsigned int m_return; 324 | }; 325 | unsigned int m_r0; 326 | unsigned int m_r1; 327 | unsigned int m_r2; 328 | unsigned int m_r3; 329 | unsigned int m_r4; 330 | unsigned int m_r5; 331 | } m_args; 332 | } m_tag; 333 | 334 | unsigned int m_endTag; 335 | } msg; 336 | int s; 337 | 338 | msg.m_msgSize = sizeof(msg); 339 | msg.m_response = 0; 340 | msg.m_endTag = 0; 341 | 342 | //fill in the tag for the unlock command 343 | msg.m_tag.m_tagId = 0x30010; 344 | msg.m_tag.m_sendBufferSize = 28; 345 | msg.m_tag.m_sendDataSize = 28; 346 | 347 | //pass across the handle 348 | msg.m_tag.m_args.m_pCode = code; 349 | msg.m_tag.m_args.m_r0 = r0; 350 | msg.m_tag.m_args.m_r1 = r1; 351 | msg.m_tag.m_args.m_r2 = r2; 352 | msg.m_tag.m_args.m_r3 = r3; 353 | msg.m_tag.m_args.m_r4 = r4; 354 | msg.m_tag.m_args.m_r5 = r5; 355 | 356 | s = bcm_mailbox_property(&msg, sizeof(msg)); 357 | 358 | //check the error code too 359 | if (s == 0 && msg.m_response == 0x80000000 && msg.m_tag.m_recvDataSize == 0x80000004) 360 | return msg.m_tag.m_args.m_return; 361 | else 362 | { 363 | printk(KERN_ERR "failed to execute: s=%d response=%08x recv data size=%08x\n", 364 | s, msg.m_response, msg.m_tag.m_recvDataSize); 365 | return 1; 366 | } 367 | } 368 | 369 | -------------------------------------------------------------------------------- /dmaer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include "vc_support.h" 24 | 25 | #ifdef ECLIPSE_IGNORE 26 | 27 | #define __user 28 | #define __init 29 | #define __exit 30 | #define __iomem 31 | #define KERN_DEBUG 32 | #define KERN_ERR 33 | #define KERN_WARNING 34 | #define KERN_INFO 35 | #define _IOWR(a, b, c) b 36 | #define _IOW(a, b, c) b 37 | #define _IO(a, b) b 38 | 39 | #endif 40 | 41 | //#define inline 42 | 43 | #define PRINTK(args...) printk(args) 44 | //#define PRINTK_VERBOSE(args...) printk(args) 45 | //#define PRINTK(args...) 46 | #define PRINTK_VERBOSE(args...) 47 | 48 | /***** TYPES ****/ 49 | #define PAGES_PER_LIST 500 50 | struct PageList 51 | { 52 | struct page *m_pPages[PAGES_PER_LIST]; 53 | unsigned int m_used; 54 | struct PageList *m_pNext; 55 | }; 56 | 57 | struct VmaPageList 58 | { 59 | //each vma has a linked list of pages associated with it 60 | struct PageList *m_pPageHead; 61 | struct PageList *m_pPageTail; 62 | unsigned int m_refCount; 63 | }; 64 | 65 | struct DmaControlBlock 66 | { 67 | unsigned int m_transferInfo; 68 | void __user *m_pSourceAddr; 69 | void __user *m_pDestAddr; 70 | unsigned int m_xferLen; 71 | unsigned int m_tdStride; 72 | struct DmaControlBlock *m_pNext; 73 | unsigned int m_blank1, m_blank2; 74 | }; 75 | 76 | /***** DEFINES ******/ 77 | //magic number defining the module 78 | #define DMA_MAGIC 0xdd 79 | 80 | //do user virtual to physical translation of the CB chain 81 | #define DMA_PREPARE _IOWR(DMA_MAGIC, 0, struct DmaControlBlock *) 82 | 83 | //kick the pre-prepared CB chain 84 | #define DMA_KICK _IOW(DMA_MAGIC, 1, struct DmaControlBlock *) 85 | 86 | //prepare it, kick it, wait for it 87 | #define DMA_PREPARE_KICK_WAIT _IOWR(DMA_MAGIC, 2, struct DmaControlBlock *) 88 | 89 | //prepare it, kick it, don't wait for it 90 | #define DMA_PREPARE_KICK _IOWR(DMA_MAGIC, 3, struct DmaControlBlock *) 91 | 92 | //not currently implemented 93 | #define DMA_WAIT_ONE _IO(DMA_MAGIC, 4, struct DmaControlBlock *) 94 | 95 | //wait on all kicked CB chains 96 | #define DMA_WAIT_ALL _IO(DMA_MAGIC, 5) 97 | 98 | //in order to discover the largest AXI burst that should be programmed into the transfer params 99 | #define DMA_MAX_BURST _IO(DMA_MAGIC, 6) 100 | 101 | //set the address range through which the user address is assumed to already by a physical address 102 | #define DMA_SET_MIN_PHYS _IOW(DMA_MAGIC, 7, unsigned long) 103 | #define DMA_SET_MAX_PHYS _IOW(DMA_MAGIC, 8, unsigned long) 104 | #define DMA_SET_PHYS_OFFSET _IOW(DMA_MAGIC, 9, unsigned long) 105 | 106 | //used to define the size for the CMA-based allocation *in pages*, can only be done once once the file is opened 107 | #define DMA_CMA_SET_SIZE _IOW(DMA_MAGIC, 10, unsigned long) 108 | 109 | //used to get the version of the module, to test for a capability 110 | #define DMA_GET_VERSION _IO(DMA_MAGIC, 99) 111 | 112 | #define VERSION_NUMBER 1 113 | 114 | #define VIRT_TO_BUS_CACHE_SIZE 8 115 | 116 | /***** FILE OPS *****/ 117 | static int Open(struct inode *pInode, struct file *pFile); 118 | static int Release(struct inode *pInode, struct file *pFile); 119 | static long Ioctl(struct file *pFile, unsigned int cmd, unsigned long arg); 120 | static ssize_t Read(struct file *pFile, char __user *pUser, size_t count, loff_t *offp); 121 | static int Mmap(struct file *pFile, struct vm_area_struct *pVma); 122 | 123 | /***** VMA OPS ****/ 124 | static void VmaOpen4k(struct vm_area_struct *pVma); 125 | static void VmaClose4k(struct vm_area_struct *pVma); 126 | static int VmaFault4k(struct vm_area_struct *pVma, struct vm_fault *pVmf); 127 | 128 | /**** DMA PROTOTYPES */ 129 | static struct DmaControlBlock __user *DmaPrepare(struct DmaControlBlock __user *pUserCB, int *pError); 130 | static int DmaKick(struct DmaControlBlock __user *pUserCB); 131 | static void DmaWaitAll(void); 132 | 133 | /**** GENERIC ****/ 134 | static int __init dmaer_init(void); 135 | static void __exit dmaer_exit(void); 136 | 137 | /*** OPS ***/ 138 | static struct vm_operations_struct g_vmOps4k = { 139 | .open = VmaOpen4k, 140 | .close = VmaClose4k, 141 | .fault = VmaFault4k, 142 | }; 143 | 144 | static struct file_operations g_fOps = { 145 | .owner = THIS_MODULE, 146 | .llseek = 0, 147 | .read = Read, 148 | .write = 0, 149 | .unlocked_ioctl = Ioctl, 150 | .open = Open, 151 | .release = Release, 152 | .mmap = Mmap, 153 | }; 154 | 155 | /***** GLOBALS ******/ 156 | static dev_t g_majorMinor; 157 | 158 | //tracking usage of the two files 159 | static atomic_t g_oneLock4k = ATOMIC_INIT(1); 160 | 161 | //device operations 162 | static struct cdev g_cDev; 163 | static int g_trackedPages = 0; 164 | 165 | //dma control 166 | static unsigned int *g_pDmaChanBase; 167 | static int g_dmaIrq; 168 | static int g_dmaChan; 169 | 170 | //cma allocation 171 | static int g_cmaHandle; 172 | 173 | //user virtual to bus address translation acceleration 174 | static unsigned long g_virtAddr[VIRT_TO_BUS_CACHE_SIZE]; 175 | static unsigned long g_busAddr[VIRT_TO_BUS_CACHE_SIZE]; 176 | static unsigned long g_cbVirtAddr; 177 | static unsigned long g_cbBusAddr; 178 | static int g_cacheInsertAt; 179 | static int g_cacheHit, g_cacheMiss; 180 | 181 | //off by default 182 | static void __user *g_pMinPhys; 183 | static void __user *g_pMaxPhys; 184 | static unsigned long g_physOffset; 185 | 186 | /****** CACHE OPERATIONS ********/ 187 | static inline void FlushAddrCache(void) 188 | { 189 | int count = 0; 190 | for (count = 0; count < VIRT_TO_BUS_CACHE_SIZE; count++) 191 | g_virtAddr[count] = 0xffffffff; //never going to match as we always chop the bottom bits anyway 192 | 193 | g_cbVirtAddr = 0xffffffff; 194 | 195 | g_cacheInsertAt = 0; 196 | } 197 | 198 | //translate from a user virtual address to a bus address by mapping the page 199 | //NB this won't lock a page in memory, so to avoid potential paging issues using kernel logical addresses 200 | static inline void __iomem *UserVirtualToBus(void __user *pUser) 201 | { 202 | int mapped; 203 | struct page *pPage; 204 | void *phys; 205 | 206 | //map it (requiring that the pointer points to something that does not hang off the page boundary) 207 | mapped = get_user_pages(current, current->mm, 208 | (unsigned long)pUser, 1, 209 | 1, 0, 210 | &pPage, 211 | 0); 212 | 213 | if (mapped <= 0) //error 214 | return 0; 215 | 216 | PRINTK_VERBOSE(KERN_DEBUG "user virtual %p arm phys %p bus %p\n", 217 | pUser, page_address(pPage), (void __iomem *)__virt_to_bus(page_address(pPage))); 218 | 219 | //get the arm physical address 220 | phys = page_address(pPage) + offset_in_page(pUser); 221 | page_cache_release(pPage); 222 | 223 | //and now the bus address 224 | return (void __iomem *)__virt_to_bus(phys); 225 | } 226 | 227 | static inline void __iomem *UserVirtualToBusViaCbCache(void __user *pUser) 228 | { 229 | unsigned long virtual_page = (unsigned long)pUser & ~4095; 230 | unsigned long page_offset = (unsigned long)pUser & 4095; 231 | unsigned long bus_addr; 232 | 233 | if (g_cbVirtAddr == virtual_page) 234 | { 235 | bus_addr = g_cbBusAddr + page_offset; 236 | g_cacheHit++; 237 | return (void __iomem *)bus_addr; 238 | } 239 | else 240 | { 241 | bus_addr = (unsigned long)UserVirtualToBus(pUser); 242 | 243 | if (!bus_addr) 244 | return 0; 245 | 246 | g_cbVirtAddr = virtual_page; 247 | g_cbBusAddr = bus_addr & ~4095; 248 | g_cacheMiss++; 249 | 250 | return (void __iomem *)bus_addr; 251 | } 252 | } 253 | 254 | //do the same as above, by query our virt->bus cache 255 | static inline void __iomem *UserVirtualToBusViaCache(void __user *pUser) 256 | { 257 | int count; 258 | //get the page and its offset 259 | unsigned long virtual_page = (unsigned long)pUser & ~4095; 260 | unsigned long page_offset = (unsigned long)pUser & 4095; 261 | unsigned long bus_addr; 262 | 263 | if (pUser >= g_pMinPhys && pUser < g_pMaxPhys) 264 | { 265 | PRINTK_VERBOSE(KERN_DEBUG "user->phys passthrough on %p\n", pUser); 266 | return (void __iomem *)((unsigned long)pUser + g_physOffset); 267 | } 268 | 269 | //check the cache for our entry 270 | for (count = 0; count < VIRT_TO_BUS_CACHE_SIZE; count++) 271 | if (g_virtAddr[count] == virtual_page) 272 | { 273 | bus_addr = g_busAddr[count] + page_offset; 274 | g_cacheHit++; 275 | return (void __iomem *)bus_addr; 276 | } 277 | 278 | //not found, look up manually and then insert its page address 279 | bus_addr = (unsigned long)UserVirtualToBus(pUser); 280 | 281 | if (!bus_addr) 282 | return 0; 283 | 284 | g_virtAddr[g_cacheInsertAt] = virtual_page; 285 | g_busAddr[g_cacheInsertAt] = bus_addr & ~4095; 286 | 287 | //round robin 288 | g_cacheInsertAt++; 289 | if (g_cacheInsertAt == VIRT_TO_BUS_CACHE_SIZE) 290 | g_cacheInsertAt = 0; 291 | 292 | g_cacheMiss++; 293 | 294 | return (void __iomem *)bus_addr; 295 | } 296 | 297 | /***** FILE OPERATIONS ****/ 298 | static int Open(struct inode *pInode, struct file *pFile) 299 | { 300 | PRINTK(KERN_DEBUG "file opening: %d/%d\n", imajor(pInode), iminor(pInode)); 301 | 302 | //check which device we are 303 | if (iminor(pInode) == 0) //4k 304 | { 305 | //only one at a time 306 | if (!atomic_dec_and_test(&g_oneLock4k)) 307 | { 308 | atomic_inc(&g_oneLock4k); 309 | return -EBUSY; 310 | } 311 | } 312 | else 313 | return -EINVAL; 314 | 315 | //todo there will be trouble if two different processes open the files 316 | 317 | //reset after any file is opened 318 | g_pMinPhys = (void __user *)-1; 319 | g_pMaxPhys = (void __user *)0; 320 | g_physOffset = 0; 321 | g_cmaHandle = 0; 322 | 323 | return 0; 324 | } 325 | 326 | static int Release(struct inode *pInode, struct file *pFile) 327 | { 328 | PRINTK(KERN_DEBUG "file closing, %d pages tracked\n", g_trackedPages); 329 | if (g_trackedPages) 330 | PRINTK(KERN_ERR "we\'re leaking memory!\n"); 331 | 332 | //wait for any dmas to finish 333 | DmaWaitAll(); 334 | 335 | //free this memory on the application closing the file or it crashing (implicitly closing the file) 336 | if (g_cmaHandle) 337 | { 338 | PRINTK(KERN_DEBUG "unlocking vc memory\n"); 339 | if (UnlockVcMemory(g_cmaHandle)) 340 | PRINTK(KERN_ERR "uh-oh, unable to unlock vc memory!\n"); 341 | PRINTK(KERN_DEBUG "releasing vc memory\n"); 342 | if (ReleaseVcMemory(g_cmaHandle)) 343 | PRINTK(KERN_ERR "uh-oh, unable to release vc memory!\n"); 344 | } 345 | 346 | if (iminor(pInode) == 0) 347 | atomic_inc(&g_oneLock4k); 348 | else 349 | return -EINVAL; 350 | 351 | return 0; 352 | } 353 | 354 | static struct DmaControlBlock __user *DmaPrepare(struct DmaControlBlock __user *pUserCB, int *pError) 355 | { 356 | struct DmaControlBlock kernCB; 357 | struct DmaControlBlock __user *pUNext; 358 | void __iomem *pSourceBus, __iomem *pDestBus; 359 | 360 | //get the control block into kernel memory so we can work on it 361 | if (copy_from_user(&kernCB, pUserCB, sizeof(struct DmaControlBlock)) != 0) 362 | { 363 | PRINTK(KERN_ERR "copy_from_user failed for user cb %p\n", pUserCB); 364 | *pError = 1; 365 | return 0; 366 | } 367 | 368 | if (kernCB.m_pSourceAddr == 0 || kernCB.m_pDestAddr == 0) 369 | { 370 | PRINTK(KERN_ERR "faulty source (%p) dest (%p) addresses for user cb %p\n", 371 | kernCB.m_pSourceAddr, kernCB.m_pDestAddr, pUserCB); 372 | *pError = 1; 373 | return 0; 374 | } 375 | 376 | pSourceBus = UserVirtualToBusViaCache(kernCB.m_pSourceAddr); 377 | pDestBus = UserVirtualToBusViaCache(kernCB.m_pDestAddr); 378 | 379 | if (!pSourceBus || !pDestBus) 380 | { 381 | PRINTK(KERN_ERR "virtual to bus translation failure for source/dest %p/%p->%p/%p\n", 382 | kernCB.m_pSourceAddr, kernCB.m_pDestAddr, 383 | pSourceBus, pDestBus); 384 | *pError = 1; 385 | return 0; 386 | } 387 | 388 | //update the user structure with the new bus addresses 389 | kernCB.m_pSourceAddr = pSourceBus; 390 | kernCB.m_pDestAddr = pDestBus; 391 | 392 | PRINTK_VERBOSE(KERN_DEBUG "final source %p dest %p\n", kernCB.m_pSourceAddr, kernCB.m_pDestAddr); 393 | 394 | //sort out the bus address for the next block 395 | pUNext = kernCB.m_pNext; 396 | 397 | if (kernCB.m_pNext) 398 | { 399 | void __iomem *pNextBus; 400 | pNextBus = UserVirtualToBusViaCbCache(kernCB.m_pNext); 401 | 402 | if (!pNextBus) 403 | { 404 | PRINTK(KERN_ERR "virtual to bus translation failure for m_pNext\n"); 405 | *pError = 1; 406 | return 0; 407 | } 408 | 409 | //update the pointer with the bus address 410 | kernCB.m_pNext = pNextBus; 411 | } 412 | 413 | //write it back to user space 414 | if (copy_to_user(pUserCB, &kernCB, sizeof(struct DmaControlBlock)) != 0) 415 | { 416 | PRINTK(KERN_ERR "copy_to_user failed for cb %p\n", pUserCB); 417 | *pError = 1; 418 | return 0; 419 | } 420 | 421 | __cpuc_flush_dcache_area(pUserCB, 32); 422 | 423 | *pError = 0; 424 | return pUNext; 425 | } 426 | 427 | static int DmaKick(struct DmaControlBlock __user *pUserCB) 428 | { 429 | void __iomem *pBusCB; 430 | 431 | pBusCB = UserVirtualToBusViaCbCache(pUserCB); 432 | if (!pBusCB) 433 | { 434 | PRINTK(KERN_ERR "virtual to bus translation failure for cb\n"); 435 | return 1; 436 | } 437 | 438 | //flush_cache_all(); 439 | 440 | bcm_dma_start(g_pDmaChanBase, (dma_addr_t)pBusCB); 441 | 442 | return 0; 443 | } 444 | 445 | static void DmaWaitAll(void) 446 | { 447 | int counter = 0; 448 | volatile int inner_count; 449 | volatile unsigned int cs; 450 | unsigned long time_before, time_after; 451 | 452 | time_before = jiffies; 453 | //bcm_dma_wait_idle(g_pDmaChanBase); 454 | dsb(); 455 | 456 | cs = readl(g_pDmaChanBase); 457 | 458 | while ((cs & 1) == 1) 459 | { 460 | cs = readl(g_pDmaChanBase); 461 | counter++; 462 | 463 | for (inner_count = 0; inner_count < 32; inner_count++); 464 | 465 | asm volatile ("MCR p15,0,r0,c7,c0,4 \n"); 466 | //cpu_do_idle(); 467 | if (counter >= 1000000) 468 | { 469 | PRINTK(KERN_WARNING "DMA failed to finish in a timely fashion\n"); 470 | break; 471 | } 472 | } 473 | time_after = jiffies; 474 | PRINTK_VERBOSE(KERN_DEBUG "done, counter %d, cs %08x", counter, cs); 475 | PRINTK_VERBOSE(KERN_DEBUG "took %ld jiffies, %d HZ\n", time_after - time_before, HZ); 476 | } 477 | 478 | static long Ioctl(struct file *pFile, unsigned int cmd, unsigned long arg) 479 | { 480 | int error = 0; 481 | PRINTK_VERBOSE(KERN_DEBUG "ioctl cmd %x arg %lx\n", cmd, arg); 482 | 483 | switch (cmd) 484 | { 485 | case DMA_PREPARE: 486 | case DMA_PREPARE_KICK: 487 | case DMA_PREPARE_KICK_WAIT: 488 | { 489 | struct DmaControlBlock __user *pUCB = (struct DmaControlBlock *)arg; 490 | int steps = 0; 491 | unsigned long start_time = jiffies; 492 | (void)start_time; 493 | 494 | //flush our address cache 495 | FlushAddrCache(); 496 | 497 | PRINTK_VERBOSE(KERN_DEBUG "dma prepare\n"); 498 | 499 | //do virtual to bus translation for each entry 500 | do 501 | { 502 | pUCB = DmaPrepare(pUCB, &error); 503 | } while (error == 0 && ++steps && pUCB); 504 | PRINTK_VERBOSE(KERN_DEBUG "prepare done in %d steps, %ld\n", steps, jiffies - start_time); 505 | 506 | //carry straight on if we want to kick too 507 | if (cmd == DMA_PREPARE || error) 508 | { 509 | PRINTK_VERBOSE(KERN_DEBUG "falling out\n"); 510 | return error ? -EINVAL : 0; 511 | } 512 | } 513 | case DMA_KICK: 514 | PRINTK_VERBOSE(KERN_DEBUG "dma begin\n"); 515 | 516 | if (cmd == DMA_KICK) 517 | FlushAddrCache(); 518 | 519 | DmaKick((struct DmaControlBlock __user *)arg); 520 | 521 | if (cmd != DMA_PREPARE_KICK_WAIT) 522 | break; 523 | /* case DMA_WAIT_ONE: 524 | //PRINTK(KERN_DEBUG "dma wait one\n"); 525 | break;*/ 526 | case DMA_WAIT_ALL: 527 | //PRINTK(KERN_DEBUG "dma wait all\n"); 528 | DmaWaitAll(); 529 | break; 530 | case DMA_MAX_BURST: 531 | if (g_dmaChan == 0) 532 | return 10; 533 | else 534 | return 5; 535 | case DMA_SET_MIN_PHYS: 536 | g_pMinPhys = (void __user *)arg; 537 | PRINTK(KERN_DEBUG "min/max user/phys bypass set to %p %p\n", g_pMinPhys, g_pMaxPhys); 538 | break; 539 | case DMA_SET_MAX_PHYS: 540 | g_pMaxPhys = (void __user *)arg; 541 | PRINTK(KERN_DEBUG "min/max user/phys bypass set to %p %p\n", g_pMinPhys, g_pMaxPhys); 542 | break; 543 | case DMA_SET_PHYS_OFFSET: 544 | g_physOffset = arg; 545 | PRINTK(KERN_DEBUG "user/phys bypass offset set to %ld\n", g_physOffset); 546 | break; 547 | case DMA_CMA_SET_SIZE: 548 | { 549 | unsigned int pBusAddr; 550 | 551 | if (g_cmaHandle) 552 | { 553 | PRINTK(KERN_ERR "memory has already been allocated (handle %d)\n", g_cmaHandle); 554 | return -EINVAL; 555 | } 556 | 557 | PRINTK(KERN_INFO "allocating %ld bytes of VC memory\n", arg * 4096); 558 | 559 | //get the memory 560 | if (AllocateVcMemory(&g_cmaHandle, arg * 4096, 4096, MEM_FLAG_L1_NONALLOCATING | MEM_FLAG_NO_INIT | MEM_FLAG_HINT_PERMALOCK)) 561 | { 562 | PRINTK(KERN_ERR "failed to allocate %ld bytes of VC memory\n", arg * 4096); 563 | g_cmaHandle = 0; 564 | return -EINVAL; 565 | } 566 | 567 | //get an address for it 568 | PRINTK(KERN_INFO "trying to map VC memory\n"); 569 | 570 | if (LockVcMemory(&pBusAddr, g_cmaHandle)) 571 | { 572 | PRINTK(KERN_ERR "failed to map CMA handle %d, releasing memory\n", g_cmaHandle); 573 | ReleaseVcMemory(g_cmaHandle); 574 | g_cmaHandle = 0; 575 | } 576 | 577 | PRINTK(KERN_INFO "bus address for CMA memory is %x\n", pBusAddr); 578 | return pBusAddr; 579 | } 580 | case DMA_GET_VERSION: 581 | PRINTK(KERN_DEBUG "returning version number, %d\n", VERSION_NUMBER); 582 | return VERSION_NUMBER; 583 | default: 584 | PRINTK(KERN_DEBUG "unknown ioctl: %d\n", cmd); 585 | return -EINVAL; 586 | } 587 | 588 | return 0; 589 | } 590 | 591 | static ssize_t Read(struct file *pFile, char __user *pUser, size_t count, loff_t *offp) 592 | { 593 | return -EIO; 594 | } 595 | 596 | static int Mmap(struct file *pFile, struct vm_area_struct *pVma) 597 | { 598 | struct PageList *pPages; 599 | struct VmaPageList *pVmaList; 600 | 601 | PRINTK_VERBOSE(KERN_DEBUG "MMAP vma %p, length %ld (%s %d)\n", 602 | pVma, pVma->vm_end - pVma->vm_start, 603 | current->comm, current->pid); 604 | PRINTK_VERBOSE(KERN_DEBUG "MMAP %p %d (tracked %d)\n", pVma, current->pid, g_trackedPages); 605 | 606 | //make a new page list 607 | pPages = (struct PageList *)kmalloc(sizeof(struct PageList), GFP_KERNEL); 608 | if (!pPages) 609 | { 610 | PRINTK(KERN_ERR "couldn\'t allocate a new page list (%s %d)\n", 611 | current->comm, current->pid); 612 | return -ENOMEM; 613 | } 614 | 615 | //clear the page list 616 | pPages->m_used = 0; 617 | pPages->m_pNext = 0; 618 | 619 | //insert our vma and new page list somewhere 620 | if (!pVma->vm_private_data) 621 | { 622 | struct VmaPageList *pList; 623 | 624 | PRINTK_VERBOSE(KERN_DEBUG "new vma list, making new one (%s %d)\n", 625 | current->comm, current->pid); 626 | 627 | //make a new vma list 628 | pList = (struct VmaPageList *)kmalloc(sizeof(struct VmaPageList), GFP_KERNEL); 629 | if (!pList) 630 | { 631 | PRINTK(KERN_ERR "couldn\'t allocate vma page list (%s %d)\n", 632 | current->comm, current->pid); 633 | kfree(pPages); 634 | return -ENOMEM; 635 | } 636 | 637 | //clear this list 638 | pVma->vm_private_data = (void *)pList; 639 | pList->m_refCount = 0; 640 | } 641 | 642 | pVmaList = (struct VmaPageList *)pVma->vm_private_data; 643 | 644 | //add it to the vma list 645 | pVmaList->m_pPageHead = pPages; 646 | pVmaList->m_pPageTail = pPages; 647 | 648 | pVma->vm_ops = &g_vmOps4k; 649 | pVma->vm_flags |= VM_RESERVED; 650 | 651 | VmaOpen4k(pVma); 652 | 653 | return 0; 654 | } 655 | 656 | /****** VMA OPERATIONS ******/ 657 | 658 | static void VmaOpen4k(struct vm_area_struct *pVma) 659 | { 660 | struct VmaPageList *pVmaList; 661 | 662 | PRINTK_VERBOSE(KERN_DEBUG "vma open %p private %p (%s %d), %d live pages\n", pVma, pVma->vm_private_data, current->comm, current->pid, g_trackedPages); 663 | PRINTK_VERBOSE(KERN_DEBUG "OPEN %p %d %ld pages (tracked pages %d)\n", 664 | pVma, current->pid, (pVma->vm_end - pVma->vm_start) >> 12, 665 | g_trackedPages); 666 | 667 | pVmaList = (struct VmaPageList *)pVma->vm_private_data; 668 | 669 | if (pVmaList) 670 | { 671 | pVmaList->m_refCount++; 672 | PRINTK_VERBOSE(KERN_DEBUG "ref count is now %d\n", pVmaList->m_refCount); 673 | } 674 | else 675 | { 676 | PRINTK_VERBOSE(KERN_DEBUG "err, open but no vma page list\n"); 677 | } 678 | } 679 | 680 | static void VmaClose4k(struct vm_area_struct *pVma) 681 | { 682 | struct VmaPageList *pVmaList; 683 | int freed = 0; 684 | 685 | PRINTK_VERBOSE(KERN_DEBUG "vma close %p private %p (%s %d)\n", pVma, pVma->vm_private_data, current->comm, current->pid); 686 | 687 | //wait for any dmas to finish 688 | DmaWaitAll(); 689 | 690 | //find our vma in the list 691 | pVmaList = (struct VmaPageList *)pVma->vm_private_data; 692 | 693 | //may be a fork 694 | if (pVmaList) 695 | { 696 | struct PageList *pPages; 697 | 698 | pVmaList->m_refCount--; 699 | 700 | if (pVmaList->m_refCount == 0) 701 | { 702 | PRINTK_VERBOSE(KERN_DEBUG "found vma, freeing pages (%s %d)\n", 703 | current->comm, current->pid); 704 | 705 | pPages = pVmaList->m_pPageHead; 706 | 707 | if (!pPages) 708 | { 709 | PRINTK(KERN_ERR "no page list (%s %d)!\n", 710 | current->comm, current->pid); 711 | return; 712 | } 713 | 714 | while (pPages) 715 | { 716 | struct PageList *next; 717 | int count; 718 | 719 | PRINTK_VERBOSE(KERN_DEBUG "page list (%s %d)\n", 720 | current->comm, current->pid); 721 | 722 | next = pPages->m_pNext; 723 | for (count = 0; count < pPages->m_used; count++) 724 | { 725 | PRINTK_VERBOSE(KERN_DEBUG "freeing page %p (%s %d)\n", 726 | pPages->m_pPages[count], 727 | current->comm, current->pid); 728 | __free_pages(pPages->m_pPages[count], 0); 729 | g_trackedPages--; 730 | freed++; 731 | } 732 | 733 | PRINTK_VERBOSE(KERN_DEBUG "freeing page list (%s %d)\n", 734 | current->comm, current->pid); 735 | kfree(pPages); 736 | pPages = next; 737 | } 738 | 739 | //remove our vma from the list 740 | kfree(pVmaList); 741 | pVma->vm_private_data = 0; 742 | } 743 | else 744 | { 745 | PRINTK_VERBOSE(KERN_DEBUG "ref count is %d, not closing\n", pVmaList->m_refCount); 746 | } 747 | } 748 | else 749 | { 750 | PRINTK_VERBOSE(KERN_ERR "uh-oh, vma %p not found (%s %d)!\n", pVma, current->comm, current->pid); 751 | PRINTK_VERBOSE(KERN_ERR "CLOSE ERR\n"); 752 | } 753 | 754 | PRINTK_VERBOSE(KERN_DEBUG "CLOSE %p %d %d pages (tracked pages %d)", 755 | pVma, current->pid, freed, g_trackedPages); 756 | 757 | PRINTK_VERBOSE(KERN_DEBUG "%d pages open\n", g_trackedPages); 758 | } 759 | 760 | static int VmaFault4k(struct vm_area_struct *pVma, struct vm_fault *pVmf) 761 | { 762 | PRINTK_VERBOSE(KERN_DEBUG "vma fault for vma %p private %p at offset %ld (%s %d)\n", pVma, pVma->vm_private_data, pVmf->pgoff, 763 | current->comm, current->pid); 764 | PRINTK_VERBOSE(KERN_DEBUG "FAULT\n"); 765 | pVmf->page = alloc_page(GFP_KERNEL); 766 | 767 | if (pVmf->page) 768 | { 769 | PRINTK_VERBOSE(KERN_DEBUG "alloc page virtual %p\n", page_address(pVmf->page)); 770 | } 771 | 772 | if (!pVmf->page) 773 | { 774 | PRINTK(KERN_ERR "vma fault oom (%s %d)\n", current->comm, current->pid); 775 | return VM_FAULT_OOM; 776 | } 777 | else 778 | { 779 | struct VmaPageList *pVmaList; 780 | 781 | get_page(pVmf->page); 782 | g_trackedPages++; 783 | 784 | //find our vma in the list 785 | pVmaList = (struct VmaPageList *)pVma->vm_private_data; 786 | 787 | if (pVmaList) 788 | { 789 | PRINTK_VERBOSE(KERN_DEBUG "vma found (%s %d)\n", current->comm, current->pid); 790 | 791 | if (pVmaList->m_pPageTail->m_used == PAGES_PER_LIST) 792 | { 793 | PRINTK_VERBOSE(KERN_DEBUG "making new page list (%s %d)\n", current->comm, current->pid); 794 | //making a new page list 795 | pVmaList->m_pPageTail->m_pNext = (struct PageList *)kmalloc(sizeof(struct PageList), GFP_KERNEL); 796 | if (!pVmaList->m_pPageTail->m_pNext) 797 | return -ENOMEM; 798 | 799 | //update the tail pointer 800 | pVmaList->m_pPageTail = pVmaList->m_pPageTail->m_pNext; 801 | pVmaList->m_pPageTail->m_used = 0; 802 | pVmaList->m_pPageTail->m_pNext = 0; 803 | } 804 | 805 | PRINTK_VERBOSE(KERN_DEBUG "adding page to list (%s %d)\n", current->comm, current->pid); 806 | 807 | pVmaList->m_pPageTail->m_pPages[pVmaList->m_pPageTail->m_used] = pVmf->page; 808 | pVmaList->m_pPageTail->m_used++; 809 | } 810 | else 811 | PRINTK(KERN_ERR "returned page for vma we don\'t know %p (%s %d)\n", pVma, current->comm, current->pid); 812 | 813 | return 0; 814 | } 815 | } 816 | 817 | /****** GENERIC FUNCTIONS ******/ 818 | static int __init dmaer_init(void) 819 | { 820 | int result = alloc_chrdev_region(&g_majorMinor, 0, 1, "dmaer"); 821 | if (result < 0) 822 | { 823 | PRINTK(KERN_ERR "unable to get major device number\n"); 824 | return result; 825 | } 826 | else 827 | PRINTK(KERN_DEBUG "major device number %d\n", MAJOR(g_majorMinor)); 828 | 829 | PRINTK(KERN_DEBUG "vma list size %d, page list size %d, page size %ld\n", 830 | sizeof(struct VmaPageList), sizeof(struct PageList), PAGE_SIZE); 831 | 832 | //get a dma channel to work with 833 | result = bcm_dma_chan_alloc(BCM_DMA_FEATURE_FAST, (void **)&g_pDmaChanBase, &g_dmaIrq); 834 | 835 | //uncomment to force to channel 0 836 | //result = 0; 837 | //g_pDmaChanBase = 0xce808000; 838 | 839 | if (result < 0) 840 | { 841 | PRINTK(KERN_ERR "failed to allocate dma channel\n"); 842 | cdev_del(&g_cDev); 843 | unregister_chrdev_region(g_majorMinor, 1); 844 | } 845 | 846 | //reset the channel 847 | PRINTK(KERN_DEBUG "allocated dma channel %d (%p), initial state %08x\n", result, g_pDmaChanBase, *g_pDmaChanBase); 848 | *g_pDmaChanBase = 1 << 31; 849 | PRINTK(KERN_DEBUG "post-reset %08x\n", *g_pDmaChanBase); 850 | 851 | g_dmaChan = result; 852 | 853 | //clear the cache stats 854 | g_cacheHit = 0; 855 | g_cacheMiss = 0; 856 | 857 | //register our device - after this we are go go go 858 | cdev_init(&g_cDev, &g_fOps); 859 | g_cDev.owner = THIS_MODULE; 860 | g_cDev.ops = &g_fOps; 861 | 862 | result = cdev_add(&g_cDev, g_majorMinor, 1); 863 | if (result < 0) 864 | { 865 | PRINTK(KERN_ERR "failed to add character device\n"); 866 | unregister_chrdev_region(g_majorMinor, 1); 867 | bcm_dma_chan_free(g_dmaChan); 868 | return result; 869 | } 870 | 871 | return 0; 872 | } 873 | 874 | static void __exit dmaer_exit(void) 875 | { 876 | PRINTK(KERN_INFO "closing dmaer device, cache stats: %d hits %d misses\n", g_cacheHit, g_cacheMiss); 877 | //unregister the device 878 | cdev_del(&g_cDev); 879 | unregister_chrdev_region(g_majorMinor, 1); 880 | //free the dma channel 881 | bcm_dma_chan_free(g_dmaChan); 882 | } 883 | 884 | MODULE_LICENSE("Dual BSD/GPL"); 885 | MODULE_AUTHOR("Simon Hall"); 886 | module_init(dmaer_init); 887 | module_exit(dmaer_exit); 888 | 889 | --------------------------------------------------------------------------------