├── .gitignore ├── Kbuild ├── LICENSE ├── Makefile ├── README.md └── rust_nvme.rs /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | # for Linux kernel 13 | .Module.symvers.cmd 14 | .modules.order.cmd 15 | .rust_nvme.ko.cmd 16 | .rust_nvme.mod.cmd 17 | .rust_nvme.mod.o.cmd 18 | .rust_nvme.o.cmd 19 | Module.symvers 20 | modules.order 21 | rust_nvme.ko 22 | rust_nvme.mod 23 | rust_nvme.mod.c 24 | rust_nvme.mod.o 25 | rust_nvme.o 26 | -------------------------------------------------------------------------------- /Kbuild: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0 2 | 3 | obj-m := rust_nvme.o 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0 2 | 3 | KDIR ?= /lib/modules/`uname -r`/build 4 | 5 | default: 6 | $(MAKE) -C $(KDIR) M=$$PWD 7 | 8 | clean: 9 | $(MAKE) -C $(KDIR) M=$$PWD clean 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rust simple NVMe device driver 2 | 3 | This is for figuring out proper Rust PCI, DMA, block layer abstraction APIs. 4 | 5 | It can handle some I/Os with QEMU. But no proper error handling or resource cleanup. 6 | 7 | [Rust-for-Linux tree](https://github.com/Rust-for-Linux/linux) doesn't support abstraction APIs (PCI, DMA, block layer, etc) yet. This driver is tested with [my fork](https://github.com/fujita/linux/tree/rust-nvme). I'll work for upstreaming. 8 | 9 | ```bash 10 | $ make KDIR=~/git/linux LLVM=1 11 | ``` 12 | -------------------------------------------------------------------------------- /rust_nvme.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | 3 | // ! Rust simple nvme driver 4 | 5 | use core::ffi::c_void; 6 | use core::{marker::PhantomPinned, mem::zeroed, pin::Pin, ptr}; 7 | use kernel::bindings; 8 | use kernel::driver; 9 | use kernel::irq; 10 | use kernel::pcidev; 11 | use kernel::prelude::*; 12 | use kernel::sync::{Ref, RefBorrow, SpinLock}; 13 | use kernel::Result; 14 | use kernel::{c_str, str::CStr}; 15 | 16 | struct BlkDevOps; 17 | 18 | impl BlkDevOps { 19 | const OPS: bindings::block_device_operations = bindings::block_device_operations { 20 | submit_bio: None, 21 | poll_bio: None, 22 | open: None, 23 | release: None, 24 | rw_page: None, 25 | ioctl: None, 26 | compat_ioctl: None, 27 | check_events: None, 28 | unlock_native_capacity: None, 29 | getgeo: None, 30 | set_read_only: None, 31 | free_disk: None, 32 | swap_slot_free_notify: None, 33 | report_zones: None, 34 | devnode: None, 35 | get_unique_id: None, 36 | owner: ptr::null_mut(), 37 | pr_ops: ptr::null_mut(), 38 | alternative_gpt_sector: None, 39 | }; 40 | } 41 | 42 | struct Adapter(T); 43 | 44 | impl Adapter { 45 | unsafe extern "C" fn init_request( 46 | set: *mut bindings::blk_mq_tag_set, 47 | rq: *mut bindings::request, 48 | arg2: core::ffi::c_uint, 49 | arg3: core::ffi::c_uint, 50 | ) -> core::ffi::c_int { 51 | T::init_request(set, rq, arg2, arg3) 52 | } 53 | unsafe extern "C" fn init_hctx( 54 | arg1: *mut bindings::blk_mq_hw_ctx, 55 | arg2: *mut core::ffi::c_void, 56 | arg3: core::ffi::c_uint, 57 | ) -> core::ffi::c_int { 58 | T::init_hctx(arg1, arg2, arg3) 59 | } 60 | 61 | unsafe extern "C" fn queue_rq( 62 | arg1: *mut bindings::blk_mq_hw_ctx, 63 | bd: *const bindings::blk_mq_queue_data, 64 | ) -> bindings::blk_status_t { 65 | T::queue_rq(arg1, bd) 66 | } 67 | 68 | unsafe extern "C" fn complete(req: *mut bindings::request) { 69 | T::complete(req) 70 | } 71 | 72 | const OPS: bindings::blk_mq_ops = bindings::blk_mq_ops { 73 | queue_rq: Some(Self::queue_rq), 74 | commit_rqs: None, 75 | queue_rqs: None, 76 | put_budget: None, 77 | get_budget: None, 78 | set_rq_budget_token: None, 79 | get_rq_budget_token: None, 80 | timeout: None, 81 | poll: None, 82 | complete: Some(Self::complete), 83 | init_hctx: Some(Self::init_hctx), 84 | exit_hctx: None, 85 | init_request: Some(Self::init_request), 86 | exit_request: None, 87 | cleanup_rq: None, 88 | busy: None, 89 | map_queues: None, 90 | show_rq: None, 91 | }; 92 | } 93 | 94 | trait BlkRqOps { 95 | fn init_request( 96 | _set: *mut bindings::blk_mq_tag_set, 97 | rq: *mut bindings::request, 98 | _arg2: core::ffi::c_uint, 99 | _arg3: core::ffi::c_uint, 100 | ) -> core::ffi::c_int; 101 | 102 | fn init_hctx( 103 | _arg1: *mut bindings::blk_mq_hw_ctx, 104 | _arg2: *mut core::ffi::c_void, 105 | _arg3: core::ffi::c_uint, 106 | ) -> core::ffi::c_int; 107 | 108 | fn queue_rq( 109 | arg1: *mut bindings::blk_mq_hw_ctx, 110 | bd: *const bindings::blk_mq_queue_data, 111 | ) -> bindings::blk_status_t; 112 | 113 | fn complete(req: *mut bindings::request); 114 | } 115 | 116 | struct BlkAdminOps; 117 | 118 | impl BlkRqOps for BlkAdminOps { 119 | fn init_request( 120 | _set: *mut bindings::blk_mq_tag_set, 121 | _rq: *mut bindings::request, 122 | _arg2: core::ffi::c_uint, 123 | _arg3: core::ffi::c_uint, 124 | ) -> core::ffi::c_int { 125 | 0 126 | } 127 | 128 | fn init_hctx( 129 | _arg1: *mut bindings::blk_mq_hw_ctx, 130 | _arg2: *mut core::ffi::c_void, 131 | _arg3: core::ffi::c_uint, 132 | ) -> core::ffi::c_int { 133 | 0 134 | } 135 | 136 | fn queue_rq( 137 | _arg1: *mut bindings::blk_mq_hw_ctx, 138 | bd: *const bindings::blk_mq_queue_data, 139 | ) -> bindings::blk_status_t { 140 | unsafe { 141 | let req = (*bd).rq; 142 | bindings::blk_mq_start_request(req); 143 | let pdu = rq_to_pdu(req); 144 | let mut a: Box = Box::from_raw((*(*req).q).queuedata as _); 145 | 146 | let mut tail = a.sq_tail as usize; 147 | 148 | core::ptr::copy( 149 | &((*pdu).cmnd), 150 | a.sq_cmds.add(tail * NvmeDriver::SQ_STRUCT_SIZE) as *mut NvmeCommonCommand, 151 | NvmeDriver::SQ_STRUCT_SIZE, 152 | ); 153 | 154 | tail += 1; 155 | if tail == a.depth { 156 | tail = 0; 157 | } 158 | bindings::writel(tail as u32, a.q_db); 159 | a.sq_tail = tail as u16; 160 | 161 | (*(*req).q).queuedata = Box::into_raw(a) as _; 162 | } 163 | 164 | bindings::BLK_STS_OK as u8 165 | } 166 | 167 | fn complete(req: *mut bindings::request) { 168 | // let pdu = rq_to_pdu(req); 169 | // pr_info!( 170 | // "admin complete {:?}, status: {}, result: {}", 171 | // req, 172 | // (*pdu).status, 173 | // (*pdu).result 174 | // ); 175 | unsafe { 176 | bindings::blk_mq_end_request(req, 0); 177 | } 178 | } 179 | } 180 | 181 | const CTRL_PAGE_SIZE: i32 = 4096; 182 | 183 | struct BlkIoOps; 184 | 185 | impl BlkRqOps for BlkIoOps { 186 | fn init_request( 187 | _set: *mut bindings::blk_mq_tag_set, 188 | _rq: *mut bindings::request, 189 | _arg2: core::ffi::c_uint, 190 | _arg3: core::ffi::c_uint, 191 | ) -> core::ffi::c_int { 192 | 0 193 | } 194 | 195 | fn init_hctx( 196 | _arg1: *mut bindings::blk_mq_hw_ctx, 197 | _arg2: *mut core::ffi::c_void, 198 | _arg3: core::ffi::c_uint, 199 | ) -> core::ffi::c_int { 200 | 0 201 | } 202 | 203 | fn queue_rq( 204 | _arg1: *mut bindings::blk_mq_hw_ctx, 205 | bd: *const bindings::blk_mq_queue_data, 206 | ) -> bindings::blk_status_t { 207 | unsafe { 208 | let mut req = (*bd).rq; 209 | let mut pdu = rq_to_pdu(req); 210 | (*pdu).nents = 0; 211 | let mut a: Box = Box::from_raw((*(*req).q).queuedata as _); 212 | let mut tail = a.sq_tail as usize; 213 | let mut sg_idx = 0; 214 | 215 | pr_info!("queue io request tail {}, {:?}", tail, req); 216 | bindings::blk_mq_start_request(req); 217 | let n = bindings::blk_rq_nr_phys_segments(req); 218 | let cmnd = a.sq_cmds.add(tail * NvmeDriver::SQ_STRUCT_SIZE) as *mut NvmeRwCommand; 219 | (*cmnd).command_id = (*req).tag as u16; 220 | (*cmnd).flags = 0; 221 | (*cmnd).nsid = 1; 222 | (*cmnd).rsvd2 = 0; 223 | (*cmnd).metadata = 0; 224 | (*cmnd).slba = bindings::blk_rq_pos(req); 225 | (*cmnd).length = ((bindings::blk_rq_bytes(req) >> 9) - 1) as u16; 226 | (*cmnd).control = 0; 227 | (*cmnd).dsmgmt = 0; 228 | (*cmnd).reftag = 0; 229 | (*cmnd).apptag = 0; 230 | (*cmnd).appmask = 0; 231 | (*cmnd).prp1 = 0; 232 | (*cmnd).prp2 = 0; 233 | 234 | if n > 0 { 235 | bindings::sg_init_table(&mut (*pdu).sg[sg_idx], n as u32); 236 | let nents = bindings::blk_rq_map_sg((*req).q, req, &mut (*pdu).sg[sg_idx]); 237 | let dir = if ((bindings::REQ_OP_MASK & (*req).cmd_flags) & 1) == 1 { 238 | bindings::dma_data_direction_DMA_TO_DEVICE 239 | } else { 240 | bindings::dma_data_direction_DMA_FROM_DEVICE 241 | }; 242 | 243 | if dir == bindings::dma_data_direction_DMA_TO_DEVICE { 244 | (*cmnd).opcode = NvmeDriver::CMD_WRITE; 245 | } else { 246 | (*cmnd).opcode = NvmeDriver::CMD_READ; 247 | } 248 | 249 | let mapped = bindings::dma_map_sg_attrs( 250 | a.dev, 251 | &mut (*pdu).sg[sg_idx], 252 | nents, 253 | dir, 254 | bindings::DMA_ATTR_NO_WARN as u64, 255 | ); 256 | 257 | let mut sg = (*pdu).sg[sg_idx]; 258 | let mut dma_len: i32 = sg.dma_length as i32; 259 | let mut dma_addr = sg.dma_address; 260 | let offset = (dma_addr % CTRL_PAGE_SIZE as u64) as i32; 261 | (*cmnd).prp1 = dma_addr; 262 | 263 | let mut length: i32 = bindings::blk_rq_payload_bytes(req) as i32; 264 | // pr_info!("len {}, dma mapped: {}, offset: {}", length, mapped, offset); 265 | (*pdu).nents = mapped as i32; 266 | length -= CTRL_PAGE_SIZE - offset; 267 | if length > 0 { 268 | dma_len -= CTRL_PAGE_SIZE - offset; 269 | if dma_len > 0 { 270 | dma_addr += (CTRL_PAGE_SIZE - offset) as u64; 271 | } else { 272 | sg_idx += 1; 273 | sg = (*pdu).sg[sg_idx]; 274 | dma_len = sg.dma_length as i32; 275 | dma_addr = sg.dma_address; 276 | } 277 | 278 | if length > CTRL_PAGE_SIZE { 279 | let mut prp_dma: u64 = 0; 280 | let l: *mut u64 = bindings::dma_alloc_attrs( 281 | a.dev, 282 | 8 * MAX_SG, 283 | &mut prp_dma, 284 | bindings::___GFP_ATOMIC, 285 | 0, 286 | ) as _; 287 | // pr_info!("alloc coherent {:?}", l); 288 | (*cmnd).prp2 = prp_dma; 289 | for i in 0..MAX_SG { 290 | *(l.add(i)) = dma_addr; 291 | dma_len -= CTRL_PAGE_SIZE; 292 | dma_addr += CTRL_PAGE_SIZE as u64; 293 | length -= CTRL_PAGE_SIZE; 294 | // pr_info!("alloc coherent {} {:?} {} {}", i, l.add(i), dma_len, length); 295 | 296 | if length <= 0 { 297 | break; 298 | } 299 | if dma_len > 0 { 300 | continue; 301 | } 302 | sg_idx += 1; 303 | sg = (*pdu).sg[sg_idx]; 304 | dma_len = sg.dma_length as i32; 305 | dma_addr = sg.dma_address; 306 | } 307 | } else { 308 | // pr_info!("no coherent {}", dma_addr); 309 | (*cmnd).prp2 = dma_addr; 310 | } 311 | } 312 | 313 | tail += 1; 314 | if tail == (*a).depth { 315 | tail = 0; 316 | } 317 | 318 | bindings::writel(tail as u32, a.q_db); 319 | a.sq_tail = tail as u16; 320 | } 321 | (*(*req).q).queuedata = Box::into_raw(a) as _; 322 | } 323 | bindings::BLK_STS_OK as u8 324 | } 325 | 326 | fn complete(req: *mut bindings::request) { 327 | let pdu = rq_to_pdu(req); 328 | pr_info!( 329 | "io complete {:?}, status: {}, result: {}", 330 | req, 331 | (*pdu).status, 332 | (*pdu).result 333 | ); 334 | unsafe { 335 | let a: Box = Box::from_raw((*(*req).q).queuedata as _); 336 | if (*pdu).nents > 0 { 337 | bindings::dma_unmap_sg_attrs(a.dev, &mut (*pdu).sg[0], (*pdu).nents, 2, 0); 338 | } 339 | bindings::blk_mq_end_request(req, 0); 340 | 341 | (*(*req).q).queuedata = Box::into_raw(a) as _; 342 | } 343 | } 344 | } 345 | 346 | struct Tag { 347 | set: bindings::blk_mq_tag_set, 348 | _pin: PhantomPinned, 349 | } 350 | 351 | impl Tag { 352 | fn new() -> Self { 353 | let mut set: bindings::blk_mq_tag_set = unsafe { zeroed() }; 354 | set.ops = &Adapter::::OPS; 355 | 356 | Tag { 357 | set, 358 | _pin: PhantomPinned, 359 | } 360 | } 361 | } 362 | 363 | struct RequestQueue { 364 | ptr: *mut bindings::request_queue, 365 | } 366 | 367 | impl RequestQueue { 368 | fn new(tag: &mut Tag) -> Self { 369 | unsafe { 370 | tag.set.nr_hw_queues = 1; 371 | tag.set.queue_depth = 30; 372 | tag.set.timeout = 30 * bindings::HZ; 373 | tag.set.numa_node = -1; 374 | tag.set.cmd_size = core::mem::size_of::() as u32; 375 | tag.set.flags = bindings::BLK_MQ_F_NO_SCHED; 376 | let _r = bindings::blk_mq_alloc_tag_set(&mut tag.set); 377 | } 378 | 379 | let ptr = unsafe { bindings::blk_mq_init_queue(&mut tag.set) }; 380 | if unsafe { bindings::IS_ERR(ptr as *const c_void) } { 381 | pr_info!("init queue error"); 382 | } 383 | RequestQueue { ptr } 384 | } 385 | } 386 | 387 | #[repr(C)] 388 | struct Pdu { 389 | cmnd: NvmeCommonCommand, 390 | status: u16, 391 | result: u32, 392 | // should be allocated dynamically 393 | nents: i32, 394 | sg: [bindings::scatterlist; MAX_SG], 395 | } 396 | 397 | const MAX_SG: usize = 16; 398 | 399 | fn rq_to_pdu(rq: *mut bindings::request) -> *mut Pdu { 400 | let ptr = unsafe { bindings::blk_mq_rq_to_pdu(rq) }; 401 | ptr as *mut Pdu 402 | } 403 | 404 | #[repr(C)] 405 | struct NvmeCompletion { 406 | result: u32, 407 | _rsvd: u32, 408 | sq_head: u16, 409 | sq_id: u16, 410 | command_id: u16, 411 | status: u16, 412 | } 413 | 414 | #[repr(C)] 415 | struct NvmeCommonCommand { 416 | opcode: u8, 417 | flags: u8, 418 | command_id: u16, 419 | nsid: u32, 420 | cdw2: [u32; 2], 421 | metadata: u64, 422 | prp1: u64, 423 | prp2: u64, 424 | cdw10: [u32; 6], 425 | } 426 | 427 | #[repr(C)] 428 | struct NvmeIdentify { 429 | opcode: u8, 430 | flags: u8, 431 | command_id: u16, 432 | nsid: u32, 433 | rsvd2: [u64; 2], 434 | prp1: u64, 435 | prp2: u64, 436 | cns: u32, 437 | rsvd11: [u32; 5], 438 | } 439 | 440 | #[repr(C)] 441 | struct NvmeFeatures { 442 | opcode: u8, 443 | flags: u8, 444 | command_id: u16, 445 | nsid: u32, 446 | rsvd2: [u64; 2], 447 | prp1: u64, 448 | prp2: u64, 449 | fib: u32, 450 | dword11: u32, 451 | rsvd11: [u32; 4], 452 | } 453 | 454 | #[repr(C)] 455 | struct NvmeCreateCq { 456 | opcode: u8, 457 | flags: u8, 458 | command_id: u16, 459 | rsvd1: [u32; 5], 460 | prp1: u64, 461 | rsvd8: u64, 462 | cqid: u16, 463 | qsize: u16, 464 | cq_flags: u16, 465 | irq_vector: u16, 466 | rsvd12: [u32; 4], 467 | } 468 | 469 | #[repr(C)] 470 | struct NvmeCreateSq { 471 | opcode: u8, 472 | flags: u8, 473 | command_id: u16, 474 | rsvd1: [u32; 5], 475 | prp1: u64, 476 | rsvd8: u64, 477 | sqid: u16, 478 | qsize: u16, 479 | sq_flags: u16, 480 | cqid: u16, 481 | rsvd12: [u32; 4], 482 | } 483 | 484 | #[repr(C)] 485 | struct NvmeIdPowerState { 486 | max_power: u16, 487 | rsvd2: u16, 488 | entry_lat: u32, 489 | exit_lat: u32, 490 | read_tput: u8, 491 | read_lat: u8, 492 | write_tput: u8, 493 | write_lat: u8, 494 | rsvd16: [u8; 16], 495 | } 496 | 497 | #[repr(C)] 498 | struct NvmeIdCtrl { 499 | vid: u16, 500 | ssvid: u16, 501 | sn: [u8; 20], 502 | mn: [u8; 40], 503 | fr: [u8; 8], 504 | rab: u8, 505 | ieee: [u8; 3], 506 | mic: u8, 507 | mdts: u8, 508 | rsvd78: [u8; 178], 509 | oacs: u16, 510 | acl: u8, 511 | aerl: u8, 512 | frmw: u8, 513 | lpa: u8, 514 | elpe: u8, 515 | npss: u8, 516 | rsvd264: [u8; 248], 517 | sqes: u8, 518 | cqes: u8, 519 | rsvd514: [u8; 2], 520 | nn: u32, 521 | oncs: u16, 522 | fuses: u16, 523 | fna: u8, 524 | vwc: u8, 525 | awun: u16, 526 | awupf: u16, 527 | rsvd530: [u8; 1518], 528 | psd: [NvmeIdPowerState; 32], 529 | vs: [u8; 1024], 530 | } 531 | 532 | #[repr(C)] 533 | struct NvmeRwCommand { 534 | opcode: u8, 535 | flags: u8, 536 | command_id: u16, 537 | nsid: u32, 538 | rsvd2: u64, 539 | metadata: u64, 540 | prp1: u64, 541 | prp2: u64, 542 | slba: u64, 543 | length: u16, 544 | control: u16, 545 | dsmgmt: u32, 546 | reftag: u32, 547 | apptag: u16, 548 | appmask: u16, 549 | } 550 | 551 | #[repr(C)] 552 | struct NvmeLbaf { 553 | ms: u16, 554 | ds: u8, 555 | rp: u8, 556 | } 557 | 558 | #[repr(C)] 559 | struct NvmeIdNs { 560 | nsze: u64, 561 | ncap: u64, 562 | nuse: u64, 563 | nsfeat: u8, 564 | nlbaf: u8, 565 | flbas: u8, 566 | mc: u8, 567 | dpc: u8, 568 | dps: u8, 569 | rsvd30: [u8; 98], 570 | lbaf: [NvmeLbaf; 16], 571 | rsvd192: [u8; 192], 572 | vs: [u8; 3712], 573 | } 574 | 575 | #[repr(C)] 576 | struct NvmeLbaRangeType { 577 | rt_type: u8, 578 | attributes: u8, 579 | rsvd2: [u8; 14], 580 | slba: u64, 581 | nlb: u64, 582 | guid: [u8; 16], 583 | rsvd48: [u8; 16], 584 | } 585 | 586 | struct DevInfo { 587 | bar: *mut core::ffi::c_void, 588 | 589 | admin_rq: RequestQueue, 590 | admin_irq: irq::Registration, 591 | 592 | io_irq: irq::Registration, 593 | } 594 | 595 | unsafe impl Send for DevInfo {} 596 | 597 | unsafe impl Sync for DevInfo {} 598 | 599 | impl driver::DeviceRemoval for DevInfo { 600 | fn device_remove(&self) {} 601 | } 602 | 603 | struct NvmeQueue { 604 | cq_dma_addr: u64, 605 | cqes: *mut c_void, 606 | 607 | sq_dma_addr: u64, 608 | sq_cmds: *mut c_void, 609 | sq_tail: u16, 610 | depth: usize, 611 | cq_head: u32, 612 | cq_phase: u16, 613 | 614 | q_db: *mut c_void, 615 | 616 | dev: *mut bindings::device, 617 | } 618 | 619 | impl NvmeQueue { 620 | fn new(pdev: &mut pcidev::Device, depth: usize, q_db: *mut c_void) -> Self { 621 | let mut cq_dma_addr: u64 = 0; 622 | let cqes = pdev 623 | .dma_alloc_coherent( 624 | depth * NvmeDriver::CQ_STRUCT_SIZE, 625 | &mut cq_dma_addr, 626 | bindings::GFP_KERNEL, 627 | ) 628 | .unwrap(); 629 | let mut sq_dma_addr: u64 = 0; 630 | let sq_cmds = pdev 631 | .dma_alloc_coherent( 632 | depth * NvmeDriver::SQ_STRUCT_SIZE, 633 | &mut sq_dma_addr, 634 | bindings::GFP_KERNEL, 635 | ) 636 | .unwrap(); 637 | 638 | NvmeQueue { 639 | dev: unsafe { &mut (*pdev.ptr).dev }, 640 | cq_dma_addr, 641 | cqes, 642 | sq_dma_addr, 643 | sq_cmds, 644 | sq_tail: 0, 645 | depth, 646 | q_db, 647 | cq_phase: 1, 648 | cq_head: 0, 649 | } 650 | } 651 | } 652 | 653 | struct NvmeIrq { 654 | tag: Tag, 655 | cq_head: u32, 656 | cq_phase: u16, 657 | depth: usize, 658 | 659 | cqes: *mut c_void, 660 | q_db: *mut c_void, 661 | } 662 | 663 | impl irq::Handler for NvmeDriver { 664 | type Data = Ref>; 665 | 666 | fn handle_irq(data: RefBorrow<'_, SpinLock>) -> irq::Return { 667 | unsafe { 668 | let mut q = data.lock(); 669 | let mut head = q.cq_head; 670 | let mut phase = q.cq_phase; 671 | let maps = core::slice::from_raw_parts_mut(q.tag.set.tags, 1); 672 | 673 | loop { 674 | let cqe = 675 | q.cqes.add(head as usize * NvmeDriver::CQ_STRUCT_SIZE) as *mut NvmeCompletion; 676 | 677 | let status = (*cqe).status; 678 | pr_info!( 679 | "irq: head {}, phase {}, status {}, id {}", 680 | head, 681 | phase, 682 | status, 683 | (*cqe).command_id 684 | ); 685 | if (status & 1) != phase { 686 | break; 687 | } 688 | 689 | let rq = bindings::blk_mq_tag_to_rq(maps[0], (*cqe).command_id as u32); 690 | 691 | if !rq.is_null() { 692 | let pdu = rq_to_pdu(rq); 693 | (*pdu).result = (*cqe).result; 694 | (*pdu).status = (*cqe).status >> 1; 695 | 696 | bindings::blk_mq_complete_request(rq); 697 | } 698 | 699 | head += 1; 700 | if head == q.depth as u32 { 701 | head = 0; 702 | phase = !phase; 703 | } 704 | } 705 | 706 | if q.cq_head == head && phase == q.cq_phase { 707 | return irq::Return::None; 708 | } 709 | 710 | bindings::writel(head, q.q_db); 711 | q.cq_head = head; 712 | q.cq_phase = phase; 713 | } 714 | irq::Return::Handled 715 | } 716 | } 717 | 718 | struct NvmeDriver; 719 | 720 | impl NvmeDriver { 721 | const NVME_CC_ENABLE: u32 = 1 << 0; 722 | const NVME_CC_CSS_NVM: u32 = 0 << 4; 723 | const NVME_CC_MPS_SHIFT: u32 = 7; 724 | const NVME_CC_ARB_RR: u32 = 0 << 11; 725 | // const NVME_CC_ARB_WRRU: u32 = 1 << 11; 726 | // const NVME_CC_ARB_VS: u32 = 7 << 11; 727 | const NVME_CC_SHN_NONE: u32 = 0 << 14; 728 | // const NVME_CC_SHN_NORMAL: u32 = 1 << 14; 729 | // const NVME_CC_SHN_ABRUPT: u32 = 2 << 14; 730 | const NVME_CC_IOSQES: u32 = 6 << 16; 731 | const NVME_CC_IOCQES: u32 = 4 << 20; 732 | // const NVME_CSTS_RDY: u32 = 1 << 0; 733 | // const NVME_CSTS_CFS: u32 = 1 << 1; 734 | // const NVME_CSTS_SHST_NORMAL: u32 = 0 << 2; 735 | // const NVME_CSTS_SHST_OCCUR: u32 = 1 << 2; 736 | // const NVME_CSTS_SHST_CMPLT: u32 = 2 << 2; 737 | 738 | const IOMEM_SIZE: u64 = 8192; 739 | const QUEUE_DEPTH: usize = 64; 740 | const SQ_STRUCT_SIZE: usize = 64; 741 | const CQ_STRUCT_SIZE: usize = 16; 742 | 743 | const OFFSET_CAP: usize = 0; 744 | // const OFFSET_VS: usize = 8; 745 | // const OFFSET_INTMS: usize = 12; 746 | // const OFFSET_INTMC: usize = 16; 747 | const OFFSET_CC: usize = 20; 748 | const OFFSET_CSTS: usize = 28; 749 | const OFFSET_AQA: usize = 36; 750 | const OFFSET_ASQ: usize = 40; 751 | const OFFSET_ACQ: usize = 48; 752 | 753 | const CMD_WRITE: u8 = 0x01; 754 | const CMD_READ: u8 = 0x02; 755 | 756 | // admin opcode 757 | const ADMIN_CREATE_SQ: u8 = 0x01; 758 | const ADMIN_CREATE_CQ: u8 = 0x05; 759 | const ADMIN_IDENTIFY: u8 = 0x06; 760 | const ADMIN_SET_FEATURES: u8 = 0x09; 761 | const ADMIN_GET_FEATURES: u8 = 0x0a; 762 | 763 | // misc 764 | const QUEUE_PHYS_CONTIG: u16 = (1 << 0); 765 | const CQ_IRQ_ENABLED: u16 = (1 << 1); 766 | const SQ_PRIO_MEDIUM: u16 = (2 << 1); 767 | 768 | const FEAT_LBA_RANGE: u32 = 0x03; 769 | const FEAT_NUM_QUEUES: u32 = 0x07; 770 | 771 | fn execute_cmnd(rq: &mut RequestQueue, cmnd: &NvmeCommonCommand) -> u32 { 772 | unsafe { 773 | let mut rq = { 774 | let rw = bindings::req_opf_REQ_OP_DRV_IN; 775 | let rq = bindings::blk_mq_alloc_request(rq.ptr, rw, 0); 776 | let err = bindings::IS_ERR(rq as *const c_void); 777 | let mut pdu = rq_to_pdu(rq); 778 | if err { 779 | pr_info!("alloc error {}", bindings::PTR_ERR(rq as *const c_void)); 780 | } 781 | 782 | if pdu.is_null() { 783 | pr_info!("pdu is null!"); 784 | } else { 785 | //core::ptr::copy(cmnd, &mut ((*pdu).cmnd), NvmeDriver::SQ_STRUCT_SIZE); 786 | (*pdu).cmnd.opcode = cmnd.opcode; 787 | (*pdu).cmnd.flags = cmnd.flags; 788 | (*pdu).cmnd.command_id = (*rq).tag as u16; 789 | (*pdu).cmnd.nsid = cmnd.nsid; 790 | (*pdu).cmnd.cdw2 = cmnd.cdw2; 791 | (*pdu).cmnd.metadata = cmnd.metadata; 792 | (*pdu).cmnd.prp1 = cmnd.prp1; 793 | (*pdu).cmnd.prp2 = cmnd.prp2; 794 | (*pdu).cmnd.cdw10 = cmnd.cdw10; 795 | } 796 | rq 797 | }; 798 | 799 | (*rq).rq_flags |= bindings::req_flag_bits___REQ_FAILFAST_DRIVER; 800 | (*rq).rq_flags |= 1 << 7; 801 | 802 | let _status = bindings::blk_execute_rq(rq, false); 803 | let pdu = rq_to_pdu(rq); 804 | let result = (*pdu).result; 805 | bindings::blk_mq_free_request(rq); 806 | result 807 | } 808 | } 809 | } 810 | 811 | impl pcidev::Driver for NvmeDriver { 812 | type Data = Box; 813 | 814 | // for simplicity, probe() does everything synchronously 815 | fn probe(pdev: &mut pcidev::Device) -> Result { 816 | pr_info!("rustnvme probe"); 817 | 818 | const NR_IRQS: u32 = 2; 819 | 820 | let bars = pdev.select_bars(bindings::IORESOURCE_MEM.into()); 821 | pdev.request_selected_regions(bars, c_str!("nvme"))?; 822 | 823 | let bar = unsafe { bindings::ioremap(pdev.resource_start(0), NvmeDriver::IOMEM_SIZE) }; 824 | let dbs = unsafe { bar.add(4096) }; 825 | 826 | pdev.enable_device_mem()?; 827 | pdev.set_master(); 828 | 829 | unsafe { 830 | let n = bindings::pci_alloc_irq_vectors_affinity( 831 | pdev.ptr, 832 | NR_IRQS, 833 | NR_IRQS, 834 | bindings::PCI_IRQ_ALL_TYPES, 835 | ptr::null_mut(), 836 | ); 837 | assert_eq!(n, NR_IRQS as i32); 838 | } 839 | 840 | pdev.dma_set_mask(!0)?; 841 | pdev.dma_set_coherent_mask(!0)?; 842 | 843 | let admin_queue = Box::try_new(NvmeQueue::new(pdev, NvmeDriver::QUEUE_DEPTH, unsafe { 844 | bar.add(4096) 845 | })) 846 | .unwrap(); 847 | 848 | let mut admin_tag = Tag::new::(); 849 | let mut admin_rq = RequestQueue::new(&mut admin_tag); 850 | 851 | let mut aqa: u32 = (NvmeDriver::QUEUE_DEPTH - 1).try_into().unwrap(); 852 | aqa |= aqa << 16; 853 | 854 | let mut ctrl_config: u32 = NvmeDriver::NVME_CC_ENABLE | NvmeDriver::NVME_CC_CSS_NVM; 855 | ctrl_config |= (bindings::PAGE_SHIFT - 12) << NvmeDriver::NVME_CC_MPS_SHIFT; 856 | ctrl_config |= NvmeDriver::NVME_CC_ARB_RR | NvmeDriver::NVME_CC_SHN_NONE; 857 | ctrl_config |= NvmeDriver::NVME_CC_IOSQES | NvmeDriver::NVME_CC_IOCQES; 858 | 859 | unsafe { 860 | bindings::writel(0, bar.add(NvmeDriver::OFFSET_CC)); 861 | bindings::writel(aqa, bar.add(NvmeDriver::OFFSET_AQA)); 862 | bindings::writeq(admin_queue.sq_dma_addr, bar.add(NvmeDriver::OFFSET_ASQ)); 863 | bindings::writeq(admin_queue.cq_dma_addr, bar.add(NvmeDriver::OFFSET_ACQ)); 864 | bindings::writel(ctrl_config, bar.add(NvmeDriver::OFFSET_CC)); 865 | } 866 | let cap = unsafe { bindings::readq(bar.add(NvmeDriver::OFFSET_CAP)) }; 867 | let _db_stride = (cap >> 32) & 0xf; 868 | 869 | let mut ready = 0; 870 | for _ in 0..10 { 871 | ready = unsafe { bindings::readl(bar.add(NvmeDriver::OFFSET_CSTS)) }; 872 | if ready > 0 { 873 | break; 874 | } 875 | unsafe { 876 | bindings::msleep(100); 877 | } 878 | } 879 | assert!(ready > 0); 880 | 881 | let mut admin_irq_data = unsafe { 882 | SpinLock::new(NvmeIrq { 883 | tag: admin_tag, 884 | cq_head: 0, 885 | cq_phase: 1, 886 | cqes: admin_queue.cqes, 887 | depth: admin_queue.depth, 888 | q_db: admin_queue.q_db.add(4), 889 | }) 890 | }; 891 | kernel::spinlock_init!( 892 | unsafe { Pin::new_unchecked(&mut admin_irq_data) }, 893 | "nvme_admin_irq" 894 | ); 895 | unsafe { 896 | (*admin_rq.ptr).queuedata = Box::into_raw(admin_queue) as _; 897 | } 898 | 899 | let admin_irq = irq::Registration::::try_new( 900 | unsafe { bindings::pci_irq_vector(pdev.ptr, 0) } as u32, 901 | Ref::try_new(admin_irq_data).unwrap(), 902 | irq::flags::SHARED, 903 | fmt!("nvme admin irq"), 904 | ) 905 | .unwrap(); 906 | 907 | { 908 | let mut cmnd: NvmeCommonCommand = unsafe { zeroed() }; 909 | let mut c: NvmeFeatures = unsafe { zeroed() }; 910 | c.opcode = NvmeDriver::ADMIN_SET_FEATURES; 911 | c.fib = NvmeDriver::FEAT_NUM_QUEUES; 912 | c.dword11 = 0; 913 | 914 | unsafe { 915 | core::ptr::copy( 916 | &c, 917 | &mut cmnd as *mut _ as *mut NvmeFeatures, 918 | NvmeDriver::SQ_STRUCT_SIZE, 919 | ); 920 | } 921 | 922 | let r = NvmeDriver::execute_cmnd(&mut admin_rq, &cmnd); 923 | pr_info!("result {}", r); 924 | } 925 | let io_queue_depth = unsafe { 926 | let a = (bindings::readq(bar.add(NvmeDriver::OFFSET_CAP)) & 0xffff) + 1; 927 | pr_info!("queue depth {}", core::cmp::min(a, 1024)); 928 | core::cmp::min(a, 1024) 929 | }; 930 | 931 | let io_queue = Box::try_new(NvmeQueue::new( 932 | pdev, 933 | io_queue_depth.try_into().unwrap(), 934 | unsafe { dbs.add(8) }, 935 | )) 936 | .unwrap(); 937 | 938 | let mut io_tag = Tag::new::(); 939 | unsafe { 940 | io_tag.set.nr_hw_queues = 1; 941 | 942 | io_tag.set.queue_depth = io_queue_depth as u32 - 1; 943 | io_tag.set.timeout = 30 * bindings::HZ; 944 | io_tag.set.numa_node = -1; 945 | io_tag.set.cmd_size = core::mem::size_of::() as u32; 946 | let _r = bindings::blk_mq_alloc_tag_set(&mut io_tag.set); 947 | } 948 | 949 | { 950 | let mut cmnd: NvmeCommonCommand = unsafe { zeroed() }; 951 | let mut c: NvmeCreateCq = unsafe { zeroed() }; 952 | c.opcode = NvmeDriver::ADMIN_CREATE_CQ; 953 | c.prp1 = io_queue.cq_dma_addr; 954 | c.cqid = 1; 955 | c.qsize = io_queue.depth as u16 - 1; 956 | c.cq_flags = NvmeDriver::QUEUE_PHYS_CONTIG | NvmeDriver::CQ_IRQ_ENABLED; 957 | c.irq_vector = 1; 958 | 959 | unsafe { 960 | core::ptr::copy( 961 | &c, 962 | &mut cmnd as *mut _ as *mut NvmeCreateCq, 963 | NvmeDriver::SQ_STRUCT_SIZE, 964 | ); 965 | } 966 | 967 | let r = NvmeDriver::execute_cmnd(&mut admin_rq, &cmnd); 968 | pr_info!("ioirq result2 {}", r); 969 | } 970 | { 971 | let mut cmnd: NvmeCommonCommand = unsafe { zeroed() }; 972 | let mut c: NvmeCreateSq = unsafe { zeroed() }; 973 | c.opcode = NvmeDriver::ADMIN_CREATE_SQ; 974 | c.prp1 = io_queue.sq_dma_addr; 975 | c.sqid = 1; 976 | c.qsize = io_queue.depth as u16 - 1; 977 | c.sq_flags = NvmeDriver::QUEUE_PHYS_CONTIG | NvmeDriver::SQ_PRIO_MEDIUM; 978 | c.cqid = 1; 979 | 980 | unsafe { 981 | core::ptr::copy( 982 | &c, 983 | &mut cmnd as *mut _ as *mut NvmeCreateSq, 984 | NvmeDriver::SQ_STRUCT_SIZE, 985 | ); 986 | } 987 | 988 | let r = NvmeDriver::execute_cmnd(&mut admin_rq, &cmnd); 989 | pr_info!("ioirq result3 {}", r); 990 | } 991 | 992 | let mut disks = Vec::try_with_capacity(1).unwrap(); 993 | let io_queue_db = unsafe { io_queue.q_db.add(4) }; 994 | let io_queue_depth = io_queue.depth; 995 | let io_queue_cqes = io_queue.cqes; 996 | { 997 | let mut dma_addr: u64 = 0; 998 | let mem = pdev 999 | .dma_alloc_coherent(8192, &mut dma_addr, bindings::GFP_KERNEL) 1000 | .unwrap(); 1001 | 1002 | let mut cmnd: NvmeCommonCommand = unsafe { zeroed() }; 1003 | let mut c: NvmeIdentify = unsafe { zeroed() }; 1004 | c.opcode = NvmeDriver::ADMIN_IDENTIFY; 1005 | c.nsid = 0; 1006 | c.prp1 = dma_addr; 1007 | c.cns = 1; 1008 | unsafe { 1009 | core::ptr::copy( 1010 | &c, 1011 | &mut cmnd as *mut _ as *mut NvmeIdentify, 1012 | NvmeDriver::SQ_STRUCT_SIZE, 1013 | ); 1014 | } 1015 | 1016 | let r = NvmeDriver::execute_cmnd(&mut admin_rq, &cmnd); 1017 | pr_info!("result3 {}", r); 1018 | 1019 | let ctrl: *mut NvmeIdCtrl = mem as *mut NvmeIdCtrl; 1020 | let mut model: [u8; 41] = [0; 41]; 1021 | for i in 0..40 { 1022 | model[i] = unsafe { (*ctrl).mn[i] }; 1023 | } 1024 | 1025 | pr_info!("model: {}", CStr::from_bytes_with_nul(&model).unwrap()); 1026 | pr_info!("nn {}, mdts {}", (*ctrl).nn, (*ctrl).mdts); 1027 | 1028 | // handle only one ns 1029 | 1030 | let io_queue_ptr = Box::into_raw(io_queue) as _; 1031 | let nn = 1; 1032 | for i in 1..=nn { 1033 | let mut cmnd: NvmeCommonCommand = unsafe { zeroed() }; 1034 | let mut c: NvmeIdentify = unsafe { zeroed() }; 1035 | c.opcode = NvmeDriver::ADMIN_IDENTIFY; 1036 | c.nsid = i; 1037 | c.prp1 = dma_addr; 1038 | c.cns = 0; 1039 | unsafe { 1040 | core::ptr::copy( 1041 | &c, 1042 | &mut cmnd as *mut _ as *mut NvmeIdentify, 1043 | NvmeDriver::SQ_STRUCT_SIZE, 1044 | ); 1045 | } 1046 | 1047 | let r = NvmeDriver::execute_cmnd(&mut admin_rq, &cmnd); 1048 | pr_info!("nn result {} {}", i, r); 1049 | 1050 | let id_ns: *mut NvmeIdNs = mem as *mut NvmeIdNs; 1051 | pr_info!("ns {} {}", i, (*id_ns).ncap); 1052 | 1053 | { 1054 | let mut cmnd: NvmeCommonCommand = unsafe { zeroed() }; 1055 | let mut c: NvmeFeatures = unsafe { zeroed() }; 1056 | c.opcode = NvmeDriver::ADMIN_GET_FEATURES; 1057 | c.nsid = i; 1058 | c.prp1 = dma_addr + 4096; 1059 | c.fib = NvmeDriver::FEAT_LBA_RANGE; 1060 | unsafe { 1061 | core::ptr::copy( 1062 | &c, 1063 | &mut cmnd as *mut _ as *mut NvmeFeatures, 1064 | NvmeDriver::SQ_STRUCT_SIZE, 1065 | ); 1066 | } 1067 | 1068 | let r = NvmeDriver::execute_cmnd(&mut admin_rq, &cmnd); 1069 | pr_info!("nn result {} {}", i, r); 1070 | 1071 | let _rt: *mut NvmeLbaRangeType = 1072 | unsafe { mem.add(4096) as *mut NvmeLbaRangeType }; 1073 | 1074 | unsafe { 1075 | let lbaf = (*id_ns).flbas & 0xf; 1076 | let lba_shift = (*id_ns).lbaf[lbaf as usize].ds; 1077 | let capacity = (*id_ns).nsze << (lba_shift - 9); 1078 | let mut disk = bindings::blk_mq_alloc_disk(&mut io_tag.set, io_queue_ptr); 1079 | pr_info!( 1080 | "queue: {:?}, depth: {}", 1081 | (*disk).queue, 1082 | (*(*disk).queue).queue_depth 1083 | ); 1084 | bindings::blk_queue_max_segments((*disk).queue, MAX_SG as u16); 1085 | (*disk).major = MAJOR; 1086 | (*disk).minors = 64; 1087 | let name = c_str!("nvme0n1"); 1088 | for i in 0..name.len() { 1089 | (*disk).disk_name[i] = name[i] as i8; 1090 | } 1091 | (*disk).fops = &BlkDevOps::OPS; 1092 | bindings::set_capacity(disk, capacity); 1093 | disks.try_push(disk).unwrap(); 1094 | } 1095 | } 1096 | } 1097 | } 1098 | 1099 | let mut io_irq_data = unsafe { 1100 | SpinLock::new(NvmeIrq { 1101 | tag: io_tag, 1102 | cq_head: 0, 1103 | cq_phase: 1, 1104 | cqes: io_queue_cqes, 1105 | depth: io_queue_depth, 1106 | q_db: io_queue_db, 1107 | }) 1108 | }; 1109 | kernel::spinlock_init!( 1110 | unsafe { Pin::new_unchecked(&mut io_irq_data) }, 1111 | "nvme_io_irq" 1112 | ); 1113 | 1114 | let io_irq = irq::Registration::::try_new( 1115 | unsafe { bindings::pci_irq_vector(pdev.ptr, 1) } as u32, 1116 | Ref::try_new(io_irq_data).unwrap(), 1117 | irq::flags::SHARED, 1118 | fmt!("nvme io irq"), 1119 | ) 1120 | .unwrap(); 1121 | for d in disks { 1122 | unsafe { bindings::device_add_disk(&mut (*pdev.ptr).dev, d, ptr::null_mut()) }; 1123 | } 1124 | 1125 | pr_info!("end of probe"); 1126 | let info = Box::try_new(DevInfo { 1127 | bar, 1128 | admin_rq, 1129 | admin_irq, 1130 | io_irq, 1131 | })?; 1132 | Ok(info) 1133 | } 1134 | 1135 | const PCI_ID_TABLE: &'static [bindings::pci_device_id] = &[ 1136 | bindings::pci_device_id { 1137 | class: 0x010802, 1138 | class_mask: 0xffffff, 1139 | vendor: !0, 1140 | device: !0, 1141 | subvendor: !0, 1142 | subdevice: !0, 1143 | driver_data: 0, 1144 | override_only: 0, 1145 | }, 1146 | bindings::pci_device_id { 1147 | class: 0, 1148 | class_mask: 0, 1149 | vendor: 0, 1150 | device: 0, 1151 | subvendor: 0, 1152 | subdevice: 0, 1153 | driver_data: 0, 1154 | override_only: 0, 1155 | }, 1156 | ]; 1157 | } 1158 | 1159 | static mut MAJOR: i32 = 0; 1160 | 1161 | struct RustNvme { 1162 | _driver: Pin>>>, 1163 | } 1164 | 1165 | impl kernel::Module for RustNvme { 1166 | fn init(name: &'static CStr, module: &'static ThisModule) -> Result { 1167 | unsafe { 1168 | MAJOR = bindings::__register_blkdev(0, c_str!("nvme").as_char_ptr(), None); 1169 | } 1170 | let _driver = 1171 | driver::Registration::>::new_pinned(name, module)?; 1172 | Ok(RustNvme { _driver }) 1173 | } 1174 | } 1175 | 1176 | module! { 1177 | type: RustNvme, 1178 | name: b"rust_nvme", 1179 | author: b"FUJITA Tomonori ", 1180 | description: b"Rust simple NVMe driver", 1181 | license: b"GPL v2", 1182 | } 1183 | --------------------------------------------------------------------------------