├── .gitignore ├── .headroom.yaml ├── ChangeLog.md ├── LICENSE ├── README.md ├── Setup.hs ├── app └── Cli │ └── Main.hs ├── etc ├── gvm │ └── mdev-gpu │ │ ├── Intel │ │ └── .gitkeep │ │ ├── Nvidia │ │ ├── Open_vAmpere.yaml │ │ ├── Open_vGeneric.yaml │ │ ├── Open_vPascal.yaml │ │ └── Open_vTuring.yaml │ │ └── Tenstorrent │ │ └── .gitkeep └── systemd │ └── system │ └── mdev-post.service ├── headroom-templates └── haskell.mustache ├── mdev-gpu.cabal ├── package.yaml ├── src ├── Common │ ├── Config.hs │ ├── Device.hs │ └── Types.hs └── Nvidia │ ├── Device.hs │ ├── Errors.hs │ ├── MDev.hs │ ├── RMApi.hs │ └── RMApi │ ├── IoctlCodes.hs │ └── Types.hs ├── stack.yaml └── stack.yaml.lock /.gitignore: -------------------------------------------------------------------------------- 1 | .stack-work/ 2 | *~ -------------------------------------------------------------------------------- /.headroom.yaml: -------------------------------------------------------------------------------- 1 | ## This is the configuration file for Headroom. 2 | ## See https://github.com/vaclavsvejcar/headroom for more details. 3 | 4 | ## Defines with which version of Headroom this configuration is compatible. 5 | ## Headroom uses this field to detect whether your configuration doesn't need 6 | ## any manual migration steps in case that it's older than your current Headroom 7 | ## version. You don't need to touch this field unless explicitly stated in 8 | ## migration guide during upgrading Headroom to new version. 9 | version: 0.4.4.0 10 | 11 | ## Defines the behaviour how to handle license headers, possible options are: 12 | ## 13 | ## - add = (default) adds license header to files with no existing header 14 | ## (same as '-a|--add-headers' command line argument) 15 | ## - drop = drops existing license header from without replacement 16 | ## (same as '-d|--drop-headers' command line argument) 17 | ## - replace = adds or replaces existing license header 18 | ## (same as '-r|--replace-headers' command line argument) 19 | run-mode: add 20 | 21 | ## Paths to source code files (either files or directories), 22 | ## same as '-s|--source-path=PATH' command line argument (can be used multiple 23 | ## times for more than one path). 24 | source-paths: 25 | - src 26 | - app 27 | 28 | ## Allows to define list of regular expressions that will be matched against 29 | ## 'source-paths' and such paths will be excluded from processing. Same as 30 | ## '-e|--excluded-path=REGEX' command line argument (can be used multiple times 31 | ## for more than one path). 32 | excluded-paths: [] 33 | 34 | ## If set to 'true', Headroom tries to detect whether any VCS (like GIT) is used 35 | ## for current project and if yes, it loads rules for ignored files and excludes 36 | ## all source paths that matches these rules. 37 | exclude-ignored-paths: false 38 | 39 | ## Paths to template files (either files or directories), 40 | ## same as '-t|--template-path=PATH' command line argument (can be used multiple 41 | ## times for more than one path). 42 | template-paths: 43 | - headroom-templates 44 | 45 | ## Variables (key-value) to replace in templates, 46 | ## same as '-v|--variable="KEY=VALUE"' command line argument (can be used 47 | ## multiple times for more than one path). 48 | variables: {} 49 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | # Changelog for mdev-gpu 2 | 3 | ## [1.0.0] - 2022-07-04 4 | - FIXED: Limitation of 4 GBs mediated GPUs in NVIDIA code. 5 | - ADDED: Configuration parsing in JSON + YAML. 6 | - ADDED: Help dialogue for mdev-cli. 7 | - ADDED: mdev-cli to create arbitrary NVIDIA Mediated GPUs. 8 | - ADDED: Base library to expand the NVIDIA userland programs. 9 | - ADDED: Basic documentation on all components in the mdev-gpu system. 10 | - ADDED: Error reporting and stronger messages for what errors we encounter. 11 | - ADDED: License documentation, and released under GPL v2. 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mdev GPU 2 | 3 | Mdev-GPU is a user-configurable utility for GPU vendor drivers enabling the registration of arbitrary mdev types with the VFIO-Mediated Device framework. 4 | 5 | Mdev-GPU is released under GPLv2 as one component of the [GPU Virtual Machine (GVM) Project](https://gvm-docs.openmdev.io/). 6 | 7 | # How do I use Mdev-GPU? 8 | 9 | - [Documentation for Mdev-GPU](https://openmdev.io/index.php/Mdev-GPU) 10 | 11 | # What does Mdev-GPU do? 12 | 13 | GPU virtualization is not widely supported using free(libre) driver programs on commodity GPUs. 14 | 15 | Mdev-GPU enables the creation of arbitrary user-configurable Mediated Device types on existing drivers which either don't support [VFIO-Mdev functionality](https://openmdev.io/index.php/Virtual_I/O_Internals#Mdev_Mode) or only support pre-defined Mdev types. 16 | 17 | ![](https://openmdev.io/images/c/cb/Mdev-GPU-Example_Use.gif) 18 | 19 | # How does this work? 20 | 21 | In order to virtualize the GPU via the VFIO-Mdev API a driver must register devices and mdev callbacks with the Mediated Core. 22 | This may be accomplished via the vendor driver in the case that it supports pre-defined mdev types or via GVM/Mdev-GPU for drivers which do not include support for Mdev functionality and/or do not support arbitrary Mdev types. 23 | 24 | A diagram below depicts the components Mdev-GPU interacts with. 25 | 26 | ![](https://openmdev.io/images/7/75/Mdev-gpu.png) 27 | 28 | For further information see the Virtual I/O Internals article on OpenMdev.io under the section titled [Registering PCIE Device with the Mediated Core](https://openmdev.io/index.php/Virtual_I/O_Internals#Registering_PCIE_Device_with_the_Mediated_Core). 29 | 30 | # What programs work with Mdev-GPU? 31 | 32 | If you are running a program which interacts with the [VFIO-Mdev API](https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst) then you can install Mdev-GPU underneath it without modifying your programs or changing your workflow. 33 | 34 | Here is a list of some existing programs that can interact with functionality enabled by Mdev-GPU without modification: 35 | 36 | - [LibVirt](https://libvirt.org/) 37 | - [LibVF.IO](https://libvf.io/) 38 | - [mdevctl](https://github.com/mdevctl/mdevctl) 39 | 40 | # Learn More 41 | 42 | - [Documentation from the OpenMdev Foundation](https://openmdev.io) 43 | 44 | # Compilation 45 | 46 | To compile this program run the following commands: 47 | 48 | `curl -sSL https://get.haskellstack.org/ | sh` 49 | 50 | `stack install` 51 | 52 | The resulting binary file or files (depending on which GVM Project sources are compiled) will be located in ~/.local/bin/ 53 | 54 | This program can take some time to compile (around 10 minutes depending on your system) so if you would prefer to use a pre-built binary instead they are made available under [Releases](https://github.com/Arc-Compute/Mdev-GPU/releases). 55 | -------------------------------------------------------------------------------- /Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /app/Cli/Main.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Main 3 | Description : CLI Client to create MDevs. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | This is where the CLI client is stored in order to provide us access to creating arbitrary MDevs. 11 | -} 12 | {-# LANGUAGE DuplicateRecordFields #-} 13 | {-# LANGUAGE RecordWildCards #-} 14 | {-# LANGUAGE TypeApplications #-} 15 | module Main where 16 | 17 | import Control.Concurrent 18 | import Data.Semigroup ((<>)) 19 | import Data.Text (Text) 20 | import Options.Applicative 21 | import YamlParse.Applicative (confDesc) 22 | 23 | import Common.Config 24 | import Common.Types 25 | import Nvidia.MDev 26 | 27 | data MdevCliControl = MdevCliControl 28 | { config :: Text 29 | , keep :: Bool 30 | } 31 | 32 | control :: Parser MdevCliControl 33 | control = MdevCliControl 34 | <$> strOption 35 | ( long "config" 36 | <> short 'c' 37 | <> metavar "CONFIG_FILE" 38 | <> help "Configuration file to use to generate mdevs (.yaml, .json)" 39 | ) 40 | <*> switch 41 | ( long "keep" 42 | <> short 'k' 43 | <> help "Keeps previous MDevs before preceding." 44 | ) 45 | 46 | main :: IO () 47 | main = mdev =<< execParser opts 48 | where 49 | opts = info (control <**> helper) 50 | ( fullDesc 51 | <> progDesc "Creates arbitrary mediated devices for GPUs." 52 | <> header "Copyright (C) 2022 2666680 Ontario Inc. O\\A Arc Compute\n\nThis program is free software; you can redistribute it and/or\nmodify it under the terms of the GNU General Public License\nas published by the Free Software Foundation; either version 2\nof the License, or (at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program; if not, write to the Free Software\nFoundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA." 53 | <> confDesc @MdevCliConfig 54 | ) 55 | 56 | mkMdev :: [GpuConfig] -> MediatedPhysicalGpu -> Bool -> IO () 57 | mkMdev [] _ _ = return () 58 | mkMdev (curr:rem) mdev disc = do 59 | if length (filteredGpus curr) == 0 60 | then createNvMDevs mdev defaultRequestedGpu disc (mdevConfigs curr) >> 61 | mkMdev rem mdev disc 62 | else mapM_ (\x -> createNvMDevs mdev x disc (mdevConfigs curr)) (filteredGpus curr) >> 63 | mkMdev rem mdev disc 64 | 65 | mdev :: MdevCliControl -> IO () 66 | mdev (MdevCliControl { .. }) = do 67 | configs <- getConfig config 68 | gpu <- openNvMDevInterface True 69 | mkMdev configs gpu (not keep) 70 | closeNvMDevInterface gpu 71 | -------------------------------------------------------------------------------- /etc/gvm/mdev-gpu/Intel/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Arc-Compute/Mdev-GPU/48649a0ed9f2285c883c98610309bd677e8690ae/etc/gvm/mdev-gpu/Intel/.gitkeep -------------------------------------------------------------------------------- /etc/gvm/mdev-gpu/Nvidia/Open_vAmpere.yaml: -------------------------------------------------------------------------------- 1 | gpus: 2 | - 3 | mdevConfigs: 4 | - 5 | num: 529 6 | name: "Open vAmpere" 7 | vDevId: 0x20B01479 8 | maxInstances: 1 9 | fbLen: 38336 10 | fbRes: 2624 11 | virtDisplay: 12 | numHeads: 4 13 | maxResX: 1920 14 | maxResY: 1080 15 | frlConfig: 120 16 | frlEnable: 1 17 | ecc: True 18 | mapVideoSize: 4 19 | bar1Len: 0x800 20 | multiMdev: True 21 | - 22 | num: 530 23 | name: "Open vAmpere/2" 24 | vDevId: 0x20B01471 25 | maxInstances: 2 26 | fbLen: 19136 27 | fbRes: 1344 28 | virtDisplay: 29 | numHeads: 4 30 | maxResX: 1920 31 | maxResY: 1080 32 | frlConfig: 120 33 | frlEnable: 1 34 | ecc: True 35 | mapVideoSize: 4 36 | bar1Len: 0x400 37 | - 38 | num: 531 39 | name: "Open vAmpere/4" 40 | vDevId: 0x20B01470 41 | maxInstances: 4 42 | fbLen: 9536 43 | fbRes: 704 44 | virtDisplay: 45 | numHeads: 4 46 | maxResX: 1920 47 | maxResY: 1080 48 | frlConfig: 120 49 | frlEnable: 1 50 | ecc: True 51 | mapVideoSize: 4 52 | bar1Len: 0x400 53 | - 54 | num: 532 55 | name: "Open vAmpere/8" 56 | vDevId: 0x20B0146F 57 | maxInstances: 8 58 | fbLen: 4736 59 | fbRes: 384 60 | virtDisplay: 61 | numHeads: 4 62 | maxResX: 1920 63 | maxResY: 1080 64 | frlConfig: 120 65 | frlEnable: 1 66 | ecc: True 67 | mapVideoSize: 4 68 | bar1Len: 0x400 69 | -------------------------------------------------------------------------------- /etc/gvm/mdev-gpu/Nvidia/Open_vGeneric.yaml: -------------------------------------------------------------------------------- 1 | gpus: 2 | - 3 | mdevConfigs: 4 | - 5 | num: 1 6 | maxInstances: 10 7 | fbLen: 896 8 | fbRes: 128 9 | name: "GVM [Geforce /10]" 10 | gpuClass: "Geforce" 11 | virtDisplay: 12 | numHeads: 4 13 | maxResX: 3840 14 | maxResY: 2160 15 | frlConfig: 120 16 | frlEnable: 1 17 | ecc: True 18 | mapVideoSize: 4 19 | bar1Len: 0x100 20 | - 21 | num: 2 22 | maxInstances: 10 23 | fbLen: 896 24 | fbRes: 128 25 | name: "GVM [NVS /10]" 26 | gpuClass: "NVS" 27 | virtDisplay: 28 | numHeads: 4 29 | maxResX: 3840 30 | maxResY: 2160 31 | frlConfig: 120 32 | frlEnable: 1 33 | ecc: True 34 | mapVideoSize: 4 35 | bar1Len: 0x100 36 | - 37 | num: 3 38 | maxInstances: 10 39 | fbLen: 896 40 | fbRes: 128 41 | name: "GVM [Quadro /10]" 42 | gpuClass: "Quadro" 43 | virtDisplay: 44 | numHeads: 4 45 | maxResX: 3840 46 | maxResY: 2160 47 | frlConfig: 120 48 | frlEnable: 1 49 | ecc: True 50 | mapVideoSize: 4 51 | bar1Len: 0x100 52 | -------------------------------------------------------------------------------- /etc/gvm/mdev-gpu/Nvidia/Open_vPascal.yaml: -------------------------------------------------------------------------------- 1 | gpus: 2 | - 3 | mdevConfigs: 4 | - 5 | num: 1 6 | maxInstances: 10 7 | fbLen: 896 8 | fbRes: 128 9 | name: "GVM [Geforce /10]" 10 | gpuClass: "Geforce" 11 | vDevId: 0x1B301B30 12 | virtDisplay: 13 | numHeads: 4 14 | maxResX: 3840 15 | maxResY: 2160 16 | frlConfig: 120 17 | frlEnable: 1 18 | ecc: True 19 | mapVideoSize: 4 20 | bar1Len: 0x100 21 | - 22 | num: 2 23 | maxInstances: 10 24 | fbLen: 896 25 | fbRes: 128 26 | name: "GVM [NVS /10]" 27 | gpuClass: "NVS" 28 | vDevId: 0x1B301B30 29 | virtDisplay: 30 | numHeads: 4 31 | maxResX: 3840 32 | maxResY: 2160 33 | frlConfig: 120 34 | frlEnable: 1 35 | ecc: True 36 | mapVideoSize: 4 37 | bar1Len: 0x100 38 | - 39 | num: 3 40 | maxInstances: 10 41 | fbLen: 896 42 | fbRes: 128 43 | name: "GVM [Quadro /10]" 44 | gpuClass: "Quadro" 45 | vDevId: 0x1B301B30 46 | virtDisplay: 47 | numHeads: 4 48 | maxResX: 3840 49 | maxResY: 2160 50 | frlConfig: 120 51 | frlEnable: 1 52 | ecc: True 53 | mapVideoSize: 4 54 | bar1Len: 0x100 55 | -------------------------------------------------------------------------------- /etc/gvm/mdev-gpu/Nvidia/Open_vTuring.yaml: -------------------------------------------------------------------------------- 1 | gpus: 2 | - 3 | mdevConfigs: 4 | - 5 | num: 1 6 | maxInstances: 10 7 | fbLen: 896 8 | fbRes: 128 9 | name: "GVM [Geforce /10]" 10 | gpuClass: "Geforce" 11 | vDevId: 0x1E3012BA 12 | virtDisplay: 13 | numHeads: 4 14 | maxResX: 3840 15 | maxResY: 2160 16 | frlConfig: 120 17 | frlEnable: 1 18 | ecc: True 19 | mapVideoSize: 4 20 | bar1Len: 0x100 21 | - 22 | num: 2 23 | maxInstances: 10 24 | fbLen: 896 25 | fbRes: 128 26 | name: "GVM [NVS /10]" 27 | gpuClass: "NVS" 28 | vDevId: 0x1E3012BA 29 | virtDisplay: 30 | numHeads: 4 31 | maxResX: 3840 32 | maxResY: 2160 33 | frlConfig: 120 34 | frlEnable: 1 35 | ecc: True 36 | mapVideoSize: 4 37 | bar1Len: 0x100 38 | - 39 | num: 3 40 | maxInstances: 10 41 | fbLen: 896 42 | fbRes: 128 43 | name: "GVM [Quadro /10]" 44 | gpuClass: "Quadro" 45 | vDevId: 0x1E3012BA 46 | virtDisplay: 47 | numHeads: 4 48 | maxResX: 3840 49 | maxResY: 2160 50 | frlConfig: 120 51 | frlEnable: 1 52 | ecc: True 53 | mapVideoSize: 4 54 | bar1Len: 0x100 55 | -------------------------------------------------------------------------------- /etc/gvm/mdev-gpu/Tenstorrent/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Arc-Compute/Mdev-GPU/48649a0ed9f2285c883c98610309bd677e8690ae/etc/gvm/mdev-gpu/Tenstorrent/.gitkeep -------------------------------------------------------------------------------- /etc/systemd/system/mdev-post.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Posts the GPU(s) in a state where they are ready to be virtualized. 3 | 4 | [Service] 5 | Type=oneshot 6 | RemainAfterExit=yes 7 | Restart=on-failure 8 | RestartSec=5s 9 | 10 | ExecStart=/usr/bin/sudo /usr/bin/mdev-cli -c /etc/gvm/mdev-gpu/generate-vgpu-types.yaml 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /headroom-templates/haskell.mustache: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : {{{ _haskell_module_name }}} 3 | Description : {{{ _haskell_module_shortdesc }}} 4 | Copyright : (c) {{ _current_year }} 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | {{{ _haskell_module_longdesc }}} 11 | -} 12 | -------------------------------------------------------------------------------- /mdev-gpu.cabal: -------------------------------------------------------------------------------- 1 | cabal-version: 1.12 2 | 3 | -- This file has been generated from package.yaml by hpack version 0.34.4. 4 | -- 5 | -- see: https://github.com/sol/hpack 6 | 7 | name: mdev-gpu 8 | version: 0.1.0.0 9 | description: Please see the README on GitHub at 10 | homepage: https://github.com/Arc-Compute/mdev-gpu#readme 11 | bug-reports: https://github.com/Arc-Compute/mdev-gpu/issues 12 | author: Michael Buchel 13 | maintainer: michael@arccompute.io 14 | copyright: 2666680 Ontario Inc. O\A Arc Compute 15 | license: GPL-2 16 | license-file: LICENSE 17 | build-type: Simple 18 | extra-source-files: 19 | README.md 20 | ChangeLog.md 21 | 22 | source-repository head 23 | type: git 24 | location: https://github.com/Arc-Compute/mdev-gpu 25 | 26 | library 27 | exposed-modules: 28 | Common.Config 29 | Common.Device 30 | Common.Types 31 | Nvidia.Device 32 | Nvidia.Errors 33 | Nvidia.MDev 34 | Nvidia.RMApi 35 | Nvidia.RMApi.IoctlCodes 36 | Nvidia.RMApi.Types 37 | other-modules: 38 | Paths_mdev_gpu 39 | hs-source-dirs: 40 | src 41 | ghc-options: -freduction-depth=0 42 | build-depends: 43 | aeson 44 | , base >=4.7 && <5 45 | , bimap 46 | , containers 47 | , directory 48 | , fixed-vector 49 | , ioctl 50 | , optparse-applicative 51 | , path 52 | , split 53 | , text 54 | , unix 55 | , yamlparse-applicative 56 | default-language: Haskell2010 57 | 58 | executable mdev-cli 59 | main-is: Main.hs 60 | other-modules: 61 | Paths_mdev_gpu 62 | hs-source-dirs: 63 | app/Cli 64 | ghc-options: -freduction-depth=0 -threaded -rtsopts -with-rtsopts=-N 65 | build-depends: 66 | aeson 67 | , base >=4.7 && <5 68 | , bimap 69 | , containers 70 | , directory 71 | , fixed-vector 72 | , ioctl 73 | , mdev-gpu 74 | , optparse-applicative 75 | , path 76 | , split 77 | , text 78 | , unix 79 | , yamlparse-applicative 80 | default-language: Haskell2010 81 | -------------------------------------------------------------------------------- /package.yaml: -------------------------------------------------------------------------------- 1 | name: mdev-gpu 2 | version: 0.1.0.0 3 | github: "Arc-Compute/mdev-gpu" 4 | license: GPL-2 5 | author: "Michael Buchel" 6 | maintainer: "michael@arccompute.io" 7 | copyright: "2666680 Ontario Inc. O\\A Arc Compute" 8 | 9 | extra-source-files: 10 | - README.md 11 | - ChangeLog.md 12 | 13 | # Metadata used when publishing your package 14 | # synopsis: Short description of your package 15 | # category: GPU 16 | 17 | # To avoid duplicated efforts in documentation and dealing with the 18 | # complications of embedding Haddock markup inside cabal files, it is 19 | # common to point users to the README.md file. 20 | description: Please see the README on GitHub at 21 | 22 | dependencies: 23 | - base >= 4.7 && < 5 24 | - optparse-applicative 25 | - yamlparse-applicative 26 | - aeson 27 | - fixed-vector 28 | - bimap 29 | - ioctl 30 | - unix 31 | - split 32 | - text 33 | - directory 34 | - containers 35 | - path 36 | 37 | library: 38 | source-dirs: src 39 | ghc-options: 40 | - -freduction-depth=0 41 | 42 | executables: 43 | mdev-cli: 44 | main: Main.hs 45 | source-dirs: app/Cli 46 | ghc-options: 47 | - -freduction-depth=0 48 | - -threaded 49 | - -rtsopts 50 | - -with-rtsopts=-N 51 | dependencies: 52 | - mdev-gpu 53 | -------------------------------------------------------------------------------- /src/Common/Config.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Common.Config 3 | Description : YAML + JSON Parsing of configuration files. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Configuration module to handle YAML + JSON parsing. 11 | -} 12 | {-# LANGUAGE OverloadedStrings #-} 13 | module Common.Config where 14 | 15 | import Data.Aeson (FromJSON(..), decodeFileStrict) 16 | import Data.Text (Text, isSuffixOf, unpack) 17 | import Path.Posix (parseSomeFile, SomeBase(..), Path(..)) 18 | import YamlParse.Applicative 19 | 20 | import Common.Types 21 | 22 | {-| Mdev configuration layer to help act as a high level wrapper. 23 | -} 24 | data MdevCliConfig = MdevCliConfig 25 | { gpus :: [GpuConfig] -- ^ List of all gpu configurations. 26 | } deriving (Show) 27 | 28 | instance YamlSchema MdevCliConfig where 29 | yamlSchema = 30 | objectParser "MdevCliConfig" $ 31 | MdevCliConfig 32 | <$> requiredField "gpus" "GPU configuration choices to use." 33 | 34 | instance FromJSON MdevCliConfig where 35 | parseJSON = viaYamlSchema 36 | 37 | {-| Wrapper around possible GPU objects. 38 | -} 39 | data GpuConfig = GpuConfig 40 | { filteredGpus :: [RequestedGpu] -- ^ Implement the following GPUs on the filtered GPU contexts. 41 | , mdevConfigs :: [MDevRequest] -- ^ List of mdevs to create. 42 | } deriving (Show) 43 | 44 | instance YamlSchema GpuConfig where 45 | yamlSchema = 46 | objectParser "GpuConfig" $ 47 | GpuConfig 48 | <$> optionalFieldWithDefault "filteredGpus" [] "Adds the mdev requests to the following filter list." 49 | <*> requiredField "mdevConfigs" "The MDev requests to create the VMs." 50 | 51 | instance FromJSON GpuConfig where 52 | parseJSON = viaYamlSchema 53 | 54 | {-| Gets the configuration script and parses it from the system. 55 | -} 56 | getConfig :: Text -- ^ File path to parse. 57 | -> IO ([GpuConfig]) -- ^ Returns the GPU configuration list. 58 | getConfig n = 59 | if isSuffixOf ".json" n 60 | then do 61 | Just mdev <- decodeFileStrict (unpack n) :: IO (Maybe MdevCliConfig) 62 | return $ gpus mdev 63 | else do 64 | d <- parseSomeFile (unpack n) 65 | case d of 66 | Rel f -> do 67 | Just mdev <- readConfigFile f :: IO (Maybe MdevCliConfig) 68 | return $ gpus mdev 69 | Abs f -> do 70 | Just mdev <- readConfigFile f :: IO (Maybe MdevCliConfig) 71 | return $ gpus mdev 72 | -------------------------------------------------------------------------------- /src/Common/Device.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Common.Device 3 | Description : Common functions to interact with \/dev files. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | This file contains a set of common functions which can be 11 | used to ensure that devices are correctly created when necessary. 12 | -} 13 | module Common.Device 14 | ( createCDevId 15 | , getCDevMajor 16 | , getParam 17 | ) where 18 | 19 | import Data.Bits 20 | import Foreign.C.Types 21 | import System.Posix.Types 22 | 23 | {-| Internal function for getting the device id from the \/proc\/devices 24 | file. 25 | -} 26 | getMajor :: String -- ^ Device to search for 27 | -> [String] -- ^ All lines to parse. 28 | -> Maybe Integer -- ^ If the device is in the list get the value. 29 | getMajor _ [] = Nothing 30 | getMajor chartype (curr:remaining) 31 | | chartype == (last $ words curr) = Just (read (head $ words curr) :: Integer) 32 | | otherwise = getMajor chartype remaining 33 | 34 | {-| Gets a param from a driver param file. 35 | -} 36 | getParam :: String -- ^ Param to get. 37 | -> [String] -- ^ All lines to parse. 38 | -> Maybe Integer -- ^ If the param is in the list, get the value. 39 | getParam _ [] = Nothing 40 | getParam param (curr:remaining) 41 | | param == (init $ head $ words curr) = Just (read (last $ words curr) :: Integer) 42 | | otherwise = getParam param remaining 43 | 44 | {-| Creates a device id for if it is necessary to 45 | mknod and create a device that does not exist. 46 | -} 47 | createCDevId :: CULong -- ^ Major of the chardev to create. 48 | -> CULong -- ^ Minor of the chardev to create. 49 | -> CDev -- ^ Chardev's device id for creation. 50 | createCDevId major minor = 51 | fromIntegral $ 52 | ((major .&. 0xFFFFF000) `shift` 32) .|. 53 | ((major .&. 0x00000FFF) `shift` 8) .|. 54 | ((minor .&. 0xFFFFFF00) `shift` 12) .|. 55 | (minor .&. 0x000000FF) 56 | 57 | {-| Gets the Major from the \/proc\/devices file. 58 | 59 | === __Examples__ 60 | 61 | This is an example to get the NVIDIA Chardev from the 62 | system: 63 | 64 | >>> getCDevMajor "nvidia-frontend" 65 | 195 66 | -} 67 | getCDevMajor :: String -- ^ Type of chardev to create. 68 | -> IO (Maybe Integer) -- ^ If the chardev is creatable, than returns 69 | -- the major of the chardev. 70 | getCDevMajor chartype = 71 | readFile "/proc/devices" >>= return . (getMajor chartype) . lines 72 | -------------------------------------------------------------------------------- /src/Common/Types.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Common.Types 3 | Description : Common types between all GPUs. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Here we develop an interface between several different GPU types for the purpose of creating a 11 | common interaction layer with our codebase. 12 | -} 13 | {-# LANGUAGE DuplicateRecordFields #-} 14 | {-# LANGUAGE OverloadedStrings #-} 15 | module Common.Types where 16 | 17 | import Data.Aeson (FromJSON(..)) 18 | import Foreign (Storable(..), Word32, Word64) 19 | import Foreign.C.Types 20 | import System.Posix.Types 21 | import YamlParse.Applicative 22 | 23 | {-| This structure is a bare bones GPU that gives us some information over how 24 | the physical connected GPU is. 25 | -} 26 | data Gpu = Gpu { domain :: CUInt -- ^ Domain for the PCI device. 27 | , bus :: CUInt -- ^ Bus for the PCI device. 28 | , slot :: CUInt -- ^ Slot for the PCI device. 29 | , function :: CUInt -- ^ Function for the PCI device. 30 | , vendorId :: CUInt -- ^ Vendor ID for the PCI device. 31 | , deviceId :: CUInt -- ^ Device ID for the PCI device. 32 | , subvendorId :: CUInt -- ^ Vendor id of the subdevice. 33 | , subdeviceId :: CUInt -- ^ Device id of the subdevice. 34 | , identifier :: CUInt -- ^ Identifier for the GPU. 35 | } 36 | deriving (Show) 37 | 38 | {-| The NVIDIA kernel module implements it's own class based system to interact 39 | with resources inside the kernel and speaking to the GPU. This is an incomplete 40 | class which lacks a lot of the different classes that can be shown. We are 41 | currently in the process of documenting and providing a tool for interacting 42 | with the different allowed classes. This tool is available on the Arc-Compute 43 | github and there is already extensive documentation efforts occuring on OpenMdev. 44 | -} 45 | data NvidiaResource = NvidiaResource -- ^ A resource for NVIDIA GPUs. 46 | { fd :: Maybe Fd -- ^ Some physical resources require 47 | -- open file descriptors for the 48 | -- duration of the resource's lifecycle. 49 | , client :: CUInt -- ^ All resources require a client/root. 50 | , parent :: CUInt -- ^ All resources require a parent. 51 | , object :: CUInt -- ^ All resources have objects. 52 | , oClass :: CUInt -- ^ All resources have a class. 53 | , children :: [(NvidiaResource, Maybe Gpu)] 54 | -- ^ The possible children of the resource. 55 | } 56 | deriving (Show) 57 | 58 | {-| Mediated physical devices is a variant object for different forms of physical GPUs 59 | that are mediated, if it is not possible to initialize GPUs, it returns a NotInitialized 60 | object. 61 | -} 62 | data MediatedPhysicalGpu = NvidiaMdev -- ^ An initialized NVIDIA mediated physical device. 63 | { ctlFd :: Fd -- ^ Control file descriptor. 64 | , root :: CUInt -- ^ Root id for the NVIDIA GPU. 65 | , children :: [(NvidiaResource, Maybe Gpu)] 66 | -- ^ Children which the mediated device can use. 67 | } 68 | | NotInitialized -- ^ A situation when the Mediated device cannot be initialized. 69 | deriving (Show) 70 | 71 | {-| Valid enumerations of supported gpus, for our mediated device creation tool. 72 | -} 73 | data SupportedGpus = Nvidia -- ^ NVIDIA GPUs are available to create mdevs. 74 | deriving (Show) 75 | 76 | {-| Structure to make it possible to fake void pointers in Haskell. 77 | -} 78 | data VoidPtr = VoidPtr 79 | 80 | instance Storable VoidPtr where 81 | sizeOf _ = 0 82 | alignment _ = 0 83 | peek _ = return VoidPtr 84 | poke _ _ = return () 85 | 86 | {-| Structure to make it possible to make a mediated virtual display. 87 | -} 88 | data VirtDisplay = VirtDisplay 89 | { numHeads :: Word32 -- ^ Number of display heads to create. 90 | , maxResX :: Word32 -- ^ Maximum width of a display. 91 | , maxResY :: Word32 -- ^ Maximum height of a display. 92 | , frlConfig :: Word32 -- ^ The maximum FPS before we get limited 93 | , frlEnable :: Word32 -- ^ If we enable frame limiting or not. 94 | } deriving (Show, Eq) 95 | 96 | instance YamlSchema VirtDisplay where 97 | yamlSchema = 98 | objectParser "Virtual Display" $ 99 | VirtDisplay 100 | <$> optionalFieldWithDefault 101 | "numHeads" 102 | 1 103 | "Number of heads on a mediated display." 104 | <*> optionalFieldWithDefault 105 | "maxResX" 106 | 1024 107 | "Maximum width of the mediated display." 108 | <*> optionalFieldWithDefault 109 | "maxResY" 110 | 1024 111 | "Maximum height of the mediated display." 112 | <*> optionalFieldWithDefault 113 | "frlConfig" 114 | 120 115 | "Frame rate limiter for the mediated display." 116 | <*> optionalFieldWithDefault 117 | "frlEnable" 118 | 0 119 | "Do we enable frame rate limiting or not?" 120 | 121 | instance FromJSON VirtDisplay where 122 | parseJSON = viaYamlSchema 123 | 124 | {-| Basic default configuration for a virtual display. 125 | 126 | NOTE: This provides a single display of 1024 x 1024 frame limited at 120 fps. 127 | -} 128 | defaultVirtDisplay :: VirtDisplay 129 | defaultVirtDisplay = VirtDisplay 1 1024 1024 120 0 130 | 131 | {-| Object to request a specific type of GPU to ensure we only generate MDevs for the filtered GPUs. 132 | -} 133 | data RequestedGpu = RequestedGpu { domain :: Maybe Word32 -- ^ Domain for the PCI device. 134 | , bus :: Maybe Word32 -- ^ Bus for the PCI device. 135 | , slot :: Maybe Word32 -- ^ Slot for the PCI device. 136 | , function :: Maybe Word32 -- ^ Function for the PCI device. 137 | , vendorId :: Maybe Word32 -- ^ Vendor ID for the PCI device. 138 | , deviceId :: Maybe Word32 -- ^ Device ID for the PCI device. 139 | , subvendorId :: Maybe Word32 -- ^ Vendor id of the subdevice. 140 | , subdeviceId :: Maybe Word32 -- ^ Device id of the subdevice. 141 | , identifier :: Maybe Word32 -- ^ Identifier for the GPU. 142 | } 143 | deriving (Show) 144 | 145 | instance YamlSchema RequestedGpu where 146 | yamlSchema = 147 | objectParser "RequestedGpu" $ 148 | RequestedGpu 149 | <$> optionalField "domain" "Domain for the PCI device." 150 | <*> optionalField "bus" "Bus for the PCI device." 151 | <*> optionalField "slot" "Slot for the PCI device." 152 | <*> optionalField "function" "Function for the PCI device." 153 | <*> optionalField "vendorId" "Vendor ID for the PCI device." 154 | <*> optionalField "deviceId" "Device ID for the PCI device." 155 | <*> optionalField "subvendorId" "Vendor ID of the subdevice." 156 | <*> optionalField "subdeviceId" "Device ID of the subdevice." 157 | <*> optionalField "identifier" "Internal GPU identification tag." 158 | 159 | instance FromJSON RequestedGpu where 160 | parseJSON = viaYamlSchema 161 | 162 | {-| Device to filter GPUs based on this structure. 163 | -} 164 | defaultRequestedGpu :: RequestedGpu 165 | defaultRequestedGpu = 166 | RequestedGpu Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing 167 | 168 | {-| Request to provide an MDev device. 169 | -} 170 | data MDevRequest = MDevRequest 171 | { num :: Word32 -- ^ Mdev number to use. 172 | , vDevId :: Maybe Word64 -- ^ Virtual device id to use (guest). 173 | , pDevId :: Maybe Word64 -- ^ Virtual device id to use (host). 174 | , name :: String -- ^ Name to use for the MDev. 175 | , gpuClass :: String -- ^ Class of the GPU to create. 176 | , maxInstances :: Word32 -- ^ Maximum number of instances. 177 | , virtDisplay :: Maybe VirtDisplay -- ^ If a virtual display is to be made. 178 | , ecc :: Bool -- ^ If we enable ECC. 179 | , multiMdev :: Bool -- ^ If we support multiple MDev mode. 180 | , fbLen :: Word32 -- ^ Frame buffer length (IN MEGABYTES). 181 | , fbRes :: Word32 -- ^ Frame buffer reserved (IN MEGABYTES). 182 | , mapVideoSize :: Word32 -- ^ Mappable video size (IN MEGABYTES). 183 | , encCap :: Maybe Word32 -- ^ What percentage of the encoder will 184 | -- be allocated to this GPU. 185 | , bar1Len :: Maybe Word32 -- ^ Bar 1 length. 186 | } deriving (Show) 187 | 188 | instance YamlSchema MDevRequest where 189 | yamlSchema = 190 | objectParser "MDevRequest" $ 191 | MDevRequest 192 | <$> requiredField "num" "MDev number to assign." 193 | <*> optionalField "vDevId" "Spoofing of the GPU to the guest." 194 | <*> optionalField "pDevId" "Spoofing of the GPU to the host." 195 | <*> optionalFieldWithDefault "name" "Arc GPU" "Name presented by the guest." 196 | <*> optionalFieldWithDefault "gpuClass" "Compute" "Class of GPU to use." 197 | <*> requiredField "maxInstances" "Number of instances to allocate." 198 | <*> optionalField "virtDisplay" "Virtual display to create for the MDev." 199 | <*> optionalFieldWithDefault "ecc" False "If ECC is enabled or not." 200 | <*> optionalFieldWithDefault "multiMdev" False "If we enable adding multiple mdevs into the same VM. (EXPERIMENTAL)" 201 | <*> requiredField "fbLen" "GPU vRAM size - reserved section in MBs." 202 | <*> requiredField "fbRes" "GPU vRAM reserved section in MBs." 203 | <*> optionalFieldWithDefault "mapVideoSize" 24 "Mappable video size in MBs, 24 by default." 204 | <*> optionalField "encCap" "Percentage we use the NvEnc hardware." 205 | <*> optionalField "bar1Len" "Bar 1 length, VERY EXPERIMENTAL." 206 | 207 | instance FromJSON MDevRequest where 208 | parseJSON = viaYamlSchema 209 | 210 | {-| Default mdev request. 211 | -} 212 | defaultMDevRequest :: MDevRequest 213 | defaultMDevRequest = 214 | MDevRequest 215 | 0 216 | Nothing 217 | Nothing 218 | "ARC GPU" 219 | "Compute" 220 | 1 221 | Nothing 222 | False 223 | False 224 | 976 225 | 48 226 | 24 227 | Nothing 228 | Nothing 229 | -------------------------------------------------------------------------------- /src/Nvidia/Device.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Nvidia.Device 3 | Description : Code to interact with a low level implementation of the nvidia device. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Here we hold all the internal code for low level nvidia device control. Such as opening and closing nvidia device 11 | files. 12 | -} 13 | {-# LANGUAGE OverloadedStrings #-} 14 | module Nvidia.Device 15 | ( openNvDev 16 | ) where 17 | 18 | import Data.Bits 19 | import Data.Maybe 20 | import Foreign.C.Types 21 | import System.Directory 22 | import System.Posix.IO 23 | import System.Posix.Files 24 | import System.Posix.Types 25 | 26 | -- Internal dependancies 27 | import Common.Device 28 | 29 | {-| Internal function to create the device name, and ensure the file exists. 30 | -} 31 | createNvDevName :: CUInt -- ^ Minor number to open. 32 | -> IO (Maybe String) -- ^ Nvidia device name. 33 | createNvDevName minor = 34 | let suffix = if minor == 255 then "ctl" else show minor 35 | filename = "/dev/nvidia" ++ suffix 36 | in do 37 | exists <- fileExist filename 38 | driverLoaded <- doesFileExist "/proc/driver/nvidia/params" 39 | if exists 40 | then return (Just filename) 41 | else if driverLoaded 42 | then do 43 | putStrLn $ "Creating the filename: " ++ filename 44 | charDev <- getCDevMajor "nvidia-frontend" 45 | mode <- readFile "/proc/driver/nvidia/params" >>= return . (getParam "DeviceFileMode") . lines 46 | if isJust mode && isJust charDev 47 | then createDevice 48 | filename 49 | (fromIntegral ((fromIntegral (fromJust mode) :: CUInt) .|. 0x00002000) :: CMode) 50 | (createCDevId (fromIntegral (fromJust charDev)) (fromIntegral minor)) >> 51 | return (Just filename) 52 | else return Nothing 53 | else return Nothing 54 | 55 | {-| Opens a nvidia device, if the device does not exist, this function creates it. 56 | -} 57 | openNvDev :: CUInt -- ^ Minor number for a \/dev\/nvidia%d number (255 == ctl) 58 | -> IO (Maybe Fd) -- ^ If we are able to load a nvidia device. 59 | openNvDev minor = 60 | createNvDevName minor >>= 61 | \s -> 62 | (if isNothing s 63 | then return Nothing 64 | else openFd (fromJust s) ReadWrite Nothing defaultFileFlags >>= return . Just) 65 | -------------------------------------------------------------------------------- /src/Nvidia/Errors.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Nvidia.Errors 3 | Description : Nvidia Error messages. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Error messages for NVIDIA. 11 | -} 12 | module Nvidia.Errors 13 | ( NvErrorStatus(..) 14 | ) 15 | where 16 | 17 | import Data.Bimap 18 | import Foreign (Storable(..)) 19 | import Foreign.C.Types 20 | 21 | {-| All possible NVIDIA error statuses. 22 | -} 23 | data NvErrorStatus = NvOk -- ^ Ok message, the command succeeded correctly. 24 | | NvBrokenFb -- ^ We have a broken framebuffer. 25 | | NvSmallFb -- ^ The framebuffer is too small. 26 | | NvBusyRetry -- ^ The command was busy, please retry after a timeout. 27 | | NvCallbackNotScheduled -- ^ The callback was not scheduled. 28 | | NvCardNotPresent -- ^ The card is not present on the system. 29 | | NvCallCycle -- ^ Call cycle was detected. 30 | | NvDMAInUse -- ^ The DMA is currently in use. 31 | | NvDMANotLocked -- ^ Could not lock the requested DMA. 32 | | NvDMANotUnlocked -- ^ Could not unlock the requested DMA. 33 | | NvDualLink -- ^ Dual Link is currently in use. 34 | | NvECC -- ^ Generic ECC Error. 35 | | NvFIFOBadAccess -- ^ Invalid access of FIFO. 36 | | NvFreqNotSupported -- ^ Requested frequency is not supported. 37 | | NvDMANotInitialized -- ^ Requested DMA is not yet initialized. 38 | | NvLostFromBus -- ^ GPU was lost from the bus. 39 | | NvFullChipReset -- ^ GPU is currently in a full-chip reset. 40 | | NvNotFullPower -- ^ GPU is not operating at full power. 41 | | NvUuidNotFound -- ^ No UUID associated with GPU. 42 | | NvHotSwitch -- ^ System is in a hot switch. 43 | | NvI2C -- ^ Generic I2C error. 44 | | NvI2CSpeedTooHigh -- ^ I2C speed is too high. 45 | | NvIllegalAction -- ^ Illegal action attempted. 46 | | NvInUse -- ^ Generic in use error. 47 | | NvInflateCompressedDataFailed -- ^ Failed to inflate compressed data. 48 | | NvInsertDuplicateName -- ^ Found a duplicate entry in the requested B-tree. 49 | | NvInsufficientResources -- ^ Ran out of a critical resource, other than memory. 50 | | NvInsufficientPermissions -- ^ The requestor does not have sufficient permissions. 51 | | NvInsufficientPower -- ^ Low power. 52 | | NvInvalidAccessType -- ^ This type of access is not allowed. 53 | | NvInvalidAddress -- ^ Address is invalid. 54 | | NvInvalidArgument -- ^ The given argument is invalid. 55 | | NvInvalidBase -- ^ The base is invalid. 56 | | NvInvalidChannel -- ^ Given channel-id is invalid. 57 | | NvInvalidClass -- ^ Given class-id is invalid. 58 | | NvInvalidClient -- ^ Client is invalid. 59 | | NvInvalidCommand -- ^ Command passed is invalid. 60 | | NvInvalidData -- ^ Data is invalid. 61 | | NvInvalidDevice -- ^ Device passed is currently invalid. 62 | | NvInvalidDMASpecifier -- ^ Requested DMA specifier is invalid. 63 | | NvInvalidEvent -- ^ Invalid event occured. 64 | | NvInvalidFlags -- ^ Invalid flags passed. 65 | | NvInvalidFunction -- ^ The called function is invalid. 66 | | NvInvalidHeap -- ^ Heap is corrupted. 67 | | NvInvalidIndex -- ^ The index is invalid. 68 | | NvInvalidIRQLevel -- ^ Requested IRQ level is invalid. 69 | | NvInvalidLimit -- ^ Invalid limit. 70 | | NvInvalidLockState -- ^ Requested lock state is invalid. 71 | | NvInvalidMethod -- ^ Requested method is invalid. 72 | | NvInvalidObject -- ^ Requested object is invalid. 73 | | NvInvalidObjectBuffer -- ^ Object buffer is invalid. 74 | | NvInvalidObjectHandle -- ^ Object handle is invalid. 75 | | NvInvalidObjectNew -- ^ New object is invalid. 76 | | NvInvalidObjectOld -- ^ Old object is invalid. 77 | | NvInvalidObjectParent -- ^ Object's parent is invalid. 78 | | NvInvalidOffset -- ^ The offset is invalid. 79 | | NvInvalidOperation -- ^ Requested operation is invalid. 80 | | NvInvalidOwner -- ^ Invalid owner. 81 | | NvInvalidParamStruct -- ^ Invalid structure parameters. 82 | | NvInvalidParameter -- ^ Invalid parameters. 83 | | NvInvalidPath -- ^ Requested path is invalid. 84 | | NvInvalidPointer -- ^ Invalid pointer. 85 | | NvInvalidRegistryKey -- ^ Invalid registry key. 86 | | NvInvalidRequest -- ^ Invalid request. 87 | | NvInvalidState -- ^ Invalid state. 88 | | NvInvalidStringLength -- ^ String length invalid. 89 | | NvInvalidRead -- ^ Read operation is invalid. 90 | | NvInvalidWrite -- ^ Write operation is invalid. 91 | | NvInvalidXlate -- ^ Translation operation is invalid. 92 | | NvIRQNotFiring -- ^ Requested IRQ is not firing. 93 | | NvIRQEdgeTriggered -- ^ IRQ is edge triggered. 94 | | NvMemoryTrainingFailed -- ^ Failed the memory training sequence. 95 | | NvMismatchedSlave -- ^ Slave mismatch. 96 | | NvMismatchedTarget -- ^ Target mismatch. 97 | | NvMissingTableEntry -- ^ Requested entry missing from table. 98 | | NvModuleLoadFailed -- ^ Failed to load requested module. 99 | | NvMoreDataAvailable -- ^ There is more data available. 100 | | NvMoreProcessingRequired -- ^ More processing is required for the given call. 101 | | NvMultipleMemoryTypes -- ^ Multiple memory types found. 102 | | NvNoFreeFifos -- ^ No more free FIFOs. 103 | | NvNoIntrPending -- ^ No interrupt is pending. 104 | | NvNoMemory -- ^ Out of memory. 105 | | NvNoSuchDomain -- ^ Requested domain doesn't exist. 106 | | NvNoValidPath -- ^ Caller did not specify a valid path. 107 | | NvNotCompatible -- ^ Incompatible types. 108 | | NvNotReady -- ^ Not ready. 109 | | NvNotSupported -- ^ Call is not supported. 110 | | NvObjectNotFound -- ^ Requested object is not found. 111 | | NvObjectTypeMismatch -- ^ Specified objects mismatch. 112 | | NvOperatingSystem -- ^ Operating system error. 113 | | NvFoundOtherDevice -- ^ Found other device requested one. 114 | | NvOutOfRange -- ^ Value is out of bounds. 115 | | NvOverlappingUVMCommit -- ^ Overlapping unified virtual memory commit. 116 | | NvPageTableNotAvailable -- ^ Requested page table unavailable. 117 | | NvPIDNotFound -- ^ PID is not found. 118 | | NvProtectionFault -- ^ Protection fault. 119 | | NvRCError -- ^ Generic RC error. 120 | | NvRejectedVBios -- ^ Rejected VBios. 121 | | NvResetRequired -- ^ Reset required. 122 | | NvStateInUse -- ^ State is currently in use. 123 | | NvSignalPending -- ^ Signal pending. 124 | | NvTimeout -- ^ Call timed out. 125 | | NvTimeoutRetry -- ^ Call timed out, retry later. 126 | | NvTooManyPrimaries -- ^ Too many primaries. 127 | | NvUVMAddressInUse -- ^ Unified virtual address is currently in use. 128 | | NvMaxSessionLimitReached -- ^ Maximum number of sessions reached. 129 | | NvRMLibVersionMismatch -- ^ RM library version mismatch. 130 | | NvPrivSecurity -- ^ Priv security violation. 131 | | NvGPUInDebug -- ^ GPU is in debug mode. 132 | | NvFeatureNotEnabled -- ^ Feature is not currently enabled. 133 | | NvResourceLost -- ^ Resource was lost. 134 | | NvPMUNotReady -- ^ PMU is not ready or initialized yet. 135 | | NvFLCNAssert -- ^ Falcon assertion/error. 136 | | NvFatalError -- ^ Fatal/unrecoverable error. 137 | | NvMemoryError -- ^ Memory error. 138 | | NvInvalidLicense -- ^ Invalid license. 139 | | NvNVLinkInit -- ^ NVLink init error. 140 | | NvNVLinkMinion -- ^ NVLink minion error. 141 | | NvNVLinkClock -- ^ NVLink clock error. 142 | | NvNVLinkTraining -- ^ NVLink training error. 143 | | NvNVLinkConfiguration -- ^ NVLink configuration error. 144 | | NvRISCVError -- ^ Generic RISC-V error. 145 | | NvHotSwitchWarn -- ^ WARNING Hot switch. 146 | | NvIncorrectPerfMonDataWarn -- ^ WARNING Incorrect performance monitoring data. 147 | | NvSlaveMismatchWarn -- ^ WARNING Slave mismatch. 148 | | NvTargetMismatchWarn -- ^ WARNING Target mismatch. 149 | | NvMoreProcessingWarn -- ^ WARNING More processing required for this call. 150 | | NvNothingWarn -- ^ WARNING Nothing to do. 151 | | NvNullWarn -- ^ WARNING Null object found. 152 | | NvOutOfRangeWarn -- ^ WARNING Value out of range. 153 | | NvGeneric -- ^ Generic RM error. 154 | deriving (Ord, Eq, Show) 155 | 156 | {-| Internal error number. 157 | -} 158 | errorEnum :: Bimap NvErrorStatus CUInt 159 | errorEnum = fromList [ (NvOk, 0x00000000) 160 | , (NvBrokenFb, 0x00000001) 161 | , (NvSmallFb, 0x0000002) 162 | , (NvBusyRetry, 0x00000003) 163 | , (NvCallbackNotScheduled, 0x00000004) 164 | , (NvCardNotPresent, 0x00000005) 165 | , (NvCallCycle, 0x00000006) 166 | , (NvDMAInUse, 0x00000007) 167 | , (NvDMANotLocked, 0x00000008) 168 | , (NvDMANotUnlocked, 0x00000009) 169 | , (NvDualLink, 0x0000000A) 170 | , (NvECC, 0x0000000B) 171 | , (NvFIFOBadAccess, 0x0000000C) 172 | , (NvFreqNotSupported, 0x0000000D) 173 | , (NvDMANotInitialized, 0x0000000E) 174 | , (NvLostFromBus, 0x0000000F) 175 | , (NvFullChipReset, 0x00000010) 176 | , (NvNotFullPower, 0x00000011) 177 | , (NvUuidNotFound, 0x00000012) 178 | , (NvHotSwitch, 0x00000013) 179 | , (NvI2C, 0x00000014) 180 | , (NvI2CSpeedTooHigh, 0x00000015) 181 | , (NvIllegalAction, 0x00000016) 182 | , (NvInUse, 0x00000017) 183 | , (NvInflateCompressedDataFailed, 0x00000018) 184 | , (NvInsertDuplicateName, 0x00000019) 185 | , (NvInsufficientResources, 0x0000001A) 186 | , (NvInsufficientPermissions, 0x0000001B) 187 | , (NvInsufficientPower, 0x0000001C) 188 | , (NvInvalidAccessType, 0x0000001D) 189 | , (NvInvalidAddress, 0x0000001E) 190 | , (NvInvalidArgument, 0x0000001F) 191 | , (NvInvalidBase, 0x00000020) 192 | , (NvInvalidChannel, 0x00000021) 193 | , (NvInvalidClass, 0x00000022) 194 | , (NvInvalidClient, 0x00000023) 195 | , (NvInvalidCommand, 0x00000024) 196 | , (NvInvalidData, 0x00000025) 197 | , (NvInvalidDevice, 0x00000026) 198 | , (NvInvalidDMASpecifier, 0x00000027) 199 | , (NvInvalidEvent, 0x00000028) 200 | , (NvInvalidFlags, 0x00000029) 201 | , (NvInvalidFunction, 0x0000002A) 202 | , (NvInvalidHeap, 0x0000002B) 203 | , (NvInvalidIndex, 0x0000002C) 204 | , (NvInvalidIRQLevel, 0x0000002D) 205 | , (NvInvalidLimit, 0x0000002E) 206 | , (NvInvalidLockState, 0x0000002F) 207 | , (NvInvalidMethod, 0x00000030) 208 | , (NvInvalidObject, 0x00000031) 209 | , (NvInvalidObjectBuffer, 0x00000032) 210 | , (NvInvalidObjectHandle, 0x00000033) 211 | , (NvInvalidObjectNew, 0x00000034) 212 | , (NvInvalidObjectOld, 0x00000035) 213 | , (NvInvalidObjectParent, 0x00000036) 214 | , (NvInvalidOffset, 0x00000037) 215 | , (NvInvalidOperation, 0x00000038) 216 | , (NvInvalidOwner, 0x00000039) 217 | , (NvInvalidParamStruct, 0x0000003A) 218 | , (NvInvalidParameter, 0x0000003B) 219 | , (NvInvalidPath, 0x0000003C) 220 | , (NvInvalidPointer, 0x0000003D) 221 | , (NvInvalidRegistryKey, 0x0000003E) 222 | , (NvInvalidRequest, 0x0000003F) 223 | , (NvInvalidState, 0x00000040) 224 | , (NvInvalidStringLength, 0x00000041) 225 | , (NvInvalidRead, 0x00000042) 226 | , (NvInvalidWrite, 0x00000043) 227 | , (NvInvalidXlate, 0x00000044) 228 | , (NvIRQNotFiring, 0x00000045) 229 | , (NvIRQEdgeTriggered, 0x00000046) 230 | , (NvMemoryTrainingFailed, 0x00000047) 231 | , (NvMismatchedSlave, 0x00000048) 232 | , (NvMismatchedTarget, 0x00000049) 233 | , (NvMissingTableEntry, 0x0000004A) 234 | , (NvModuleLoadFailed, 0x0000004B) 235 | , (NvMoreDataAvailable, 0x0000004C) 236 | , (NvMoreProcessingRequired, 0x0000004D) 237 | , (NvMultipleMemoryTypes, 0x0000004E) 238 | , (NvNoFreeFifos, 0x0000004F) 239 | , (NvNoIntrPending, 0x00000050) 240 | , (NvNoMemory, 0x00000051) 241 | , (NvNoSuchDomain, 0x00000052) 242 | , (NvNoValidPath, 0x00000053) 243 | , (NvNotCompatible, 0x00000054) 244 | , (NvNotReady, 0x00000055) 245 | , (NvNotSupported, 0x00000056) 246 | , (NvObjectNotFound, 0x00000057) 247 | , (NvObjectTypeMismatch, 0x00000058) 248 | , (NvOperatingSystem, 0x00000059) 249 | , (NvFoundOtherDevice, 0x0000005A) 250 | , (NvOutOfRange, 0x0000005B) 251 | , (NvOverlappingUVMCommit, 0x0000005C) 252 | , (NvPageTableNotAvailable, 0x0000005D) 253 | , (NvPIDNotFound, 0x0000005E) 254 | , (NvProtectionFault, 0x0000005F) 255 | , (NvRCError, 0x00000060) 256 | , (NvRejectedVBios, 0x00000061) 257 | , (NvResetRequired, 0x00000062) 258 | , (NvStateInUse, 0x00000063) 259 | , (NvSignalPending, 0x00000064) 260 | , (NvTimeout, 0x00000065) 261 | , (NvTimeoutRetry, 0x00000066) 262 | , (NvTooManyPrimaries, 0x00000067) 263 | , (NvUVMAddressInUse, 0x00000068) 264 | , (NvMaxSessionLimitReached, 0x00000069) 265 | , (NvRMLibVersionMismatch, 0x0000006A) 266 | , (NvPrivSecurity, 0x0000006B) 267 | , (NvGPUInDebug, 0x0000006C) 268 | , (NvFeatureNotEnabled, 0x0000006D) 269 | , (NvResourceLost, 0x0000006E) 270 | , (NvPMUNotReady, 0x0000006F) 271 | , (NvFLCNAssert, 0x00000070) 272 | , (NvFatalError, 0x00000071) 273 | , (NvMemoryError, 0x00000072) 274 | , (NvInvalidLicense, 0x00000073) 275 | , (NvNVLinkInit, 0x00000074) 276 | , (NvNVLinkMinion, 0x00000075) 277 | , (NvNVLinkClock, 0x00000076) 278 | , (NvNVLinkTraining, 0x00000077) 279 | , (NvNVLinkConfiguration, 0x00000078) 280 | , (NvRISCVError, 0x00000079) 281 | , (NvHotSwitchWarn, 0x00010001) 282 | , (NvIncorrectPerfMonDataWarn, 0x00010002) 283 | , (NvSlaveMismatchWarn, 0x00010003) 284 | , (NvTargetMismatchWarn, 0x00010004) 285 | , (NvMoreProcessingWarn, 0x00010005) 286 | , (NvNothingWarn, 0x00010006) 287 | , (NvNullWarn, 0x00010007) 288 | , (NvOutOfRangeWarn, 0x00010008) 289 | , (NvGeneric, 0x0000FFFF) 290 | ] 291 | 292 | {-| Internal from error. 293 | -} 294 | fromError :: NvErrorStatus -> CUInt 295 | fromError x = errorEnum ! x 296 | 297 | {-| Internal to error. 298 | -} 299 | toError :: CUInt -> NvErrorStatus 300 | toError x = errorEnum !> x 301 | 302 | instance Storable NvErrorStatus where 303 | sizeOf _ = 4 304 | alignment _ = 4 305 | peek ptr = peekByteOff ptr 0 >>= return . toError 306 | poke ptr d = pokeByteOff ptr 0 (fromError d) 307 | -------------------------------------------------------------------------------- /src/Nvidia/MDev.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Nvidia.MDev 3 | Description : NVIDIA MDev interface implementation. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | This file creates mdev interface commands which can be used to provide a better communication with 11 | specifically the NVIDIA Kernel module. 12 | -} 13 | {-# LANGUAGE DataKinds #-} 14 | {-# LANGUAGE DuplicateRecordFields #-} 15 | {-# LANGUAGE FlexibleContexts #-} 16 | {-# LANGUAGE RecordWildCards #-} 17 | module Nvidia.MDev 18 | ( openNvMDevInterface 19 | , closeNvMDevInterface 20 | , createNvMDevs 21 | ) where 22 | 23 | import Data.Bits 24 | import Data.Maybe 25 | import Foreign (Storable(..), Word32) 26 | import Foreign.C.Types 27 | import Foreign.C.String 28 | import Foreign.Ptr 29 | import Foreign.Marshal.Utils 30 | import System.Posix.IO 31 | import System.Posix.Types 32 | import Text.Printf 33 | 34 | import qualified Data.Vector.Fixed as V 35 | 36 | -- Internal imports. 37 | import Common.Types 38 | import Nvidia.Device 39 | import Nvidia.RMApi 40 | 41 | {-| Function to create a list of MDevs on a requested GPU. 42 | -} 43 | createNvMDevs :: MediatedPhysicalGpu -- ^ Device to create mdevs on. 44 | -> RequestedGpu -- ^ Requested GPU to create the mdevs. 45 | -> Bool -- ^ Discard previous gpus. 46 | -> [MDevRequest] -- ^ List of mediated devices to create. 47 | -> IO () -- ^ Creates the mediated devices requested by the user. 48 | createNvMDevs mpgpu rgpu discard requests = do 49 | let filteredResources = 50 | filter (\(_, gpu) -> validGpu gpu) 51 | (children (fst $ head (children (mpgpu :: MediatedPhysicalGpu)) :: NvidiaResource)) 52 | vgpuConfigurators = 53 | map (\(nvg, Just gpu) -> 54 | (fst $ head (children ((fst $ head (children (nvg :: NvidiaResource))) :: NvidiaResource)), gpu) 55 | ) filteredResources 56 | ctl = (ctlFd (mpgpu :: MediatedPhysicalGpu)) 57 | mapM_ (\(vgpuConfig, gpu) -> do 58 | let 59 | c = (client (vgpuConfig :: NvidiaResource)) 60 | o = (object (vgpuConfig :: NvidiaResource)) 61 | v = fromIntegral (vendorId (gpu :: Gpu)) :: Integer 62 | d = fromIntegral (deviceId (gpu :: Gpu)) :: Integer 63 | sv = fromIntegral (subvendorId (gpu :: Gpu)) :: Integer 64 | sd = fromIntegral (subdeviceId (gpu :: Gpu)) :: Integer 65 | printf "Attempting to create MDEVs on (0x%.4X, 0x%.4X, 0x%.4X, 0x%.4X)\n" v d sv sd 66 | mapM_ (\(idx, mdevReq) -> do 67 | let discardPrev = discard && (idx == 0) 68 | virtDisp = if isJust (virtDisplay (mdevReq :: MDevRequest)) 69 | then fromJust (virtDisplay (mdevReq :: MDevRequest)) 70 | else defaultVirtDisplay 71 | mx = fromIntegral $ maxResX (virtDisp :: VirtDisplay) 72 | my = fromIntegral $ maxResY (virtDisp :: VirtDisplay) 73 | ec = if isJust (encCap (mdevReq :: MDevRequest)) 74 | then fromJust (encCap (mdevReq :: MDevRequest)) 75 | else 100 76 | vdi = if isJust (vDevId (mdevReq :: MDevRequest)) 77 | then fromJust (vDevId (mdevReq :: MDevRequest)) 78 | else fromIntegral ((d `shift` 16) .|. sd) 79 | pdi = if isJust (pDevId (mdevReq :: MDevRequest)) 80 | then fromJust (pDevId (mdevReq :: MDevRequest)) 81 | else fromIntegral d 82 | b1Len = if isJust (bar1Len (mdevReq :: MDevRequest)) 83 | then fromJust (bar1Len (mdevReq :: MDevRequest)) 84 | else 0x100 85 | cfg = defaultRmVgpuConfig { discardPrevious = if discardPrev then 1 else 0 86 | , vgpuTypeNumber = 87 | fromIntegral (num (mdevReq :: MDevRequest)) 88 | , vgpuName = castToVec (name (mdevReq :: MDevRequest)) 89 | , vgpuClass = castToVec (gpuClass (mdevReq :: MDevRequest)) 90 | , maxInstances = 91 | fromIntegral (maxInstances (mdevReq :: MDevRequest)) 92 | , numHeads = fromIntegral (numHeads (virtDisp :: VirtDisplay)) 93 | , maxResX = mx 94 | , maxResY = my 95 | , maxPixel = mx * my 96 | , frlConfig = fromIntegral (frlConfig (virtDisp :: VirtDisplay)) 97 | , cuda = 1 98 | , eccSupported = if (ecc (mdevReq :: MDevRequest)) 99 | then 1 100 | else 0 101 | , multiMdev = if (multiMdev (mdevReq :: MDevRequest)) 102 | then 1 103 | else 0 104 | , encCap = fromIntegral ec 105 | , vdevId = fromIntegral vdi 106 | , pdevId = fromIntegral pdi 107 | , fbLen = toBytes (fbLen (mdevReq :: MDevRequest)) 108 | , mappableVideoSize = toBytes (mapVideoSize (mdevReq :: MDevRequest)) 109 | , fbRes = toBytes (fbRes (mdevReq :: MDevRequest)) 110 | , bar1Len = fromIntegral b1Len 111 | , frlEnable = fromIntegral (frlEnable (virtDisp :: VirtDisplay)) 112 | } :: RmVgpuConfig 113 | _ <- with cfg (\par -> ctrlRes_ ctl c o 0xA0810101 par) 114 | printf "Created mdev number %d: %s\n" idx (show mdevReq)) 115 | (zip ([0..] :: [Integer]) requests) 116 | ctrlRes' ctl c o 0xA0810109 117 | printf "Created MDEVs on (0x%.4X, 0x%.4X, 0x%.4X, 0x%.4X)\n" v d sv sd 118 | ) 119 | vgpuConfigurators 120 | where 121 | isEqual :: Maybe Word32 -> CUInt -> Bool 122 | isEqual (Just x) y = (fromIntegral x) == y 123 | isEqual Nothing _ = True 124 | 125 | validGpu :: Maybe Gpu -> Bool 126 | validGpu (Just gpu) = 127 | (isEqual (domain (rgpu :: RequestedGpu)) (domain (gpu :: Gpu))) && 128 | (isEqual (bus (rgpu :: RequestedGpu)) (bus (gpu :: Gpu))) && 129 | (isEqual (slot (rgpu :: RequestedGpu)) (slot (gpu :: Gpu))) && 130 | (isEqual (function (rgpu :: RequestedGpu)) (function (gpu :: Gpu))) && 131 | (isEqual (vendorId (rgpu :: RequestedGpu)) (vendorId (gpu :: Gpu))) && 132 | (isEqual (deviceId (rgpu :: RequestedGpu)) (deviceId (gpu :: Gpu))) && 133 | (isEqual (subvendorId (rgpu :: RequestedGpu)) (subvendorId (gpu :: Gpu))) && 134 | (isEqual (subdeviceId (rgpu :: RequestedGpu)) (subdeviceId (gpu :: Gpu))) && 135 | (isEqual (identifier (rgpu :: RequestedGpu)) (identifier (gpu :: Gpu))) 136 | validGpu _ = error "No GPU found." 137 | 138 | castToVec s = V.fromList ((map castCharToCChar s) ++ repeat 0) 139 | toBytes x = (fromIntegral x :: CULong) * 1024 * 1024 140 | 141 | {-| Frees an internal resource tree from the GPU kernel module. 142 | -} 143 | freeNvResTree :: Fd -- ^ File descriptor to free the resource on. 144 | -> [(NvidiaResource, Maybe Gpu)] -- ^ Resource tree. 145 | -> IO () -- ^ Cleans the resources from the nvidia resource tree. 146 | freeNvResTree _ [] = return () 147 | freeNvResTree ctlFd resources = 148 | mapM_ 149 | (\(NvidiaResource { .. }, a) -> 150 | let freeRm = defaultRmFreeRes { hRoot = client 151 | , hObjectParent = parent 152 | , hObjectOld = object 153 | } :: RmFreeRes 154 | in freeNvResTree ctlFd children >> freeRes ctlFd freeRm >> 155 | if isJust fd 156 | then if isJust a 157 | then let gpuId = identifier ((fromJust a) :: Gpu) 158 | detach = defaultNv0000CtrlGpuDetachIdsParams { gpuIds = V.fromList ([ gpuId 159 | , 0xFFFFFFFF 160 | ] ++ repeat 0) 161 | } :: Nv0000CtrlGpuDetachIdsParams 162 | in with detach (\par -> ctrlRes_ ctlFd client client 0x216 par) >> 163 | closeFd (fromJust fd) 164 | else closeFd (fromJust fd) 165 | else return ()) 166 | resources 167 | 168 | {-| Command to get the GPU list from the kernel. 169 | -} 170 | getGpus :: Fd -- ^ File descriptor for the control file. 171 | -> CUInt -- ^ Client ID to allocate commands on. 172 | -> IO ([(NvidiaResource, Maybe Gpu)]) -- ^ Returns a list of all known resources. 173 | getGpus fd clientId = do 174 | probedIds <- with defaultNv0000CtrlGpuGetProbedIdsParams (\par -> ctrlRes fd clientId clientId 0x214 par >>= peek) 175 | let probedIds' = zip [0..] $ filter (/= 0xFFFFFFFF) $ V.toList (gpuIds (probedIds :: Nv0000CtrlGpuGetProbedIdsParams)) 176 | devices <- mapM (\(idx, x) -> 177 | let 178 | idInfoParams = defaultNv0000CtrlGpuGetIdInfoParams { gpuId = x } :: Nv0000CtrlGpuGetIdInfoParams 179 | pciInfo = defaultNv0000CtrlGpuGetPciInfoParams { gpuId = x } :: Nv0000CtrlGpuGetPciInfoParams 180 | attachIds = defaultNv0000CtrlGpuAttachIdsParams { gpuIds = V.fromList ([x, 0xFFFFFFFF] ++ repeat 0) } :: Nv0000CtrlGpuAttachIdsParams 181 | deviceId = composeNormalDevId clientId x 1 182 | subDeviceId = composeNormalDevId clientId x 2 183 | subDeviceAlloc = 0 :: Nv2080AllocParams 184 | vgpuDeviceId = composeNormalDevId clientId x 3 185 | in do 186 | pci <- with pciInfo (\par -> ctrlRes fd clientId clientId 0x21B par >>= peek) 187 | _ <- with attachIds (\par -> ctrlRes_ fd clientId clientId 0x215 par) 188 | nvfd <- openNvDev idx 189 | gpuInfo <- with idInfoParams (\par -> ctrlRes fd clientId clientId 0x202 par >>= peek) 190 | let deviceAlloc = defaultNv0080AllocParams { deviceId = (deviceInstance gpuInfo) } :: Nv0080AllocParams 191 | _ <- with deviceAlloc (\par -> allocRes 192 | fd 193 | (defaultRmAllocRes { hRoot = clientId 194 | , hObjectParent = clientId 195 | , hObjectNew = deviceId 196 | , hClass = 0x00000080 197 | , pAllocParams = (castPtr par) 198 | } :: RmAllocRes)) 199 | _ <- with subDeviceAlloc (\par -> allocRes 200 | fd 201 | (defaultRmAllocRes { hRoot = clientId 202 | , hObjectParent = deviceId 203 | , hObjectNew = subDeviceId 204 | , hClass = 0x00002080 205 | , pAllocParams = (castPtr par) 206 | } :: RmAllocRes)) 207 | busPciInfo <- with defaultBusGetPciInfo (\par -> ctrlRes fd clientId subDeviceId 0x20801801 par >>= peek) 208 | _ <- allocRes 209 | fd 210 | (defaultRmAllocRes { hRoot = clientId 211 | , hObjectParent = subDeviceId 212 | , hObjectNew = vgpuDeviceId 213 | , hClass = 0x0000A081 214 | } :: RmAllocRes) 215 | let devid = ((devId (busPciInfo :: BusGetPciInfo)) `shift` (-16)) 216 | subid = ((subDevId (busPciInfo :: BusGetPciInfo)) `shift` (-16)) 217 | gpu = 218 | Gpu (domain (pci :: Nv0000CtrlGpuGetPciInfoParams)) 219 | (fromIntegral $ bus (pci :: Nv0000CtrlGpuGetPciInfoParams)) 220 | (fromIntegral $ slot (pci :: Nv0000CtrlGpuGetPciInfoParams)) 221 | 0 222 | 0x10DE 223 | devid 224 | 0x10DE 225 | subid 226 | x 227 | vgpudev = (NvidiaResource Nothing clientId subDeviceId vgpuDeviceId 0x0000A081 [], Nothing) 228 | subdev = (NvidiaResource Nothing clientId deviceId subDeviceId 0x00002080 [vgpudev], Nothing) 229 | res = (NvidiaResource nvfd clientId clientId deviceId 0x00000080 [subdev], Just gpu) 230 | printf 231 | "Opened GPU 0x%.4X (vendor: 0x%.4X, device: 0x%.4X, subvendor: 0x%.4X, subdevice: 0x%4X)\n" 232 | (fromIntegral x :: Integer) 233 | (0x10DE :: Integer) 234 | (fromIntegral devid :: Integer) 235 | (0x10DE :: Integer) 236 | (fromIntegral subid :: Integer) 237 | return res) probedIds' 238 | return devices 239 | 240 | {-| Opens an MDev Interface for the NVIDIA GPUs on the system. 241 | -} 242 | openNvMDevInterface :: Bool -- ^ If we use the check for version mismatch or not. 243 | -> IO (MediatedPhysicalGpu) -- ^ Returns a fully functioning mediated physical GPU root, 244 | -- that can be used to create MDevs on that particular GPU. 245 | openNvMDevInterface mismatchCheck = do 246 | ctlFd <- openNvDev 255 247 | if isJust ctlFd 248 | then do 249 | let ctlFd' = fromJust ctlFd 250 | 251 | -- Version check to verify that we are capable to run the current 252 | -- version of the mdev gpu for nvidia. 253 | versionCheck ctlFd' mismatchCheck 254 | 255 | -- Allocates a root resource. 256 | rootRes <- allocRes ctlFd' defaultRmAllocRes 257 | let 258 | clientId = hObjectNew (rootRes :: RmAllocRes) 259 | 260 | -- Gets the GPUs. 261 | gpus <- getGpus ctlFd' clientId 262 | 263 | let 264 | rootResource = 265 | (NvidiaResource Nothing clientId clientId clientId 0x000000 gpus, Nothing) 266 | 267 | return $ NvidiaMdev ctlFd' clientId [rootResource] 268 | else return NotInitialized 269 | 270 | {-| This closes the interface to the Nvidia MDev interface. 271 | -} 272 | closeNvMDevInterface :: MediatedPhysicalGpu -- ^ Mediated GPU. 273 | -> IO () -- ^ IO Side effects of closing 274 | -- the gpu. 275 | closeNvMDevInterface (NvidiaMdev { .. }) = 276 | freeNvResTree ctlFd children >> 277 | closeFd ctlFd 278 | closeNvMDevInterface _ = 279 | error "Invalid Mediated Physical GPU, function expects NvidiaMDev" 280 | -------------------------------------------------------------------------------- /src/Nvidia/RMApi.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Nvidia.RMApi 3 | Description : Base implementation for nvidia interaction for mdev-gpu. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Implementation of the RMApi as required by mdev-gpu. 11 | -} 12 | {-# LANGUAGE DataKinds #-} 13 | {-# LANGUAGE DuplicateRecordFields #-} 14 | {-# LANGUAGE RecordWildCards #-} 15 | module Nvidia.RMApi 16 | ( module Nvidia.RMApi.Types 17 | , module Nvidia.RMApi 18 | ) where 19 | 20 | import Data.Bits 21 | import Foreign (Storable(..)) 22 | import Foreign.C.Types 23 | import Foreign.Ptr 24 | import System.Posix.Types 25 | import System.Posix.IOCtl 26 | import Text.Printf 27 | 28 | -- Internal imports. 29 | import Nvidia.Errors 30 | import Nvidia.RMApi.IoctlCodes 31 | import Nvidia.RMApi.Types 32 | 33 | {-| Composes a normal object id. 34 | -} 35 | composeNormalDevId :: CUInt -- ^ Root id to use. 36 | -> CUInt -- ^ GPU id to use. 37 | -> CUInt -- ^ Handle id to use. 38 | -> CUInt -- ^ Resulting object id to connect to the system. 39 | composeNormalDevId root gpu handle = 40 | ((root + gpu) `shift` 8) .|. (handle .&. ((1 `shift` 8) - 1)) 41 | 42 | {-| Version check to ensure we are capable of running our code. 43 | -} 44 | versionCheck :: Fd -- ^ File descriptor to check. 45 | -> Bool -- ^ If we skip the version check or not. 46 | -> IO () -- ^ Runs the version check command. 47 | versionCheck fd skip = 48 | ioctl_ fd NvIOCVersionCheck (defaultNvVersionCheck skip) 49 | 50 | {-| Allocates a resource in NVIDIA Kernel module. 51 | -} 52 | allocRes :: Fd -- ^ File descriptor to allocate the resource on. 53 | -> RmAllocRes -- ^ Resource allocation type. 54 | -> IO (RmAllocRes) -- ^ Allocated resource. 55 | allocRes fd alloc = do 56 | ret <- ioctl fd NvIOCAllocRes alloc 57 | let s = status (ret :: RmAllocRes) 58 | if s /= NvOk 59 | then do 60 | printf 61 | "Failed to create resource:\n\tclient: 0x%X\n\tparent: 0x%.8X\n\tobject: 0x%.8X\n\tclass: 0x%.8X\n\tstatus: %s\n" 62 | (fromIntegral (hRoot (ret :: RmAllocRes)) :: Integer) 63 | (fromIntegral (hObjectParent (ret :: RmAllocRes)) :: Integer) 64 | (fromIntegral (hObjectNew (ret :: RmAllocRes)) :: Integer) 65 | (fromIntegral (hClass (ret :: RmAllocRes)) :: Integer) 66 | (show (status (ret :: RmAllocRes))) 67 | error "Status failed" 68 | else 69 | printf "Created resource with id: 0x%.8X\n" (fromIntegral (hObjectNew (ret :: RmAllocRes)) :: Integer) >> 70 | return ret 71 | 72 | {-| Frees a resource allocated in the NVIDIA Kernel module. 73 | -} 74 | freeRes :: Fd -- ^ File descriptor to allocate the resource on. 75 | -> RmFreeRes -- ^ Resource deallocation type. 76 | -> IO (RmFreeRes) -- ^ Deallocated resource. 77 | freeRes fd free = do 78 | ret <- ioctl fd NvIOCFreeRes free 79 | let s = status (ret :: RmFreeRes) 80 | if s /= NvOk 81 | then do 82 | putStrLn $ "Failed to free: " ++ (show ret) 83 | error "Status failed" 84 | else 85 | printf "Freed resource with id: 0x%.8X\n" (fromIntegral (hObjectOld (ret :: RmFreeRes)) :: Integer) >> 86 | return ret 87 | 88 | {-| This command sends a command to the NVIDIA Kernel module. 89 | -} 90 | ctrlRes :: (Storable d) 91 | => Fd -- ^ File descriptor to run the control event on. 92 | -> CUInt -- ^ Client Id. 93 | -> CUInt -- ^ Object Id. 94 | -> CUInt -- ^ Command to run. 95 | -> Ptr d -- ^ Pointer to control parameter. 96 | -> IO (Ptr d) -- ^ Modified pointer paramter 97 | ctrlRes fd clientId objectId command params = do 98 | paramDeref <- peek params 99 | let paramSize = fromIntegral $ sizeOf paramDeref 100 | newPtr = castPtr params 101 | cmdStruct = defaultRmControlRes { client = clientId 102 | , object = objectId 103 | , cmd = command 104 | , params = newPtr 105 | , paramsSize = paramSize 106 | } :: RmControlRes 107 | ret <- ioctl fd NvIOCControlRes cmdStruct 108 | let s = status (ret :: RmControlRes) 109 | if s /= NvOk 110 | then do 111 | putStrLn $ "Failed to control: " ++ (show ret) 112 | error "Status failed" 113 | else 114 | return params 115 | 116 | {-| This command sends a command to the NVIDIA Kernel module, but does not 117 | respond with a return value. 118 | -} 119 | ctrlRes_ :: (Storable d) => 120 | Fd -- ^ File descriptor to run the control event on. 121 | -> CUInt -- ^ Client Id. 122 | -> CUInt -- ^ Object Id. 123 | -> CUInt -- ^ Command to run. 124 | -> Ptr d -- ^ Pointer to control parameter. 125 | -> IO (Ptr d) -- ^ Modified pointer paramter 126 | ctrlRes_ fd clientId objectId command params = do 127 | paramDeref <- peek params 128 | let paramSize = fromIntegral $ sizeOf paramDeref 129 | newPtr = castPtr params 130 | cmdStruct = defaultRmControlRes { client = clientId 131 | , object = objectId 132 | , cmd = command 133 | , params = newPtr 134 | , paramsSize = paramSize 135 | } :: RmControlRes 136 | ret <- ioctl fd NvIOCControlRes cmdStruct 137 | let s = status (ret :: RmControlRes) 138 | if s /= NvOk 139 | then do 140 | putStrLn $ "Failed to control: " ++ (show ret) 141 | error "Status failed" 142 | else 143 | return params 144 | 145 | {-| This command sends a command with no parameters to the NVIDIA Kernel module. 146 | -} 147 | ctrlRes' :: Fd -- ^ File descriptor to run the control event on. 148 | -> CUInt -- ^ Client Id. 149 | -> CUInt -- ^ Object Id. 150 | -> CUInt -- ^ Command to run. 151 | -> IO () -- ^ Command only executed for side effects 152 | ctrlRes' fd clientId objectId command = do 153 | let cmdStruct = defaultRmControlRes { client = clientId 154 | , object = objectId 155 | , cmd = command 156 | } :: RmControlRes 157 | ret <- ioctl fd NvIOCControlRes cmdStruct 158 | let s = status (ret :: RmControlRes) 159 | if s /= NvOk 160 | then do 161 | putStrLn $ "Failed to control: " ++ (show ret) 162 | error "Status failed" 163 | else 164 | return () 165 | -------------------------------------------------------------------------------- /src/Nvidia/RMApi/IoctlCodes.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Nvidia.RMApi.IoctlCodes 3 | Description : Different possible IOCtl commands that we need to allow. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Different possible IOCtls that can be called at different times. 11 | -} 12 | {-# LANGUAGE MultiParamTypeClasses #-} 13 | module Nvidia.RMApi.IoctlCodes where 14 | 15 | import Foreign (Storable(..)) 16 | import System.Posix.IOCtl 17 | 18 | -- Internal imports. 19 | import Nvidia.RMApi.Types 20 | 21 | {-| This type is a holder to ensure we can perform a version check over 22 | ioctl. 23 | -} 24 | data NvIOCVersionCheck = NvIOCVersionCheck 25 | 26 | {-| This type is a holder to ensure we can allocate devices. 27 | -} 28 | data NvIOCAllocRes = NvIOCAllocRes 29 | 30 | {-| This type is a holder to ensure we can free resources. 31 | -} 32 | data NvIOCFreeRes = NvIOCFreeRes 33 | 34 | {-| Control command for a resource. 35 | -} 36 | data NvIOCControlRes = NvIOCControlRes 37 | 38 | instance IOControl NvIOCVersionCheck NvVersionCheck where 39 | ioctlReq _ = 0xC04846D2 40 | 41 | instance IOControl NvIOCAllocRes RmAllocRes where 42 | ioctlReq _ = 0xC020462B 43 | 44 | instance IOControl NvIOCFreeRes RmFreeRes where 45 | ioctlReq _ = 0xC0104629 46 | 47 | instance IOControl NvIOCControlRes RmControlRes where 48 | ioctlReq _ = 0xC020462A 49 | -------------------------------------------------------------------------------- /src/Nvidia/RMApi/Types.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Nvidia.RMApi.Types 3 | Description : Types the are available to the RM API. 4 | Copyright : (c) 2022 2666680 Ontario Inc. O\A Arc Compute 5 | License : GNU GPL v.2 6 | Maintainer : michael@arccompute.io 7 | Stability : experimental 8 | Portability : POSIX 9 | 10 | Here is where we store types that have to be exposed to the 11 | RM Api. 12 | -} 13 | {-# LANGUAGE DataKinds #-} 14 | {-# LANGUAGE DuplicateRecordFields #-} 15 | {-# LANGUAGE RecordWildCards #-} 16 | module Nvidia.RMApi.Types where 17 | 18 | import Control.Applicative ((<$>), (<*>)) 19 | import Data.Vector.Fixed.Storable (Vec(..)) 20 | import GHC.Generics 21 | import Foreign (Storable(..)) 22 | import Foreign.Ptr 23 | import Foreign.C.Types 24 | import Foreign.C.String 25 | 26 | import qualified Data.Vector.Fixed as V 27 | 28 | -- Internal imports. 29 | import Common.Types (VoidPtr) 30 | import Nvidia.Errors 31 | 32 | {-| NVIDIA requires a version check to ensure that the binary offsets are correct between the 33 | userland program and the kernel module. 34 | -} 35 | data NvVersionCheck = NvVersionCheck 36 | { cmd :: CUInt -- ^ Command override (Allows you to hide the version) 37 | , reply :: CUInt -- ^ Reply if the driver is correct or not. 38 | , driverVersion :: Vec 64 CChar -- ^ Where we store the driver string. 39 | } deriving (Show) 40 | 41 | instance Storable NvVersionCheck where 42 | sizeOf _ = 72 43 | alignment _ = 8 44 | peek ptr = NvVersionCheck 45 | <$> peekByteOff ptr 0 46 | <*> peekByteOff ptr 4 47 | <*> peekByteOff ptr 8 48 | poke ptr (NvVersionCheck { .. }) = do 49 | pokeByteOff ptr 0 cmd 50 | pokeByteOff ptr 4 reply 51 | pokeByteOff ptr 8 driverVersion 52 | 53 | {-| Provides a default NVIDIA version to be sent to the RM Core. 54 | 55 | NOTE: This is currently hardcoded to the 510.47.03 driver version as it is our current testing version. 56 | -} 57 | defaultNvVersionCheck :: Bool -- ^ If we skip the check or not. 58 | -> NvVersionCheck -- ^ Valid version for version check of the GPU. 59 | defaultNvVersionCheck skip = 60 | let command = if skip then 0x32 else 0x00 61 | in NvVersionCheck command 0 $ 62 | V.fromList ([0x35, 0x31, 0x30, 0x2E, 0x34, 0x37, 0x2E, 0x30, 0x33] ++ repeat 0) 63 | 64 | {-| Type from the NVIDIA Kernel module for allocating a resource in the kernel module. 65 | -} 66 | data RmAllocRes = RmAllocRes 67 | { hRoot :: CUInt 68 | , hObjectParent :: CUInt 69 | , hObjectNew :: CUInt 70 | , hClass :: CUInt 71 | , pAllocParams :: Ptr VoidPtr 72 | , status :: NvErrorStatus 73 | } deriving (Show) 74 | 75 | instance Storable RmAllocRes where 76 | sizeOf _ = 32 77 | alignment _ = 8 78 | peek ptr = RmAllocRes 79 | <$> peekByteOff ptr 0 80 | <*> peekByteOff ptr 4 81 | <*> peekByteOff ptr 8 82 | <*> peekByteOff ptr 12 83 | <*> peekByteOff ptr 16 84 | <*> peekByteOff ptr 24 85 | poke ptr (RmAllocRes { .. }) = do 86 | pokeByteOff ptr 0 hRoot 87 | pokeByteOff ptr 4 hObjectParent 88 | pokeByteOff ptr 8 hObjectNew 89 | pokeByteOff ptr 12 hClass 90 | pokeByteOff ptr 16 pAllocParams 91 | pokeByteOff ptr 24 status 92 | 93 | {-| Provides a default RmAllocRes structure. 94 | -} 95 | defaultRmAllocRes :: RmAllocRes 96 | defaultRmAllocRes = 97 | RmAllocRes 0 0 0 0 nullPtr NvOk 98 | 99 | {-| Structure to free a device. 100 | -} 101 | data RmFreeRes = RmFreeRes 102 | { hRoot :: CUInt 103 | , hObjectParent :: CUInt 104 | , hObjectOld :: CUInt 105 | , status :: NvErrorStatus 106 | } deriving (Show) 107 | 108 | instance Storable RmFreeRes where 109 | sizeOf _ = 16 110 | alignment _ = 4 111 | peek ptr = RmFreeRes 112 | <$> peekByteOff ptr 0 113 | <*> peekByteOff ptr 4 114 | <*> peekByteOff ptr 8 115 | <*> peekByteOff ptr 12 116 | poke ptr (RmFreeRes { .. }) = do 117 | pokeByteOff ptr 0 hRoot 118 | pokeByteOff ptr 4 hObjectParent 119 | pokeByteOff ptr 8 hObjectOld 120 | pokeByteOff ptr 12 status 121 | 122 | {-| Provides a default RmFreeRes structure. 123 | -} 124 | defaultRmFreeRes :: RmFreeRes 125 | defaultRmFreeRes = RmFreeRes 0 0 0 NvOk 126 | 127 | {-| Object sent over an RMControl to attach a set of available GPU ids. 128 | -} 129 | data Nv0000CtrlGpuAttachIdsParams = Nv0000CtrlGpuAttachIdsParams 130 | { gpuIds :: Vec 32 CUInt 131 | , failedGpuId :: CUInt 132 | } deriving (Show) 133 | 134 | instance Storable Nv0000CtrlGpuAttachIdsParams where 135 | sizeOf _ = 132 136 | alignment _ = 4 137 | peek ptr = Nv0000CtrlGpuAttachIdsParams 138 | <$> peekByteOff ptr 0 139 | <*> peekByteOff ptr 128 140 | poke ptr (Nv0000CtrlGpuAttachIdsParams { .. }) = do 141 | pokeByteOff ptr 0 gpuIds 142 | pokeByteOff ptr 128 failedGpuId 143 | 144 | {-| Default object for attaching GPU Ids. 145 | -} 146 | defaultNv0000CtrlGpuAttachIdsParams :: Nv0000CtrlGpuAttachIdsParams 147 | defaultNv0000CtrlGpuAttachIdsParams = 148 | Nv0000CtrlGpuAttachIdsParams (V.replicate 0) 0 149 | 150 | {-| Object to get a list of attached GPU Ids. 151 | -} 152 | data Nv0000CtrlGpuGetAttachIdsParams = Nv0000CtrlGpuGetAttachIdsParams 153 | { gpuIds :: Vec 32 CUInt 154 | } deriving (Show) 155 | 156 | instance Storable Nv0000CtrlGpuGetAttachIdsParams where 157 | sizeOf _ = 128 158 | alignment _ = 4 159 | peek ptr = Nv0000CtrlGpuGetAttachIdsParams 160 | <$> peekByteOff ptr 0 161 | poke ptr (Nv0000CtrlGpuGetAttachIdsParams { .. }) = 162 | pokeByteOff ptr 0 gpuIds 163 | 164 | {-| Default object to get the GPU Ids. 165 | -} 166 | defaultNv0000CtrlGpuGetAttachIdsParams :: Nv0000CtrlGpuGetAttachIdsParams 167 | defaultNv0000CtrlGpuGetAttachIdsParams = 168 | Nv0000CtrlGpuGetAttachIdsParams (V.replicate 0) 169 | 170 | {-| Object to detach the GPU Ids. 171 | -} 172 | data Nv0000CtrlGpuDetachIdsParams = Nv0000CtrlGpuDetachIdsParams 173 | { gpuIds :: Vec 32 CUInt 174 | } deriving (Show) 175 | 176 | instance Storable Nv0000CtrlGpuDetachIdsParams where 177 | sizeOf _ = 128 178 | alignment _ = 4 179 | peek ptr = Nv0000CtrlGpuDetachIdsParams 180 | <$> peekByteOff ptr 0 181 | poke ptr (Nv0000CtrlGpuDetachIdsParams { .. }) = 182 | pokeByteOff ptr 0 gpuIds 183 | 184 | {-| Default object to detach all the GPU Ids. 185 | -} 186 | defaultNv0000CtrlGpuDetachIdsParams :: Nv0000CtrlGpuDetachIdsParams 187 | defaultNv0000CtrlGpuDetachIdsParams = 188 | Nv0000CtrlGpuDetachIdsParams (V.replicate 0) 189 | 190 | {-| Object to send a control message to the NVIDIA kernel module. 191 | -} 192 | data RmControlRes = RmControlRes 193 | { client :: CUInt 194 | , object :: CUInt 195 | , cmd :: CUInt 196 | , flags :: CUInt 197 | , params :: Ptr VoidPtr 198 | , paramsSize :: CUInt 199 | , status :: NvErrorStatus 200 | } deriving (Show) 201 | 202 | instance Storable RmControlRes where 203 | sizeOf _ = 32 204 | alignment _ = 8 205 | peek ptr = RmControlRes 206 | <$> peekByteOff ptr 0 207 | <*> peekByteOff ptr 4 208 | <*> peekByteOff ptr 8 209 | <*> peekByteOff ptr 12 210 | <*> peekByteOff ptr 16 211 | <*> peekByteOff ptr 24 212 | <*> peekByteOff ptr 28 213 | poke ptr (RmControlRes { .. }) = do 214 | pokeByteOff ptr 0 client 215 | pokeByteOff ptr 4 object 216 | pokeByteOff ptr 8 cmd 217 | pokeByteOff ptr 12 flags 218 | pokeByteOff ptr 16 params 219 | pokeByteOff ptr 24 paramsSize 220 | pokeByteOff ptr 28 status 221 | 222 | {-| Command to send an RM Control call. 223 | -} 224 | defaultRmControlRes :: RmControlRes 225 | defaultRmControlRes = 226 | RmControlRes 0 0 0 0 nullPtr 0 NvOk 227 | 228 | {-| Helper alias for handling get id info. 229 | -} 230 | type SzName = Vec 128 CChar 231 | 232 | {-| Gets the id info from the given GPU. 233 | -} 234 | data Nv0000CtrlGpuGetIdInfoParams = Nv0000CtrlGpuGetIdInfoParams 235 | { gpuId :: CUInt 236 | , gpuFlags :: CUInt 237 | , deviceInstance :: CUInt 238 | , subDeviceInstance :: CUInt 239 | , szName :: Ptr SzName 240 | , sliStatus :: CUInt 241 | , boardId :: CUInt 242 | , gpuInstance :: CUInt 243 | , numaId :: CUInt 244 | } deriving (Show) 245 | 246 | instance Storable Nv0000CtrlGpuGetIdInfoParams where 247 | sizeOf _ = 40 248 | alignment _ = 8 249 | peek ptr = Nv0000CtrlGpuGetIdInfoParams 250 | <$> peekByteOff ptr 0 251 | <*> peekByteOff ptr 4 252 | <*> peekByteOff ptr 8 253 | <*> peekByteOff ptr 12 254 | <*> peekByteOff ptr 16 255 | <*> peekByteOff ptr 24 256 | <*> peekByteOff ptr 28 257 | <*> peekByteOff ptr 32 258 | <*> peekByteOff ptr 36 259 | poke ptr (Nv0000CtrlGpuGetIdInfoParams { .. }) = do 260 | pokeByteOff ptr 0 gpuId 261 | pokeByteOff ptr 4 gpuFlags 262 | pokeByteOff ptr 8 deviceInstance 263 | pokeByteOff ptr 12 subDeviceInstance 264 | pokeByteOff ptr 16 szName 265 | pokeByteOff ptr 24 sliStatus 266 | pokeByteOff ptr 28 boardId 267 | pokeByteOff ptr 32 gpuInstance 268 | pokeByteOff ptr 36 numaId 269 | 270 | {-| Default values for getting the id info from the system. 271 | -} 272 | defaultNv0000CtrlGpuGetIdInfoParams :: Nv0000CtrlGpuGetIdInfoParams 273 | defaultNv0000CtrlGpuGetIdInfoParams = 274 | Nv0000CtrlGpuGetIdInfoParams 0 0 0 0 nullPtr 0 0 0 0 275 | 276 | {-| Gets the list of probed gpu ids. 277 | -} 278 | data Nv0000CtrlGpuGetProbedIdsParams = Nv0000CtrlGpuGetProbedIdsParams 279 | { gpuIds :: Vec 32 CUInt 280 | , excludedGpuIds :: Vec 32 CUInt 281 | } deriving (Show) 282 | 283 | instance Storable Nv0000CtrlGpuGetProbedIdsParams where 284 | sizeOf _ = 256 285 | alignment _ = 8 286 | peek ptr = Nv0000CtrlGpuGetProbedIdsParams 287 | <$> peekByteOff ptr 0 288 | <*> peekByteOff ptr 128 289 | poke ptr (Nv0000CtrlGpuGetProbedIdsParams { .. }) = do 290 | pokeByteOff ptr 0 gpuIds 291 | pokeByteOff ptr 128 excludedGpuIds 292 | 293 | {-| Default object to get the probed ids. 294 | -} 295 | defaultNv0000CtrlGpuGetProbedIdsParams :: Nv0000CtrlGpuGetProbedIdsParams 296 | defaultNv0000CtrlGpuGetProbedIdsParams = 297 | Nv0000CtrlGpuGetProbedIdsParams (V.replicate 0) (V.replicate 0) 298 | 299 | {-| Object to get base PCI info. 300 | -} 301 | data Nv0000CtrlGpuGetPciInfoParams = Nv0000CtrlGpuGetPciInfoParams 302 | { gpuId :: CUInt 303 | , domain :: CUInt 304 | , bus :: CUShort 305 | , slot :: CUShort 306 | } deriving (Show) 307 | 308 | instance Storable Nv0000CtrlGpuGetPciInfoParams where 309 | sizeOf _ = 12 310 | alignment _ = 4 311 | peek ptr = Nv0000CtrlGpuGetPciInfoParams 312 | <$> peekByteOff ptr 0 313 | <*> peekByteOff ptr 4 314 | <*> peekByteOff ptr 8 315 | <*> peekByteOff ptr 10 316 | poke ptr (Nv0000CtrlGpuGetPciInfoParams { .. }) = do 317 | pokeByteOff ptr 0 gpuId 318 | pokeByteOff ptr 4 domain 319 | pokeByteOff ptr 8 bus 320 | pokeByteOff ptr 10 slot 321 | 322 | {-| Default object to get Base PCI info. 323 | -} 324 | defaultNv0000CtrlGpuGetPciInfoParams :: Nv0000CtrlGpuGetPciInfoParams 325 | defaultNv0000CtrlGpuGetPciInfoParams = 326 | Nv0000CtrlGpuGetPciInfoParams 0 0 0 0 327 | 328 | {-| Object to allocate a Nv0080 object. 329 | -} 330 | data Nv0080AllocParams = Nv0080AllocParams 331 | { deviceId :: CUInt 332 | , hClientShare :: CUInt 333 | , hTargetClient :: CUInt 334 | , hTargetDevice :: CUInt 335 | , flags :: CUInt 336 | , vaSpaceSize :: CULong 337 | , vaStartInternal :: CULong 338 | , vaLimitInternal :: CULong 339 | , vaMode :: CUInt 340 | } deriving (Show) 341 | 342 | instance Storable Nv0080AllocParams where 343 | sizeOf _ = 64 344 | alignment _ = 8 345 | peek ptr = Nv0080AllocParams 346 | <$> peekByteOff ptr 0 347 | <*> peekByteOff ptr 4 348 | <*> peekByteOff ptr 8 349 | <*> peekByteOff ptr 12 350 | <*> peekByteOff ptr 16 351 | <*> peekByteOff ptr 24 352 | <*> peekByteOff ptr 32 353 | <*> peekByteOff ptr 40 354 | <*> peekByteOff ptr 48 355 | poke ptr (Nv0080AllocParams { .. }) = do 356 | pokeByteOff ptr 0 deviceId 357 | pokeByteOff ptr 4 hClientShare 358 | pokeByteOff ptr 8 hTargetClient 359 | pokeByteOff ptr 12 hTargetDevice 360 | pokeByteOff ptr 16 flags 361 | pokeByteOff ptr 24 vaSpaceSize 362 | pokeByteOff ptr 32 vaStartInternal 363 | pokeByteOff ptr 40 vaLimitInternal 364 | pokeByteOff ptr 48 vaMode 365 | 366 | {-| Default object to allocate a Nv0080 object. 367 | -} 368 | defaultNv0080AllocParams :: Nv0080AllocParams 369 | defaultNv0080AllocParams = 370 | Nv0080AllocParams 0 0 0 0 0 0 0 0 0 371 | 372 | {-| Type alias to help create a sub device allocation. 373 | -} 374 | type Nv2080AllocParams = CUInt 375 | 376 | {-| Data to get PCI information. 377 | -} 378 | data BusGetPciInfo = BusGetPciInfo 379 | { devId :: CUInt 380 | , subDevId :: CUInt 381 | , revisionId :: CUInt 382 | , extId :: CUInt 383 | } deriving (Show) 384 | 385 | instance Storable BusGetPciInfo where 386 | sizeOf _ = 16 387 | alignment _ = 8 388 | peek ptr = BusGetPciInfo 389 | <$> peekByteOff ptr 0 390 | <*> peekByteOff ptr 4 391 | <*> peekByteOff ptr 8 392 | <*> peekByteOff ptr 12 393 | poke ptr (BusGetPciInfo { .. }) = do 394 | pokeByteOff ptr 0 devId 395 | pokeByteOff ptr 4 subDevId 396 | pokeByteOff ptr 8 revisionId 397 | pokeByteOff ptr 12 extId 398 | 399 | {-| Gets PCI Bus information. 400 | -} 401 | defaultBusGetPciInfo :: BusGetPciInfo 402 | defaultBusGetPciInfo = BusGetPciInfo 0 0 0 0 403 | 404 | {-| vGPU configuration object. 405 | -} 406 | data RmVgpuConfig = RmVgpuConfig 407 | { discardPrevious :: CUInt 408 | , vgpuTypeNumber :: CUInt 409 | , vgpuName :: Vec 32 CChar 410 | , vgpuClass :: Vec 32 CChar 411 | , sign :: Vec 128 CChar 412 | , pact :: Vec 128 CChar 413 | , maxInstances :: CUInt 414 | , numHeads :: CUInt 415 | , maxResX :: CUInt 416 | , maxResY :: CUInt 417 | , maxPixel :: CUInt 418 | , frlConfig :: CUInt 419 | , cuda :: CUInt 420 | , eccSupported :: CUInt 421 | , gpuInstanceSize :: CUInt 422 | , multiMdev :: CUInt 423 | , encCap :: CULong 424 | , vdevId :: CULong 425 | , pdevId :: CULong 426 | , fbLen :: CULong 427 | , mappableVideoSize :: CULong 428 | , fbRes :: CULong 429 | , bar1Len :: CULong 430 | , frlEnable :: CUInt 431 | , vgpuExtraParams :: Vec 1027 CUInt 432 | } 433 | 434 | instance Storable RmVgpuConfig where 435 | sizeOf _ = 0x11C0 436 | alignment _ = 8 437 | peek ptr = RmVgpuConfig 438 | <$> peekByteOff ptr 0 -- discardPrevious 4 + (4 padding) 439 | <*> peekByteOff ptr 8 -- vgpuTypeNumber 4 440 | <*> peekByteOff ptr 12 -- vgpuName 32 441 | <*> peekByteOff ptr 44 -- vgpuClass 32 442 | <*> peekByteOff ptr 76 -- sign 128 443 | <*> peekByteOff ptr 204 -- pact 128 + (4 padding) 444 | <*> peekByteOff ptr 336 -- maxInstances 4 445 | <*> peekByteOff ptr 340 -- numHeads 4 446 | <*> peekByteOff ptr 344 -- maxResX 4 447 | <*> peekByteOff ptr 348 -- maxResY 4 448 | <*> peekByteOff ptr 352 -- maxPixel 4 449 | <*> peekByteOff ptr 356 -- frlConfig 4 450 | <*> peekByteOff ptr 360 -- cuda 4 451 | <*> peekByteOff ptr 364 -- eccSupported 4 452 | <*> peekByteOff ptr 368 -- gpuInstanceSize 4 453 | <*> peekByteOff ptr 372 -- multiMdev 4 454 | <*> peekByteOff ptr 376 -- encCap 8 455 | <*> peekByteOff ptr 384 -- vdevId 8 456 | <*> peekByteOff ptr 392 -- pdevId 8 457 | <*> peekByteOff ptr 400 -- fbLen 8 458 | <*> peekByteOff ptr 408 -- mappableVideoSize 8 459 | <*> peekByteOff ptr 416 -- fbRes 8 460 | <*> peekByteOff ptr 424 -- bar1Len 8 461 | <*> peekByteOff ptr 432 -- frlEnable 4 462 | <*> peekByteOff ptr 436 -- vgpuExtraParams 4108 463 | poke ptr (RmVgpuConfig { .. }) = do 464 | pokeByteOff ptr 0 discardPrevious 465 | pokeByteOff ptr 8 vgpuTypeNumber 466 | pokeByteOff ptr 12 vgpuName 467 | pokeByteOff ptr 44 vgpuClass 468 | pokeByteOff ptr 76 sign 469 | pokeByteOff ptr 204 pact 470 | pokeByteOff ptr 336 maxInstances 471 | pokeByteOff ptr 340 numHeads 472 | pokeByteOff ptr 344 maxResX 473 | pokeByteOff ptr 348 maxResY 474 | pokeByteOff ptr 352 maxPixel 475 | pokeByteOff ptr 356 frlConfig 476 | pokeByteOff ptr 360 cuda 477 | pokeByteOff ptr 364 eccSupported 478 | pokeByteOff ptr 368 gpuInstanceSize 479 | pokeByteOff ptr 372 multiMdev 480 | pokeByteOff ptr 376 encCap 481 | pokeByteOff ptr 384 vdevId 482 | pokeByteOff ptr 392 pdevId 483 | pokeByteOff ptr 400 fbLen 484 | pokeByteOff ptr 408 mappableVideoSize 485 | pokeByteOff ptr 416 fbRes 486 | pokeByteOff ptr 424 bar1Len 487 | pokeByteOff ptr 432 frlEnable 488 | pokeByteOff ptr 436 vgpuExtraParams 489 | 490 | {-| Default vgpu configuration object. 491 | -} 492 | defaultRmVgpuConfig :: RmVgpuConfig 493 | defaultRmVgpuConfig = 494 | RmVgpuConfig 495 | 0 496 | 0 497 | (V.replicate 0) 498 | (V.fromList ((map castCharToCChar "Compute") ++ repeat 0)) 499 | (V.fromList [ 0x47, 0xab, 0x8d, 0x39, 0xb3, 0xaf, 0xd0, 0x2c, 0x79, 0x6f, 0xd0, 0xc7, 0x7a, 0x78, 0x84, 0x68 500 | , 0xf0, 0x9b, 0x69, 0xe8, 0xb6, 0xc2, 0xc5, 0x05, 0x59, 0x97, 0xf2, 0x0f, 0x77, 0x3a, 0x94, 0x91 501 | , 0x92, 0x56, 0x2d, 0xf0, 0x4f, 0xae, 0xa6, 0x4d, 0xcc, 0x51, 0x32, 0x17, 0xe5, 0xda, 0xf0, 0x94 502 | , 0x42, 0x93, 0x51, 0x05, 0x49, 0xe9, 0x61, 0xfd, 0x22, 0x24, 0x6c, 0x8f, 0x88, 0xe3, 0x16, 0x63 503 | , 0x91, 0x04, 0x20, 0x6a, 0xea, 0x27, 0xc4, 0xe7, 0x11, 0xfc, 0x88, 0x24, 0xb9, 0xaa, 0x1b, 0x85 504 | , 0xc5, 0x1a, 0x48, 0x7f, 0x99, 0xf4, 0x8e, 0xda, 0x55, 0x2b, 0x4a, 0xfe, 0x48, 0x79, 0x75, 0x78 505 | , 0x48, 0x16, 0x02, 0x0f, 0x22, 0xc2, 0x9d, 0x20, 0xfa, 0xbb, 0x21, 0x39, 0x56, 0x78, 0xd8, 0x80 506 | , 0x96, 0x5c, 0x5c, 0xe4, 0x7c, 0xad, 0x87, 0x24, 0x20, 0x70, 0xad, 0x63, 0x84, 0x96, 0x92, 0x3f 507 | ]) 508 | (V.fromList ((map castCharToCChar "NVIDIA-vComputeServer,9.0;Quadro-Virtual-DWS,5.0") ++ repeat 0)) 509 | 0 510 | 0 511 | 0 512 | 0 513 | 0 514 | 0 515 | 1 516 | 0 517 | 0 518 | 0 519 | 0 520 | 0 521 | 0 522 | 0 523 | 0 524 | 0 525 | 0 526 | 0 527 | (V.replicate 0) 528 | -------------------------------------------------------------------------------- /stack.yaml: -------------------------------------------------------------------------------- 1 | # This file was automatically generated by 'stack init' 2 | # 3 | # Some commonly used options have been documented as comments in this file. 4 | # For advanced use and comprehensive documentation of the format, please see: 5 | # https://docs.haskellstack.org/en/stable/yaml_configuration/ 6 | 7 | # Resolver to choose a 'specific' stackage snapshot or a compiler version. 8 | # A snapshot resolver dictates the compiler version and the set of packages 9 | # to be used for project dependencies. For example: 10 | # 11 | # resolver: lts-3.5 12 | # resolver: nightly-2015-09-21 13 | # resolver: ghc-7.10.2 14 | # 15 | # The location of a snapshot can be provided as a file or url. Stack assumes 16 | # a snapshot provided as a file might change, whereas a url resource does not. 17 | # 18 | # resolver: ./custom-snapshot.yaml 19 | # resolver: https://example.com/snapshots/2018-01-01.yaml 20 | resolver: 21 | url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/19/12.yaml 22 | 23 | # User packages to be built. 24 | # Various formats can be used as shown in the example below. 25 | # 26 | # packages: 27 | # - some-directory 28 | # - https://example.com/foo/bar/baz-0.0.2.tar.gz 29 | # subdirs: 30 | # - auto-update 31 | # - wai 32 | packages: 33 | - . 34 | # Dependency packages to be pulled from upstream that are not in the resolver. 35 | # These entries can reference officially published versions as well as 36 | # forks / in-progress versions pinned to a git hash. For example: 37 | # 38 | # extra-deps: 39 | # - acme-missiles-0.3 40 | # - git: https://github.com/commercialhaskell/stack.git 41 | # commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a 42 | # 43 | extra-deps: 44 | - ioctl-0.0.1@sha256:5861ee9cdf3a2159fce85e3f9213b99bbbb0132e0ff85af7fc23df5215bd2096,982 45 | - network-2.8.0.1@sha256:0f165dffa752d8cde30c2bde86f80609c4f1dc5eeb3182d593041f97839c5b3b,3011 46 | 47 | # Override default flag values for local packages and extra-deps 48 | # flags: {} 49 | 50 | # Extra package databases containing global packages 51 | # extra-package-dbs: [] 52 | 53 | # Control whether we use the GHC we find on the path 54 | # system-ghc: true 55 | # 56 | # Require a specific version of stack, using version ranges 57 | # require-stack-version: -any # Default 58 | # require-stack-version: ">=2.7" 59 | # 60 | # Override the architecture used by stack, especially useful on Windows 61 | # arch: i386 62 | # arch: x86_64 63 | # 64 | # Extra directories used by stack for building 65 | # extra-include-dirs: [/path/to/dir] 66 | # extra-lib-dirs: [/path/to/dir] 67 | # 68 | # Allow a newer minor version of GHC than the snapshot specifies 69 | # compiler-check: newer-minor 70 | -------------------------------------------------------------------------------- /stack.yaml.lock: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by Stack. 2 | # You should not edit this file by hand. 3 | # For more information, please see the documentation at: 4 | # https://docs.haskellstack.org/en/stable/lock_files 5 | 6 | packages: 7 | - completed: 8 | hackage: ioctl-0.0.1@sha256:5861ee9cdf3a2159fce85e3f9213b99bbbb0132e0ff85af7fc23df5215bd2096,982 9 | pantry-tree: 10 | size: 355 11 | sha256: 42e51444f4ff5035d138e3ad51ded3b161681a0423185153e6a5c622cb116964 12 | original: 13 | hackage: ioctl-0.0.1@sha256:5861ee9cdf3a2159fce85e3f9213b99bbbb0132e0ff85af7fc23df5215bd2096,982 14 | - completed: 15 | hackage: network-2.8.0.1@sha256:0f165dffa752d8cde30c2bde86f80609c4f1dc5eeb3182d593041f97839c5b3b,3011 16 | pantry-tree: 17 | size: 2488 18 | sha256: 2dbdda2225969dcd452d983a2ac7e52341e5fc1df76103342c9fac08fc686931 19 | original: 20 | hackage: network-2.8.0.1@sha256:0f165dffa752d8cde30c2bde86f80609c4f1dc5eeb3182d593041f97839c5b3b,3011 21 | snapshots: 22 | - completed: 23 | size: 619158 24 | url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/19/12.yaml 25 | sha256: d4d7a4ac80485f3c21d0c6bfe5e7a21c3f71ca4fdfea022f98574047ebb58ebb 26 | original: 27 | url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/19/12.yaml 28 | --------------------------------------------------------------------------------