├── .github └── workflows │ └── tests.yml ├── .gitignore ├── COPYING ├── Makefile ├── README ├── mods.h ├── mods_acpi.c ├── mods_arm_ffa.c ├── mods_bpmpipc.c ├── mods_config.h ├── mods_debugfs.c ├── mods_internal.h ├── mods_irq.c ├── mods_krnl.c ├── mods_mem.c ├── mods_pci.c └── mods_ppc64.c /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | basic: 10 | runs-on: ubuntu-latest 11 | timeout-minutes: 1 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Compile 15 | run: make -j $(grep -c ^processor /proc/cpuinfo) 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Vim swap files 2 | .*.swp 3 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | 2 | GNU General Public License, version 2 (this license) applies to all files 3 | in the NVIDIA MODS kernel driver package. 4 | 5 | The NVIDIA MODS kernel driver package contains the following files: 6 | COPYING 7 | Makefile 8 | mods_acpi.c 9 | mods_arm_ffa.c 10 | mods_bpmpipc.c 11 | mods_config.h 12 | mods_debugfs.c 13 | mods.h 14 | mods_internal.h 15 | mods_irq.c 16 | mods_krnl.c 17 | mods_mem.c 18 | mods_pci.c 19 | mods_ppc64.c 20 | README 21 | 22 | ----------------------------------------------------------------------------- 23 | 24 | GNU GENERAL PUBLIC LICENSE 25 | Version 2, June 1991 26 | 27 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 28 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 29 | Everyone is permitted to copy and distribute verbatim copies 30 | of this license document, but changing it is not allowed. 31 | 32 | Preamble 33 | 34 | The licenses for most software are designed to take away your 35 | freedom to share and change it. By contrast, the GNU General Public 36 | License is intended to guarantee your freedom to share and change free 37 | software--to make sure the software is free for all its users. This 38 | General Public License applies to most of the Free Software 39 | Foundation's software and to any other program whose authors commit to 40 | using it. (Some other Free Software Foundation software is covered by 41 | the GNU Lesser General Public License instead.) You can apply it to 42 | your programs, too. 43 | 44 | When we speak of free software, we are referring to freedom, not 45 | price. Our General Public Licenses are designed to make sure that you 46 | have the freedom to distribute copies of free software (and charge for 47 | this service if you wish), that you receive source code or can get it 48 | if you want it, that you can change the software or use pieces of it 49 | in new free programs; and that you know you can do these things. 50 | 51 | To protect your rights, we need to make restrictions that forbid 52 | anyone to deny you these rights or to ask you to surrender the rights. 53 | These restrictions translate to certain responsibilities for you if you 54 | distribute copies of the software, or if you modify it. 55 | 56 | For example, if you distribute copies of such a program, whether 57 | gratis or for a fee, you must give the recipients all the rights that 58 | you have. You must make sure that they, too, receive or can get the 59 | source code. And you must show them these terms so they know their 60 | rights. 61 | 62 | We protect your rights with two steps: (1) copyright the software, and 63 | (2) offer you this license which gives you legal permission to copy, 64 | distribute and/or modify the software. 65 | 66 | Also, for each author's protection and ours, we want to make certain 67 | that everyone understands that there is no warranty for this free 68 | software. If the software is modified by someone else and passed on, we 69 | want its recipients to know that what they have is not the original, so 70 | that any problems introduced by others will not reflect on the original 71 | authors' reputations. 72 | 73 | Finally, any free program is threatened constantly by software 74 | patents. We wish to avoid the danger that redistributors of a free 75 | program will individually obtain patent licenses, in effect making the 76 | program proprietary. To prevent this, we have made it clear that any 77 | patent must be licensed for everyone's free use or not licensed at all. 78 | 79 | The precise terms and conditions for copying, distribution and 80 | modification follow. 81 | 82 | GNU GENERAL PUBLIC LICENSE 83 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 84 | 85 | 0. This License applies to any program or other work which contains 86 | a notice placed by the copyright holder saying it may be distributed 87 | under the terms of this General Public License. The "Program", below, 88 | refers to any such program or work, and a "work based on the Program" 89 | means either the Program or any derivative work under copyright law: 90 | that is to say, a work containing the Program or a portion of it, 91 | either verbatim or with modifications and/or translated into another 92 | language. (Hereinafter, translation is included without limitation in 93 | the term "modification".) Each licensee is addressed as "you". 94 | 95 | Activities other than copying, distribution and modification are not 96 | covered by this License; they are outside its scope. The act of 97 | running the Program is not restricted, and the output from the Program 98 | is covered only if its contents constitute a work based on the 99 | Program (independent of having been made by running the Program). 100 | Whether that is true depends on what the Program does. 101 | 102 | 1. You may copy and distribute verbatim copies of the Program's 103 | source code as you receive it, in any medium, provided that you 104 | conspicuously and appropriately publish on each copy an appropriate 105 | copyright notice and disclaimer of warranty; keep intact all the 106 | notices that refer to this License and to the absence of any warranty; 107 | and give any other recipients of the Program a copy of this License 108 | along with the Program. 109 | 110 | You may charge a fee for the physical act of transferring a copy, and 111 | you may at your option offer warranty protection in exchange for a fee. 112 | 113 | 2. You may modify your copy or copies of the Program or any portion 114 | of it, thus forming a work based on the Program, and copy and 115 | distribute such modifications or work under the terms of Section 1 116 | above, provided that you also meet all of these conditions: 117 | 118 | a) You must cause the modified files to carry prominent notices 119 | stating that you changed the files and the date of any change. 120 | 121 | b) You must cause any work that you distribute or publish, that in 122 | whole or in part contains or is derived from the Program or any 123 | part thereof, to be licensed as a whole at no charge to all third 124 | parties under the terms of this License. 125 | 126 | c) If the modified program normally reads commands interactively 127 | when run, you must cause it, when started running for such 128 | interactive use in the most ordinary way, to print or display an 129 | announcement including an appropriate copyright notice and a 130 | notice that there is no warranty (or else, saying that you provide 131 | a warranty) and that users may redistribute the program under 132 | these conditions, and telling the user how to view a copy of this 133 | License. (Exception: if the Program itself is interactive but 134 | does not normally print such an announcement, your work based on 135 | the Program is not required to print an announcement.) 136 | 137 | These requirements apply to the modified work as a whole. If 138 | identifiable sections of that work are not derived from the Program, 139 | and can be reasonably considered independent and separate works in 140 | themselves, then this License, and its terms, do not apply to those 141 | sections when you distribute them as separate works. But when you 142 | distribute the same sections as part of a whole which is a work based 143 | on the Program, the distribution of the whole must be on the terms of 144 | this License, whose permissions for other licensees extend to the 145 | entire whole, and thus to each and every part regardless of who wrote it. 146 | 147 | Thus, it is not the intent of this section to claim rights or contest 148 | your rights to work written entirely by you; rather, the intent is to 149 | exercise the right to control the distribution of derivative or 150 | collective works based on the Program. 151 | 152 | In addition, mere aggregation of another work not based on the Program 153 | with the Program (or with a work based on the Program) on a volume of 154 | a storage or distribution medium does not bring the other work under 155 | the scope of this License. 156 | 157 | 3. You may copy and distribute the Program (or a work based on it, 158 | under Section 2) in object code or executable form under the terms of 159 | Sections 1 and 2 above provided that you also do one of the following: 160 | 161 | a) Accompany it with the complete corresponding machine-readable 162 | source code, which must be distributed under the terms of Sections 163 | 1 and 2 above on a medium customarily used for software interchange; or, 164 | 165 | b) Accompany it with a written offer, valid for at least three 166 | years, to give any third party, for a charge no more than your 167 | cost of physically performing source distribution, a complete 168 | machine-readable copy of the corresponding source code, to be 169 | distributed under the terms of Sections 1 and 2 above on a medium 170 | customarily used for software interchange; or, 171 | 172 | c) Accompany it with the information you received as to the offer 173 | to distribute corresponding source code. (This alternative is 174 | allowed only for noncommercial distribution and only if you 175 | received the program in object code or executable form with such 176 | an offer, in accord with Subsection b above.) 177 | 178 | The source code for a work means the preferred form of the work for 179 | making modifications to it. For an executable work, complete source 180 | code means all the source code for all modules it contains, plus any 181 | associated interface definition files, plus the scripts used to 182 | control compilation and installation of the executable. However, as a 183 | special exception, the source code distributed need not include 184 | anything that is normally distributed (in either source or binary 185 | form) with the major components (compiler, kernel, and so on) of the 186 | operating system on which the executable runs, unless that component 187 | itself accompanies the executable. 188 | 189 | If distribution of executable or object code is made by offering 190 | access to copy from a designated place, then offering equivalent 191 | access to copy the source code from the same place counts as 192 | distribution of the source code, even though third parties are not 193 | compelled to copy the source along with the object code. 194 | 195 | 4. You may not copy, modify, sublicense, or distribute the Program 196 | except as expressly provided under this License. Any attempt 197 | otherwise to copy, modify, sublicense or distribute the Program is 198 | void, and will automatically terminate your rights under this License. 199 | However, parties who have received copies, or rights, from you under 200 | this License will not have their licenses terminated so long as such 201 | parties remain in full compliance. 202 | 203 | 5. You are not required to accept this License, since you have not 204 | signed it. However, nothing else grants you permission to modify or 205 | distribute the Program or its derivative works. These actions are 206 | prohibited by law if you do not accept this License. Therefore, by 207 | modifying or distributing the Program (or any work based on the 208 | Program), you indicate your acceptance of this License to do so, and 209 | all its terms and conditions for copying, distributing or modifying 210 | the Program or works based on it. 211 | 212 | 6. Each time you redistribute the Program (or any work based on the 213 | Program), the recipient automatically receives a license from the 214 | original licensor to copy, distribute or modify the Program subject to 215 | these terms and conditions. You may not impose any further 216 | restrictions on the recipients' exercise of the rights granted herein. 217 | You are not responsible for enforcing compliance by third parties to 218 | this License. 219 | 220 | 7. If, as a consequence of a court judgment or allegation of patent 221 | infringement or for any other reason (not limited to patent issues), 222 | conditions are imposed on you (whether by court order, agreement or 223 | otherwise) that contradict the conditions of this License, they do not 224 | excuse you from the conditions of this License. If you cannot 225 | distribute so as to satisfy simultaneously your obligations under this 226 | License and any other pertinent obligations, then as a consequence you 227 | may not distribute the Program at all. For example, if a patent 228 | license would not permit royalty-free redistribution of the Program by 229 | all those who receive copies directly or indirectly through you, then 230 | the only way you could satisfy both it and this License would be to 231 | refrain entirely from distribution of the Program. 232 | 233 | If any portion of this section is held invalid or unenforceable under 234 | any particular circumstance, the balance of the section is intended to 235 | apply and the section as a whole is intended to apply in other 236 | circumstances. 237 | 238 | It is not the purpose of this section to induce you to infringe any 239 | patents or other property right claims or to contest validity of any 240 | such claims; this section has the sole purpose of protecting the 241 | integrity of the free software distribution system, which is 242 | implemented by public license practices. Many people have made 243 | generous contributions to the wide range of software distributed 244 | through that system in reliance on consistent application of that 245 | system; it is up to the author/donor to decide if he or she is willing 246 | to distribute software through any other system and a licensee cannot 247 | impose that choice. 248 | 249 | This section is intended to make thoroughly clear what is believed to 250 | be a consequence of the rest of this License. 251 | 252 | 8. If the distribution and/or use of the Program is restricted in 253 | certain countries either by patents or by copyrighted interfaces, the 254 | original copyright holder who places the Program under this License 255 | may add an explicit geographical distribution limitation excluding 256 | those countries, so that distribution is permitted only in or among 257 | countries not thus excluded. In such case, this License incorporates 258 | the limitation as if written in the body of this License. 259 | 260 | 9. The Free Software Foundation may publish revised and/or new versions 261 | of the General Public License from time to time. Such new versions will 262 | be similar in spirit to the present version, but may differ in detail to 263 | address new problems or concerns. 264 | 265 | Each version is given a distinguishing version number. If the Program 266 | specifies a version number of this License which applies to it and "any 267 | later version", you have the option of following the terms and conditions 268 | either of that version or of any later version published by the Free 269 | Software Foundation. If the Program does not specify a version number of 270 | this License, you may choose any version ever published by the Free Software 271 | Foundation. 272 | 273 | 10. If you wish to incorporate parts of the Program into other free 274 | programs whose distribution conditions are different, write to the author 275 | to ask for permission. For software which is copyrighted by the Free 276 | Software Foundation, write to the Free Software Foundation; we sometimes 277 | make exceptions for this. Our decision will be guided by the two goals 278 | of preserving the free status of all derivatives of our free software and 279 | of promoting the sharing and reuse of software generally. 280 | 281 | NO WARRANTY 282 | 283 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 284 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 285 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 286 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 287 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 288 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 289 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 290 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 291 | REPAIR OR CORRECTION. 292 | 293 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 294 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 295 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 296 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 297 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 298 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 299 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 300 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 301 | POSSIBILITY OF SUCH DAMAGES. 302 | 303 | END OF TERMS AND CONDITIONS 304 | 305 | How to Apply These Terms to Your New Programs 306 | 307 | If you develop a new program, and you want it to be of the greatest 308 | possible use to the public, the best way to achieve this is to make it 309 | free software which everyone can redistribute and change under these terms. 310 | 311 | To do so, attach the following notices to the program. It is safest 312 | to attach them to the start of each source file to most effectively 313 | convey the exclusion of warranty; and each file should have at least 314 | the "copyright" line and a pointer to where the full notice is found. 315 | 316 | 317 | Copyright (C) 318 | 319 | This program is free software; you can redistribute it and/or modify 320 | it under the terms of the GNU General Public License as published by 321 | the Free Software Foundation; either version 2 of the License, or 322 | (at your option) any later version. 323 | 324 | This program is distributed in the hope that it will be useful, 325 | but WITHOUT ANY WARRANTY; without even the implied warranty of 326 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 327 | GNU General Public License for more details. 328 | 329 | You should have received a copy of the GNU General Public License along 330 | with this program; if not, write to the Free Software Foundation, Inc., 331 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 332 | 333 | Also add information on how to contact you by electronic and paper mail. 334 | 335 | If the program is interactive, make it output a short notice like this 336 | when it starts in an interactive mode: 337 | 338 | Gnomovision version 69, Copyright (C) year name of author 339 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 340 | This is free software, and you are welcome to redistribute it 341 | under certain conditions; type `show c' for details. 342 | 343 | The hypothetical commands `show w' and `show c' should show the appropriate 344 | parts of the General Public License. Of course, the commands you use may 345 | be called something other than `show w' and `show c'; they could even be 346 | mouse-clicks or menu items--whatever suits your program. 347 | 348 | You should also get your employer (if you work as a programmer) or your 349 | school, if any, to sign a "copyright disclaimer" for the program, if 350 | necessary. Here is a sample; alter the names: 351 | 352 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 353 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 354 | 355 | , 1 April 1989 356 | Ty Coon, President of Vice 357 | 358 | This General Public License does not permit incorporating your program into 359 | proprietary programs. If your program is a subroutine library, you may 360 | consider it more useful to permit linking proprietary applications with the 361 | library. If this is what you want to do, use the GNU Lesser General 362 | Public License instead of this License. 363 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0-only 2 | # SPDX-FileCopyrightText: Copyright (c) 2008-2023, NVIDIA CORPORATION. All rights reserved. 3 | 4 | # If KERNELRELEASE is defined, we've been invoked from the 5 | # kernel build system and can use its language. 6 | ifneq ($(KERNELRELEASE),) 7 | 8 | ifeq ($(CONFIG_ARM_FFA_TRANSPORT),m) 9 | ifdef ALLOW_ARM_FFA_TRANSPORT_AS_MODULE 10 | HAVE_ARM_FFA = $(CONFIG_ARM_FFA_TRANSPORT) 11 | KBUILD_CFLAGS += -DMODS_HAS_ARM_FFA 12 | endif 13 | else 14 | ifeq ($(CONFIG_ARM_FFA_TRANSPORT),y) 15 | HAVE_ARM_FFA = $(CONFIG_ARM_FFA_TRANSPORT) 16 | KBUILD_CFLAGS += -DMODS_HAS_ARM_FFA 17 | endif 18 | endif 19 | HAVE_ARM_FFA ?= 20 | 21 | obj-m := mods.o 22 | mods-y := mods_krnl.o 23 | mods-y += mods_mem.o 24 | mods-y += mods_irq.o 25 | mods-$(CONFIG_PCI) += mods_pci.o 26 | mods-$(CONFIG_ACPI) += mods_acpi.o 27 | mods-$(HAVE_ARM_FFA) += mods_arm_ffa.o 28 | mods-$(CONFIG_DEBUG_FS) += mods_debugfs.o 29 | mods-$(CONFIG_PPC64) += mods_ppc64.o 30 | mods-$(CONFIG_TEGRA_IVC) += mods_bpmpipc.o 31 | 32 | # Otherwise we were called directly from the command 33 | # line; invoke the kernel build system. 34 | else 35 | KERNELDIR ?= /lib/modules/$(shell uname -r)/build 36 | PWD := $(shell pwd) 37 | default: module 38 | module: 39 | $(MAKE) -C $(KERNELDIR) M=$(PWD) modules 40 | install: 41 | $(MAKE) -C $(KERNELDIR) M=$(PWD) modules_install 42 | clean: 43 | $(MAKE) -C $(KERNELDIR) M=$(PWD) clean 44 | endif 45 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | 2 | NVIDIA MODS kernel driver is a simple kernel module which provides access 3 | to devices on the PCI bus for user mode programs. 4 | 5 | -------------------------------------------------------------------------------- /mods_acpi.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2008-2023, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | static acpi_status mods_acpi_find_acpi_handler(acpi_handle, 12 | u32, 13 | void *, 14 | void **); 15 | 16 | /********************* 17 | * PRIVATE FUNCTIONS * 18 | *********************/ 19 | 20 | struct acpi_dev_children_in_data { 21 | u32 expected_acpi_ids_buf[4]; 22 | u32 num_expected_acpi_ids; 23 | u32 acpi_dev_mask; 24 | acpi_handle dev_handle; 25 | }; 26 | 27 | typedef int (*dev_children_fptr)(struct mods_client *, 28 | u32, 29 | acpi_handle, 30 | void *); 31 | 32 | static int acpi_store_dev_children(struct mods_client *client, 33 | u32 curr_acpi_id, 34 | acpi_handle dev_handle, 35 | void *out_data) 36 | { 37 | struct MODS_GET_ACPI_DEV_CHILDREN *acpi_out_data 38 | = (struct MODS_GET_ACPI_DEV_CHILDREN *)out_data; 39 | 40 | if (acpi_out_data->num_children >= ACPI_MAX_DEV_CHILDREN) { 41 | cl_error("ACPI: output buffer too small to store all children\n"); 42 | return -ENOBUFS; 43 | } 44 | 45 | acpi_out_data->children[acpi_out_data->num_children] = curr_acpi_id; 46 | ++acpi_out_data->num_children; 47 | 48 | return OK; 49 | } 50 | 51 | static int acpi_compare_dev_id(struct mods_client *client, 52 | u32 curr_acpi_id, 53 | acpi_handle dev_handle, 54 | void *out_data) 55 | { 56 | u32 i; 57 | struct acpi_dev_children_in_data *dev_child_data 58 | = (struct acpi_dev_children_in_data *)out_data; 59 | 60 | for (i = 0; i < dev_child_data->num_expected_acpi_ids; ++i) { 61 | if ((dev_child_data->expected_acpi_ids_buf[i] & dev_child_data->acpi_dev_mask) 62 | == (curr_acpi_id & dev_child_data->acpi_dev_mask)) { 63 | dev_child_data->dev_handle = dev_handle; 64 | /* 65 | * returning unique error code to signal caller that value is found 66 | */ 67 | return -EALREADY; 68 | } 69 | } 70 | return -ENODEV; 71 | } 72 | 73 | struct acpi_dev_check_context { 74 | struct mods_client *client; 75 | dev_children_fptr fptr; 76 | void *out_data; 77 | }; 78 | 79 | static int acpi_dev_check_one(struct device *dev, void *data) 80 | { 81 | struct acpi_dev_check_context *adwc = data; 82 | struct acpi_device *adev = to_acpi_device(dev); 83 | int err = OK; 84 | unsigned long long device_id = 0; 85 | acpi_status status; 86 | 87 | if (!adev) 88 | /* No ACPI device corresponding to this device */ 89 | return OK; 90 | 91 | status = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &device_id); 92 | if (ACPI_FAILURE(status)) 93 | /* Couldnt query device_id for this device */ 94 | return OK; 95 | 96 | err = adwc->fptr(adwc->client, device_id, adev->handle, adwc->out_data); 97 | return ((err == -EALREADY) ? OK : err); 98 | } 99 | 100 | static int acpi_get_dev_children(struct mods_client *client, 101 | dev_children_fptr fptr, 102 | acpi_handle dev_handle, 103 | void *out_data) 104 | { 105 | int err = OK; 106 | struct acpi_device *device = NULL; 107 | struct acpi_dev_check_context adcc = { 108 | .client = client, 109 | .fptr = fptr, 110 | .out_data = out_data, 111 | }; 112 | 113 | LOG_ENT(); 114 | 115 | #ifdef MODS_HAS_ACPI_FETCH 116 | device = acpi_fetch_acpi_dev(dev_handle); 117 | err = device ? 0 : -EINVAL; 118 | #else 119 | err = acpi_bus_get_device(dev_handle, &device); 120 | #endif 121 | if (unlikely(err)) 122 | cl_error("ACPI: device for fetching device children not found\n"); 123 | else 124 | err = device_for_each_child(&device->dev, &adcc, acpi_dev_check_one); 125 | 126 | LOG_EXT(); 127 | return err; 128 | } 129 | 130 | 131 | /* store handle if found. */ 132 | static void mods_acpi_handle_init(struct mods_client *client, 133 | char *method_name, 134 | acpi_handle *handler) 135 | { 136 | MODS_ACPI_WALK_NAMESPACE(ACPI_TYPE_ANY, 137 | ACPI_ROOT_OBJECT, 138 | ACPI_UINT32_MAX, 139 | mods_acpi_find_acpi_handler, 140 | method_name, 141 | handler); 142 | 143 | if (!(*handler)) { 144 | cl_debug(DEBUG_ACPI, 145 | "ACPI method %s not found\n", 146 | method_name); 147 | } 148 | } 149 | 150 | static acpi_status mods_acpi_find_acpi_handler( 151 | acpi_handle handle, 152 | u32 nest_level, 153 | void *dummy1, 154 | void **dummy2 155 | ) 156 | { 157 | acpi_handle acpi_method_handler_temp; 158 | 159 | if (!acpi_get_handle(handle, dummy1, &acpi_method_handler_temp)) 160 | *dummy2 = acpi_method_handler_temp; 161 | 162 | return OK; 163 | } 164 | 165 | static int mods_extract_acpi_object(struct mods_client *client, 166 | char *method, 167 | union acpi_object *obj, 168 | u8 **buf, 169 | u8 *buf_end) 170 | { 171 | int err = OK; 172 | 173 | switch (obj->type) { 174 | 175 | case ACPI_TYPE_BUFFER: 176 | if (obj->buffer.length == 0) { 177 | cl_error( 178 | "empty ACPI output buffer from ACPI method %s\n", 179 | method); 180 | err = -EINVAL; 181 | } else if (obj->buffer.length <= buf_end-*buf) { 182 | u32 size = obj->buffer.length; 183 | 184 | memcpy(*buf, obj->buffer.pointer, size); 185 | *buf += size; 186 | } else { 187 | cl_error("output buffer too small for ACPI method %s\n", 188 | method); 189 | err = -EINVAL; 190 | } 191 | break; 192 | 193 | case ACPI_TYPE_INTEGER: 194 | if (buf_end - *buf >= 4) { 195 | if (obj->integer.value > 0xFFFFFFFFU) { 196 | cl_error( 197 | "integer value from ACPI method %s out of range\n", 198 | method); 199 | err = -EINVAL; 200 | } else { 201 | memcpy(*buf, &obj->integer.value, 4); 202 | *buf += 4; 203 | } 204 | } else { 205 | cl_error("output buffer too small for ACPI method %s\n", 206 | method); 207 | err = -EINVAL; 208 | } 209 | break; 210 | 211 | case ACPI_TYPE_PACKAGE: 212 | if (obj->package.count == 0) { 213 | cl_error( 214 | "empty ACPI output package from ACPI method %s\n", 215 | method); 216 | err = -EINVAL; 217 | } else { 218 | union acpi_object *elements = obj->package.elements; 219 | u32 size = 0; 220 | u32 i; 221 | 222 | for (i = 0; i < obj->package.count; i++) { 223 | u8 *old_buf = *buf; 224 | u32 new_size; 225 | 226 | err = mods_extract_acpi_object(client, 227 | method, 228 | &elements[i], 229 | buf, 230 | buf_end); 231 | if (err) 232 | break; 233 | 234 | new_size = *buf - old_buf; 235 | 236 | if (size == 0) { 237 | size = new_size; 238 | } else if (size != new_size) { 239 | cl_error( 240 | "ambiguous package element size from ACPI method %s\n", 241 | method); 242 | err = -EINVAL; 243 | } 244 | } 245 | } 246 | break; 247 | 248 | case ACPI_TYPE_LOCAL_REFERENCE: 249 | if (obj->reference.actual_type == ACPI_TYPE_POWER) { 250 | memcpy(*buf, &obj->reference.handle, 251 | sizeof(obj->reference.handle)); 252 | *buf += sizeof(obj->reference.handle); 253 | } else { 254 | cl_error("Unsupported ACPI reference type\n"); 255 | err = -EINVAL; 256 | } 257 | break; 258 | 259 | default: 260 | cl_error("unsupported ACPI output type 0x%02x from method %s\n", 261 | (unsigned int)obj->type, 262 | method); 263 | err = -EINVAL; 264 | break; 265 | 266 | } 267 | return err; 268 | } 269 | 270 | static int mods_eval_acpi_method(struct mods_client *client, 271 | struct MODS_EVAL_ACPI_METHOD *p, 272 | struct mods_pci_dev_2 *pdevice, 273 | u32 acpi_id) 274 | { 275 | int err = OK; 276 | int i; 277 | acpi_status status; 278 | struct acpi_object_list input; 279 | struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; 280 | union acpi_object *acpi_method = NULL; 281 | union acpi_object acpi_params[ACPI_MAX_ARGUMENT_NUMBER]; 282 | acpi_handle acpi_method_handler = NULL; 283 | struct pci_dev *dev = NULL; 284 | 285 | LOG_ENT(); 286 | 287 | if (p->argument_count >= ACPI_MAX_ARGUMENT_NUMBER) { 288 | cl_error("invalid argument count for ACPI call\n"); 289 | LOG_EXT(); 290 | return -EINVAL; 291 | } 292 | 293 | if (pdevice) { 294 | cl_debug(DEBUG_ACPI, 295 | "ACPI %s for dev %04x:%02x:%02x.%x\n", 296 | p->method_name, 297 | pdevice->domain, 298 | pdevice->bus, 299 | pdevice->device, 300 | pdevice->function); 301 | 302 | err = mods_find_pci_dev(client, pdevice, &dev); 303 | if (unlikely(err)) { 304 | if (err == -ENODEV) 305 | cl_error( 306 | "ACPI: dev %04x:%02x:%02x.%x not found\n", 307 | pdevice->domain, 308 | pdevice->bus, 309 | pdevice->device, 310 | pdevice->function); 311 | LOG_EXT(); 312 | return err; 313 | } 314 | acpi_method_handler = MODS_ACPI_HANDLE(&dev->dev); 315 | } else { 316 | cl_debug(DEBUG_ACPI, "ACPI %s\n", p->method_name); 317 | mods_acpi_handle_init(client, 318 | p->method_name, 319 | &acpi_method_handler); 320 | } 321 | 322 | if (acpi_id != ACPI_MODS_IGNORE_ACPI_ID) { 323 | struct acpi_dev_children_in_data in_data = { {}, 0, 0, NULL}; 324 | 325 | in_data.expected_acpi_ids_buf[0] = acpi_id; 326 | in_data.num_expected_acpi_ids = 1; 327 | in_data.acpi_dev_mask = 0xffffffff; 328 | 329 | err = acpi_get_dev_children(client, acpi_compare_dev_id, 330 | acpi_method_handler, (void *)&in_data); 331 | if (err) { 332 | pci_dev_put(dev); 333 | LOG_EXT(); 334 | return -EINVAL; 335 | } 336 | acpi_method_handler = in_data.dev_handle; 337 | } 338 | 339 | if (!acpi_method_handler) { 340 | cl_debug(DEBUG_ACPI, 341 | "ACPI: handle for %s not found\n", 342 | p->method_name); 343 | pci_dev_put(dev); 344 | LOG_EXT(); 345 | return -EINVAL; 346 | } 347 | cl_debug(DEBUG_ACPI, 348 | "ACPI: found %s (id = 0x%x) on dev %04x:%02x:%02x.%x\n", 349 | p->method_name, 350 | (unsigned int)acpi_id, 351 | pdevice ? pdevice->domain : 0U, 352 | pdevice ? pdevice->bus : 0U, 353 | pdevice ? pdevice->device : 0U, 354 | pdevice ? pdevice->function : 0U); 355 | 356 | input.count = p->argument_count; 357 | input.pointer = acpi_params; 358 | 359 | for (i = 0; i < p->argument_count; i++) { 360 | switch (p->argument[i].type) { 361 | case ACPI_MODS_TYPE_INTEGER: { 362 | acpi_params[i].integer.type = ACPI_TYPE_INTEGER; 363 | acpi_params[i].integer.value 364 | = p->argument[i].integer.value; 365 | break; 366 | } 367 | case ACPI_MODS_TYPE_BUFFER: { 368 | acpi_params[i].buffer.type = ACPI_TYPE_BUFFER; 369 | acpi_params[i].buffer.length 370 | = p->argument[i].buffer.length; 371 | acpi_params[i].buffer.pointer 372 | = p->in_buffer + p->argument[i].buffer.offset; 373 | break; 374 | } 375 | case ACPI_MODS_TYPE_METHOD: { 376 | memcpy(&acpi_method_handler, 377 | &p->argument[i].method.handle, 378 | sizeof(acpi_method_handler)); 379 | 380 | if (!acpi_method_handler) { 381 | cl_error("ACPI: Invalid reference handle 0\n"); 382 | pci_dev_put(dev); 383 | LOG_EXT(); 384 | return -EINVAL; 385 | } 386 | 387 | if (i != p->argument_count - 1) { 388 | cl_error("ACPI: Invalid argument count\n"); 389 | pci_dev_put(dev); 390 | LOG_EXT(); 391 | return -EINVAL; 392 | } 393 | 394 | --input.count; 395 | break; 396 | } 397 | default: { 398 | cl_error("unsupported ACPI argument type\n"); 399 | pci_dev_put(dev); 400 | LOG_EXT(); 401 | return -EINVAL; 402 | } 403 | } 404 | } 405 | 406 | status = acpi_evaluate_object(acpi_method_handler, 407 | pdevice ? p->method_name : NULL, 408 | &input, 409 | &output); 410 | 411 | if (ACPI_FAILURE(status)) { 412 | if (status == AE_NOT_FOUND) { 413 | cl_debug(DEBUG_ACPI, "ACPI method %s not found\n", p->method_name); 414 | err = -ENXIO; 415 | } else { 416 | cl_error("ACPI method %s failed\n", p->method_name); 417 | err = -EIO; 418 | } 419 | pci_dev_put(dev); 420 | LOG_EXT(); 421 | return err; 422 | } 423 | 424 | acpi_method = output.pointer; 425 | if (!acpi_method) { 426 | cl_error("missing output from ACPI method %s\n", 427 | p->method_name); 428 | err = -EIO; 429 | } else { 430 | u8 *buf = p->out_buffer; 431 | 432 | err = mods_extract_acpi_object(client, 433 | p->method_name, 434 | acpi_method, 435 | &buf, 436 | buf+sizeof(p->out_buffer)); 437 | p->out_data_size = err ? 0 : (buf - p->out_buffer); 438 | } 439 | 440 | kfree(output.pointer); 441 | pci_dev_put(dev); 442 | LOG_EXT(); 443 | return err; 444 | } 445 | 446 | static int mods_acpi_get_ddc(struct mods_client *client, 447 | struct MODS_ACPI_GET_DDC_2 *p, 448 | struct mods_pci_dev_2 *pci_device) 449 | { 450 | int err; 451 | acpi_status status; 452 | union acpi_object *ddc; 453 | union acpi_object ddc_arg0 = { ACPI_TYPE_INTEGER }; 454 | struct acpi_object_list input = { 1, &ddc_arg0 }; 455 | u32 i; 456 | acpi_handle dev_handle = NULL; 457 | acpi_handle lcd_dev_handle = NULL; 458 | struct pci_dev *dev = NULL; 459 | bool data_found = false; 460 | struct acpi_dev_children_in_data in_data = { {}, 0, 0, NULL }; 461 | 462 | LOG_ENT(); 463 | 464 | cl_debug(DEBUG_ACPI, 465 | "ACPI _DDC (EDID) for dev %04x:%02x:%02x.%x\n", 466 | pci_device->domain, 467 | pci_device->bus, 468 | pci_device->device, 469 | pci_device->function); 470 | 471 | err = mods_find_pci_dev(client, pci_device, &dev); 472 | if (unlikely(err)) { 473 | if (err == -ENODEV) 474 | cl_error("ACPI: dev %04x:%02x:%02x.%x not found\n", 475 | pci_device->domain, 476 | pci_device->bus, 477 | pci_device->device, 478 | pci_device->function); 479 | LOG_EXT(); 480 | return err; 481 | } 482 | 483 | dev_handle = MODS_ACPI_HANDLE(&dev->dev); 484 | if (!dev_handle) { 485 | cl_debug(DEBUG_ACPI, "ACPI: handle for _DDC not found\n"); 486 | pci_dev_put(dev); 487 | LOG_EXT(); 488 | return -EINVAL; 489 | } 490 | 491 | /* 492 | * List of supported display's (panels) 493 | * Reference: https://uefi.org/sites/default/files/resources/ACPI_6_1.pdf 494 | * Section: Table B-390 Video Output Device Attributes 495 | */ 496 | in_data.expected_acpi_ids_buf[0] = 0x0110; 497 | in_data.expected_acpi_ids_buf[1] = 0x0118; 498 | in_data.expected_acpi_ids_buf[2] = 0x0400; 499 | in_data.expected_acpi_ids_buf[3] = 0xa450; 500 | in_data.num_expected_acpi_ids = 4; 501 | in_data.acpi_dev_mask = 0xffff; 502 | 503 | err = acpi_get_dev_children(client, acpi_compare_dev_id, 504 | dev_handle, (void *)&in_data); 505 | if (err) { 506 | pci_dev_put(dev); 507 | LOG_EXT(); 508 | return -EINVAL; 509 | } 510 | lcd_dev_handle = in_data.dev_handle; 511 | 512 | if (lcd_dev_handle == NULL) { 513 | cl_error("ACPI: LCD not found for dev %04x:%02x:%02x.%x\n", 514 | p->device.domain, 515 | p->device.bus, 516 | p->device.device, 517 | p->device.function); 518 | pci_dev_put(dev); 519 | LOG_EXT(); 520 | return -EINVAL; 521 | } 522 | 523 | cl_debug(DEBUG_ACPI, 524 | "ACPI: Found LCD on dev %04x:%02x:%02x.%x\n", 525 | p->device.domain, 526 | p->device.bus, 527 | p->device.device, 528 | p->device.function); 529 | 530 | /* 531 | * As per ACPI Spec 3.0: 532 | * ARG0 is size of EDID buffer in 128-byte blocks. 533 | */ 534 | err = -EINVAL; 535 | for (i = 4; i >= 1; --i) { 536 | struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; 537 | 538 | ddc_arg0.integer.value = i; 539 | status = acpi_evaluate_object(lcd_dev_handle, 540 | "_DDC", 541 | &input, 542 | &output); 543 | if (ACPI_SUCCESS(status)) { 544 | ddc = output.pointer; 545 | if (ddc && (ddc->type != ACPI_TYPE_BUFFER)) 546 | continue; 547 | 548 | if (!ddc || ddc->buffer.length == 0) { 549 | cl_error("unsupported ACPI output type\n"); 550 | } else if (ddc->buffer.length > sizeof(p->out_buffer)) { 551 | cl_error( 552 | "output buffer too small for ACPI method _DDC (EDID)\n"); 553 | } else { 554 | p->out_data_size = ddc->buffer.length; 555 | memcpy(p->out_buffer, 556 | ddc->buffer.pointer, 557 | p->out_data_size); 558 | err = OK; 559 | data_found = true; 560 | } 561 | 562 | kfree(ddc); 563 | 564 | break; 565 | } 566 | } 567 | 568 | if (!data_found) 569 | cl_error("ACPI method _DDC (EDID) failed\n"); 570 | 571 | pci_dev_put(dev); 572 | LOG_EXT(); 573 | return err; 574 | } 575 | 576 | /************************* 577 | * ESCAPE CALL FUNCTIONS * 578 | *************************/ 579 | 580 | int esc_mods_eval_acpi_method(struct mods_client *client, 581 | struct MODS_EVAL_ACPI_METHOD *p) 582 | { 583 | return mods_eval_acpi_method(client, p, NULL, ACPI_MODS_IGNORE_ACPI_ID); 584 | } 585 | 586 | int esc_mods_eval_dev_acpi_method_3(struct mods_client *client, 587 | struct MODS_EVAL_DEV_ACPI_METHOD_3 *p) 588 | { 589 | return mods_eval_acpi_method(client, 590 | &p->method, 591 | &p->device, 592 | p->acpi_id); 593 | } 594 | 595 | int esc_mods_eval_dev_acpi_method_2(struct mods_client *client, 596 | struct MODS_EVAL_DEV_ACPI_METHOD_2 *p) 597 | { 598 | return mods_eval_acpi_method(client, 599 | &p->method, 600 | &p->device, 601 | ACPI_MODS_IGNORE_ACPI_ID); 602 | } 603 | 604 | int esc_mods_eval_dev_acpi_method(struct mods_client *client, 605 | struct MODS_EVAL_DEV_ACPI_METHOD *p) 606 | { 607 | struct mods_pci_dev_2 device = {0}; 608 | 609 | device.domain = 0; 610 | device.bus = p->device.bus; 611 | device.device = p->device.device; 612 | device.function = p->device.function; 613 | return mods_eval_acpi_method(client, &p->method, &device, 614 | ACPI_MODS_IGNORE_ACPI_ID); 615 | } 616 | 617 | int esc_mods_acpi_get_ddc_2(struct mods_client *client, 618 | struct MODS_ACPI_GET_DDC_2 *p) 619 | { 620 | return mods_acpi_get_ddc(client, p, &p->device); 621 | } 622 | 623 | int esc_mods_acpi_get_ddc(struct mods_client *client, 624 | struct MODS_ACPI_GET_DDC *p) 625 | { 626 | struct MODS_ACPI_GET_DDC_2 *pp = (struct MODS_ACPI_GET_DDC_2 *) p; 627 | struct mods_pci_dev_2 device = {0}; 628 | 629 | device.domain = 0; 630 | device.bus = p->device.bus; 631 | device.device = p->device.device; 632 | device.function = p->device.function; 633 | 634 | return mods_acpi_get_ddc(client, pp, &device); 635 | } 636 | 637 | int esc_mods_get_acpi_dev_children(struct mods_client *client, 638 | struct MODS_GET_ACPI_DEV_CHILDREN *p) 639 | { 640 | int err; 641 | struct pci_dev *dev = NULL; 642 | acpi_handle dev_handle = NULL; 643 | 644 | LOG_ENT(); 645 | 646 | cl_debug(DEBUG_ACPI, 647 | "ACPI: failed to get children for dev %04x:%02x:%02x.%x\n", 648 | p->device.domain, 649 | p->device.bus, 650 | p->device.device, 651 | p->device.function); 652 | 653 | err = mods_find_pci_dev(client, &p->device, &dev); 654 | if (unlikely(err)) { 655 | if (err == -ENODEV) 656 | cl_error("ACPI: dev %04x:%02x:%02x.%x not found\n", 657 | p->device.domain, 658 | p->device.bus, 659 | p->device.device, 660 | p->device.function); 661 | LOG_EXT(); 662 | return err; 663 | } 664 | 665 | dev_handle = MODS_ACPI_HANDLE(&dev->dev); 666 | if (!dev_handle) { 667 | cl_error("ACPI: handle for fetching device children not found\n"); 668 | pci_dev_put(dev); 669 | LOG_EXT(); 670 | return -EINVAL; 671 | } 672 | 673 | p->num_children = 0; 674 | err = acpi_get_dev_children(client, acpi_store_dev_children, 675 | dev_handle, (void *)p); 676 | 677 | if (err) { 678 | cl_error("ACPI: failed to get children for dev %04x:%02x:%02x.%x\n", 679 | p->device.domain, 680 | p->device.bus, 681 | p->device.device, 682 | p->device.function); 683 | } 684 | 685 | pci_dev_put(dev); 686 | LOG_EXT(); 687 | return err; 688 | } 689 | 690 | #ifdef MODS_HAS_PXM_TO_NODE 691 | int esc_mods_proximity_to_numa_node(struct mods_client *client, 692 | struct MODS_PROXIMITY_TO_NUMA_NODE *p) 693 | { 694 | p->numa_node = acpi_map_pxm_to_node(p->proximity); 695 | return OK; 696 | } 697 | #endif 698 | -------------------------------------------------------------------------------- /mods_arm_ffa.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | static const struct ffa_device_id mods_ffa_device_id[] = { 15 | { UUID_INIT(0x1f4bfeb9, 0x0f48, 0xdd1e, 16 | 0x11, 0x9c, 0x2c, 0x86, 0xc9, 0x14, 0x03, 0x22) }, 17 | {} 18 | }; 19 | 20 | struct mods_ffa_ctx { 21 | struct ffa_device *ffa_dev; 22 | #if KERNEL_VERSION(6, 1, 0) <= MODS_KERNEL_VERSION || defined(FFA_PARTITION_AARCH64_EXEC) 23 | const struct ffa_msg_ops *ffa_ops; 24 | #else 25 | const struct ffa_dev_ops *ffa_ops; 26 | #endif 27 | }; 28 | 29 | static DEFINE_MUTEX(mods_ffa_lock); 30 | 31 | static struct mods_ffa_ctx mods_ffa_info; 32 | 33 | static int ffa_probe(struct ffa_device *ffa_dev) 34 | { 35 | int ret = 0; 36 | 37 | #if KERNEL_VERSION(6, 1, 0) <= MODS_KERNEL_VERSION || defined(FFA_PARTITION_AARCH64_EXEC) 38 | const struct ffa_msg_ops *ffa_ops = NULL; 39 | 40 | if (ffa_dev->ops) 41 | ffa_ops = ffa_dev->ops->msg_ops; 42 | #else 43 | const struct ffa_dev_ops *ffa_ops; 44 | 45 | ffa_ops = ffa_dev_ops_get(ffa_dev); 46 | #endif 47 | if (!ffa_ops) { 48 | mods_error_printk("failed \"method\" init: ffa\n"); 49 | return -ENOENT; 50 | } 51 | mods_ffa_info.ffa_dev = ffa_dev; 52 | mods_ffa_info.ffa_ops = ffa_ops; 53 | 54 | mods_debug_printk(DEBUG_TEGRADMA, "mods ffa driver registered\n"); 55 | 56 | return ret; 57 | } 58 | 59 | static void ffa_remove(struct ffa_device *ffa_dev) 60 | { 61 | mods_ffa_info.ffa_dev = NULL; 62 | mods_ffa_info.ffa_ops = NULL; 63 | } 64 | 65 | static struct ffa_driver mods_ffa_driver = { 66 | .name = "mods_arm_ffa", 67 | .probe = ffa_probe, 68 | .remove = ffa_remove, 69 | .id_table = mods_ffa_device_id, 70 | }; 71 | 72 | int mods_ffa_abi_register(void) 73 | { 74 | mods_debug_printk(DEBUG_TEGRADMA, "registering MODS FFA driver\n"); 75 | return ffa_register(&mods_ffa_driver); 76 | } 77 | 78 | void mods_ffa_abi_unregister(void) 79 | { 80 | ffa_unregister(&mods_ffa_driver); 81 | } 82 | 83 | int esc_mods_arm_ffa_cmd(struct mods_client *client, 84 | struct MODS_FFA_PARAMS *p) 85 | { 86 | int err = -EINVAL; 87 | struct ffa_send_direct_data data = { 0 }; 88 | 89 | // Fill the reg TX command parameters 90 | data.data0 = p->cmd; 91 | // 64 bit of the physical address 92 | data.data1 = p->indata[0]; 93 | // 32 bit of the reg value 94 | data.data2 = p->indata[1]; 95 | 96 | if (!mods_ffa_info.ffa_ops) { 97 | cl_error("mods ffa cmd error, device not found\n"); 98 | return -ENODEV; 99 | } 100 | 101 | switch (p->cmd) { 102 | case MODS_FFA_CMD_READ_REG: 103 | // Read command 104 | cl_debug(DEBUG_TEGRADMA, "sending data to SP :read cmd 0x%llx, addr:0x%llx\n", 105 | (unsigned long long)data.data0, 106 | (unsigned long long)data.data1); 107 | break; 108 | case MODS_FFA_CMD_WRITE_REG: 109 | // Write command 110 | cl_debug(DEBUG_TEGRADMA, "sending data to SP :write cmd 0x%llx,addr:0x%llx,write_val:0x%llx\n", 111 | (unsigned long long)data.data0, 112 | (unsigned long long)data.data1, 113 | (unsigned long long)data.data2); 114 | break; 115 | case MODS_FFA_CMD_READ_VER: 116 | cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_READ_VER to SP\n"); 117 | break; 118 | case MODS_FFA_CMD_SE_TESTS: 119 | cl_debug(DEBUG_TEGRADMA, "sending SE_TESTS data to SP :read cmd 0x%llx, alg|engineId:0x%llx\n", 120 | (unsigned long long)data.data0, 121 | (unsigned long long)data.data1); 122 | break; 123 | case MODS_FFA_CMD_SE_KEY_MOVER: 124 | cl_debug(DEBUG_TEGRADMA, "sending SE_KEY_MOVER data to SP :read cmd 0x%llx, data:0x%llx\n", 125 | (unsigned long long)data.data0, 126 | (unsigned long long)data.data1); 127 | break; 128 | case MODS_FFA_CMD_HSS_TEST: 129 | cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_HSS_TEST to SP\n"); 130 | break; 131 | case MODS_FFA_CMD_C2C_TEST: 132 | cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_C2C_TEST to SP\n"); 133 | break; 134 | case MODS_FFA_CMD_MISC: 135 | cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_MISC to SP\n"); 136 | break; 137 | default: 138 | cl_error("Unexpected command from SP 0x%llx\n", (unsigned long long)p->cmd); 139 | return err; 140 | } 141 | 142 | mutex_lock(&mods_ffa_lock); 143 | err = mods_ffa_info.ffa_ops->sync_send_receive(mods_ffa_info.ffa_dev, &data); 144 | mutex_unlock(&mods_ffa_lock); 145 | 146 | switch (p->cmd) { 147 | case MODS_FFA_CMD_READ_REG: 148 | // Read command 149 | cl_debug(DEBUG_TEGRADMA, "received read reg status from SP status:%d,read_val:0x%llx\n", 150 | err, (unsigned long long)data.data1); 151 | p->outdata[0] = data.data1; 152 | break; 153 | case MODS_FFA_CMD_WRITE_REG: 154 | // write command 155 | cl_debug(DEBUG_TEGRADMA, "received write reg status from SP status: %d\n", 156 | err); 157 | break; 158 | case MODS_FFA_CMD_READ_VER: 159 | cl_debug(DEBUG_TEGRADMA, "received version from SP : 0x%llx\n", 160 | (unsigned long long)data.data1); 161 | p->outdata[0] = data.data1; 162 | break; 163 | case MODS_FFA_CMD_SE_TESTS: 164 | case MODS_FFA_CMD_SE_KEY_MOVER: 165 | p->outdata[0] = data.data1; 166 | break; 167 | case MODS_FFA_CMD_HSS_TEST: 168 | case MODS_FFA_CMD_C2C_TEST: 169 | case MODS_FFA_CMD_MISC: 170 | cl_debug(DEBUG_TEGRADMA, "received response from SP: 0x%llx\n", 171 | (unsigned long long)data.data1); 172 | p->outdata[0] = data.data1; 173 | break; 174 | } 175 | 176 | if (err) { 177 | cl_error("unexpected error from SP: %d\n", err); 178 | return err; 179 | } 180 | // data.data0 always holds the error code of the ffa cmd 181 | if (data.data0) { 182 | cl_error("error response from SP: %ld\n", (long)data.data0); 183 | return -EFAULT; 184 | } 185 | return OK; 186 | } 187 | -------------------------------------------------------------------------------- /mods_bpmpipc.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #define IVC_CHANNEL_SIZE 256 14 | #define MRQ_MSG_SIZE 128 15 | #define BPMP_MAIL_DO_ACK (1U << 0U) 16 | #define BPMP_IVC_TIMEOUT 120000 /* really large timeout to support simulation platforms */ 17 | 18 | static DEFINE_MUTEX(mods_bpmpipc_lock); 19 | 20 | static const u32 MODS_CMD_UPHY_LANE_EOM_SCAN = 9; 21 | 22 | struct mods_cmd_uphy_lane_eom_scan_request { 23 | u32 brick; 24 | u32 lane; 25 | u32 pcie_gen5; 26 | }; 27 | 28 | struct mods_cmd_uphy_lane_eom_scan_response { 29 | u32 data; 30 | }; 31 | 32 | struct mods_mrq_uphy_request { 33 | u16 lane; 34 | u16 cmd; 35 | struct mods_cmd_uphy_lane_eom_scan_request lane_eom_scan; 36 | }; 37 | 38 | struct mods_mrq_uphy_response { 39 | struct mods_cmd_uphy_lane_eom_scan_response eom_status; 40 | }; 41 | 42 | struct bpmp_ipc_ch { 43 | bool is_init; 44 | struct tegra_ivc ivc; 45 | void __iomem *db_base; 46 | void __iomem *req_base; 47 | void __iomem *resp_base; 48 | phys_addr_t db_phys_addr; 49 | phys_addr_t req_phys_addr; 50 | phys_addr_t resp_phys_addr; 51 | }; 52 | 53 | static struct bpmp_ipc_ch mods_bpmp_ch = {.is_init = false}; 54 | 55 | static void bpmp_ivc_notify(struct tegra_ivc *ivc, void *data) 56 | { 57 | struct bpmp_ipc_ch *bpmp_ipc_ch = (struct bpmp_ipc_ch *)data; 58 | 59 | __raw_writel(1, bpmp_ipc_ch->db_base); 60 | } 61 | 62 | static int bpmp_ipc_send(struct mods_client *client, 63 | struct tegra_ivc *ivc, 64 | const void *data, 65 | size_t sz) 66 | { 67 | #if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) || defined(__IOSYS_MAP_H__) 68 | int err; 69 | struct iosys_map ob; 70 | 71 | err = tegra_ivc_write_get_next_frame(ivc, &ob); 72 | if (err) { 73 | cl_error("failed to get next tegra-ivc output frame!\n"); 74 | iosys_map_clear(&ob); 75 | return err; 76 | } 77 | iosys_map_memcpy_to(&ob, 0, data, sz); 78 | #else 79 | void *frame; 80 | 81 | frame = tegra_ivc_write_get_next_frame(ivc); 82 | if (IS_ERR(frame)) { 83 | cl_error("failed to get next tegra-ivc output frame!\n"); 84 | return PTR_ERR(frame); 85 | } 86 | 87 | memcpy_toio(frame, data, sz); 88 | #endif 89 | 90 | return tegra_ivc_write_advance(ivc); 91 | } 92 | 93 | static int bpmp_ipc_recv(struct mods_client *client, 94 | struct tegra_ivc *ivc, 95 | void *data, 96 | size_t sz, 97 | u32 timeout_ms) 98 | { 99 | int err; 100 | #if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) || defined(__IOSYS_MAP_H__) 101 | struct iosys_map ib; 102 | #else 103 | const void *frame; 104 | #endif 105 | ktime_t end; 106 | 107 | end = ktime_add_ms(ktime_get(), timeout_ms); 108 | 109 | #if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) || defined(__IOSYS_MAP_H__) 110 | do { 111 | err = tegra_ivc_read_get_next_frame(ivc, &ib); 112 | if (!err) 113 | break; 114 | } while (ktime_before(ktime_get(), end)); 115 | if (err) { 116 | iosys_map_clear(&ib); 117 | err = tegra_ivc_read_get_next_frame(ivc, &ib); 118 | if (err) { 119 | cl_error("get next tegra-ivc input frame timeout\n"); 120 | iosys_map_clear(&ib); 121 | return err; 122 | } 123 | } 124 | iosys_map_memcpy_from(data, &ib, 0, sz); 125 | #else 126 | do { 127 | frame = tegra_ivc_read_get_next_frame(ivc); 128 | if (!IS_ERR(frame)) 129 | break; 130 | } while (ktime_before(ktime_get(), end)); 131 | 132 | if (IS_ERR(frame)) { 133 | frame = tegra_ivc_read_get_next_frame(ivc); 134 | 135 | if (IS_ERR(frame)) { 136 | cl_error("get next tegra-ivc input frame timeout\n"); 137 | return -ETIMEDOUT; 138 | } 139 | } 140 | memcpy_fromio(data, frame, sz); 141 | #endif 142 | 143 | err = tegra_ivc_read_advance(ivc); 144 | if (err < 0) 145 | cl_error("tegra_ivc read failed: %d\n", err); 146 | 147 | return err; 148 | } 149 | 150 | static int bpmp_transfer(struct mods_client *client, 151 | struct tegra_bpmp_message *msg) 152 | { 153 | int err; 154 | struct tegra_bpmp_mb_data req; 155 | struct tegra_bpmp_mb_data resp; 156 | 157 | req.code = msg->mrq; 158 | req.flags = BPMP_MAIL_DO_ACK; 159 | memcpy(req.data, msg->tx.data, msg->tx.size); 160 | err = bpmp_ipc_send(client, &mods_bpmp_ch.ivc, &req, sizeof(req)); 161 | 162 | if (err == 0) { 163 | err = bpmp_ipc_recv(client, &mods_bpmp_ch.ivc, 164 | &resp, 165 | sizeof(resp), 166 | BPMP_IVC_TIMEOUT); 167 | } 168 | 169 | if (err == 0) { 170 | memcpy(msg->rx.data, resp.data, msg->rx.size); 171 | msg->rx.ret = resp.code; 172 | } 173 | 174 | return err; 175 | } 176 | 177 | static int mrq_uphy_lane_eom_scan(struct mods_client *client, 178 | u32 brick, 179 | u32 lane, 180 | u32 pcie_gen5, 181 | u32 *data) 182 | { 183 | int err; 184 | struct mods_mrq_uphy_request req = { 185 | .cmd = cpu_to_le32(MODS_CMD_UPHY_LANE_EOM_SCAN) 186 | }; 187 | struct mods_mrq_uphy_response resp; 188 | struct tegra_bpmp_message msg = { 189 | .mrq = MRQ_UPHY, 190 | .tx = { 191 | .data = &req, 192 | .size = sizeof(req), 193 | }, 194 | .rx = { 195 | .data = &resp, 196 | .size = sizeof(resp), 197 | }, 198 | }; 199 | 200 | req.lane_eom_scan.brick = brick; 201 | req.lane_eom_scan.lane = lane; 202 | req.lane_eom_scan.pcie_gen5 = pcie_gen5; 203 | 204 | err = bpmp_transfer(client, &msg); 205 | 206 | if (err < 0) { 207 | return err; 208 | } else if (msg.rx.ret < 0) { 209 | err = -EINVAL; 210 | return err; 211 | } 212 | 213 | *data = resp.eom_status.data; 214 | return err; 215 | } 216 | 217 | static int bpmp_ioremap(struct mods_client *client, 218 | struct bpmp_ipc_ch *bpmp_ipc_ch, 219 | u64 db_phys_addr, 220 | u64 req_phys_addr, 221 | u64 resp_phys_addr) 222 | { 223 | bpmp_ipc_ch->db_phys_addr = db_phys_addr; 224 | bpmp_ipc_ch->req_phys_addr = req_phys_addr; 225 | bpmp_ipc_ch->resp_phys_addr = resp_phys_addr; 226 | 227 | bpmp_ipc_ch->db_base = ioremap(bpmp_ipc_ch->db_phys_addr, 64); 228 | if (!bpmp_ipc_ch->db_base) { 229 | cl_error("failed to remap aperture: 0x%llx\n", 230 | (unsigned long long)bpmp_ipc_ch->db_phys_addr); 231 | return -ENOMEM; 232 | } 233 | bpmp_ipc_ch->req_base = ioremap(bpmp_ipc_ch->req_phys_addr, IVC_CHANNEL_SIZE); 234 | if (!bpmp_ipc_ch->req_base) { 235 | iounmap(bpmp_ipc_ch->db_base); 236 | cl_error("failed to remap aperture: 0x%llx\n", 237 | (unsigned long long)bpmp_ipc_ch->req_phys_addr); 238 | return -ENOMEM; 239 | } 240 | bpmp_ipc_ch->resp_base = ioremap(bpmp_ipc_ch->resp_phys_addr, IVC_CHANNEL_SIZE); 241 | if (!bpmp_ipc_ch->resp_base) { 242 | iounmap(bpmp_ipc_ch->db_base); 243 | iounmap(bpmp_ipc_ch->req_base); 244 | cl_error("failed to remap aperture: 0x%llx\n", 245 | (unsigned long long)bpmp_ipc_ch->resp_phys_addr); 246 | return -ENOMEM; 247 | } 248 | 249 | return OK; 250 | } 251 | 252 | static void bpmp_iounmap(struct bpmp_ipc_ch *bpmp_ipc_ch) 253 | { 254 | iounmap(bpmp_ipc_ch->db_base); 255 | iounmap(bpmp_ipc_ch->req_base); 256 | iounmap(bpmp_ipc_ch->resp_base); 257 | 258 | bpmp_ipc_ch->db_phys_addr = 0; 259 | bpmp_ipc_ch->req_phys_addr = 0; 260 | bpmp_ipc_ch->resp_phys_addr = 0; 261 | } 262 | 263 | static int bpmp_ipc_channel_init(struct mods_client *client, 264 | struct bpmp_ipc_ch *bpmp_ipc_ch) 265 | { 266 | int err; 267 | ktime_t end; 268 | 269 | #if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) || defined(__IOSYS_MAP_H__) 270 | struct iosys_map rx, tx; 271 | 272 | iosys_map_set_vaddr_iomem(&rx, bpmp_ipc_ch->resp_base); 273 | iosys_map_set_vaddr_iomem(&tx, bpmp_ipc_ch->req_base); 274 | 275 | err = tegra_ivc_init(&bpmp_ipc_ch->ivc, NULL, 276 | &rx, bpmp_ipc_ch->resp_phys_addr, 277 | &tx, bpmp_ipc_ch->req_phys_addr, 278 | 1, MRQ_MSG_SIZE, 279 | bpmp_ivc_notify, bpmp_ipc_ch); 280 | #else 281 | err = tegra_ivc_init(&bpmp_ipc_ch->ivc, NULL, 282 | bpmp_ipc_ch->resp_base, 0, 283 | bpmp_ipc_ch->req_base, 0, 284 | 1, MRQ_MSG_SIZE, 285 | bpmp_ivc_notify, bpmp_ipc_ch); 286 | #endif 287 | 288 | if (err != 0) { 289 | cl_error("tegra-ivc init failed: %d\n", err); 290 | return err; 291 | } 292 | 293 | tegra_ivc_reset(&bpmp_ipc_ch->ivc); 294 | 295 | end = ktime_add_us(ktime_get(), 2000 * 1000); 296 | 297 | while (tegra_ivc_notified(&bpmp_ipc_ch->ivc) != 0) { 298 | usleep_range(100, 200); 299 | if (ktime_after(ktime_get(), end)) { 300 | cl_error("initialize IVC connection timeout\n"); 301 | err = -ETIMEDOUT; 302 | break; 303 | } 304 | } 305 | 306 | bpmp_ipc_ch->is_init = true; 307 | 308 | return err; 309 | } 310 | 311 | static void bpmp_ipc_channel_uninit(struct bpmp_ipc_ch *bpmp_ipc_ch) 312 | { 313 | tegra_ivc_cleanup(&bpmp_ipc_ch->ivc); 314 | } 315 | 316 | int mods_bpmpipc_init(struct mods_client *client, 317 | u64 db_phys_addr, 318 | u64 req_phys_addr, 319 | u64 resp_phys_addr) 320 | { 321 | int err = OK; 322 | 323 | if (mods_bpmp_ch.is_init) { 324 | if (mods_bpmp_ch.db_phys_addr == db_phys_addr && 325 | mods_bpmp_ch.req_phys_addr == req_phys_addr && 326 | mods_bpmp_ch.resp_phys_addr == resp_phys_addr) 327 | return OK; 328 | mods_bpmpipc_cleanup(); 329 | } 330 | 331 | err = bpmp_ioremap(client, 332 | &mods_bpmp_ch, 333 | db_phys_addr, 334 | req_phys_addr, 335 | resp_phys_addr); 336 | if (err != OK) 337 | return err; 338 | 339 | err = bpmp_ipc_channel_init(client, &mods_bpmp_ch); 340 | if (err != OK) { 341 | bpmp_iounmap(&mods_bpmp_ch); 342 | return err; 343 | } 344 | 345 | mods_bpmp_ch.is_init = true; 346 | mods_debug_printk(DEBUG_TEGRADMA, "bpmp ipc init done\n"); 347 | 348 | return err; 349 | } 350 | 351 | void mods_bpmpipc_cleanup(void) 352 | { 353 | if (!mods_bpmp_ch.is_init) 354 | return; 355 | 356 | bpmp_ipc_channel_uninit(&mods_bpmp_ch); 357 | bpmp_iounmap(&mods_bpmp_ch); 358 | mods_bpmp_ch.is_init = false; 359 | } 360 | 361 | int esc_mods_bpmp_uphy_lane_eom_scan(struct mods_client *client, 362 | struct MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS *p) 363 | { 364 | int err = OK; 365 | 366 | mutex_lock(&mods_bpmpipc_lock); 367 | 368 | err = mods_bpmpipc_init(client, 369 | p->db_phys_addr, 370 | p->req_phys_addr, 371 | p->resp_phys_addr); 372 | if (err != OK) 373 | goto error; 374 | 375 | err = mrq_uphy_lane_eom_scan(client, 376 | p->brick, 377 | p->lane, 378 | p->pcie_gen5, 379 | &p->data); 380 | 381 | if (err != OK) 382 | cl_error("mrq uphy lane eom scan failed with brick(%u), lane(%u), pcie_gen5(%u)\n", 383 | p->brick, p->lane, p->pcie_gen5); 384 | 385 | error: 386 | mutex_unlock(&mods_bpmpipc_lock); 387 | return err; 388 | } 389 | -------------------------------------------------------------------------------- /mods_config.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* SPDX-FileCopyrightText: Copyright (c) 2008-2025, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #ifndef _MODS_CONFIG_H_ 5 | #define _MODS_CONFIG_H_ 6 | 7 | #define MODS_KERNEL_VERSION LINUX_VERSION_CODE 8 | 9 | #if KERNEL_VERSION(2, 6, 30) <= MODS_KERNEL_VERSION && \ 10 | KERNEL_VERSION(4, 16, 0) > MODS_KERNEL_VERSION && \ 11 | defined(CONFIG_X86) 12 | # define MODS_HAS_DMA_OPS 1 13 | #endif 14 | 15 | #if KERNEL_VERSION(2, 6, 30) > MODS_KERNEL_VERSION 16 | # define MODS_HASNT_PCI_RESCAN_BUS 1 17 | #endif 18 | 19 | #if KERNEL_VERSION(2, 6, 31) <= MODS_KERNEL_VERSION 20 | # define MODS_HAS_IORESOURCE_MEM_64 1 21 | #endif 22 | 23 | #if KERNEL_VERSION(2, 6, 33) <= MODS_KERNEL_VERSION 24 | # define MODS_HAS_NEW_ACPI_WALK 1 25 | #else 26 | # define MODS_HASNT_NUMA_NO_NODE 1 27 | #endif 28 | 29 | #if KERNEL_VERSION(2, 6, 34) <= MODS_KERNEL_VERSION 30 | # define MODS_HAS_SET_COHERENT_MASK 1 31 | #endif 32 | 33 | #if KERNEL_VERSION(2, 6, 38) <= MODS_KERNEL_VERSION 34 | # if defined(CONFIG_X86) 35 | # define MODS_HAS_CONSOLE_LOCK 1 36 | # endif 37 | #endif 38 | 39 | #if KERNEL_VERSION(3, 4, 0) > MODS_KERNEL_VERSION 40 | # define MODS_HASNT_PCI_BUS_REMOVE_DEV 1 41 | #endif 42 | 43 | #if KERNEL_VERSION(3, 8, 0) <= MODS_KERNEL_VERSION 44 | # define MODS_HAS_NEW_ACPI_HANDLE 1 45 | # define MODS_HAS_SRIOV 1 46 | #endif 47 | 48 | #if KERNEL_VERSION(3, 13, 0) <= MODS_KERNEL_VERSION 49 | # define MODS_HAS_REINIT_COMPLETION 1 50 | #endif 51 | 52 | #if KERNEL_VERSION(3, 14, 0) <= MODS_KERNEL_VERSION 53 | # define MODS_HAS_MSIX_RANGE 1 54 | #else 55 | # define MODS_HASNT_PCI_LOCK_RESCAN_REMOVE 1 56 | #endif 57 | 58 | #if KERNEL_VERSION(3, 16, 0) <= MODS_KERNEL_VERSION && \ 59 | defined(CONFIG_VT_HW_CONSOLE_BINDING) 60 | # define MODS_HAS_CONSOLE_BINDING 1 61 | #endif 62 | 63 | #if KERNEL_VERSION(3, 19, 0) <= MODS_KERNEL_VERSION 64 | # define MODS_HAS_DEV_PROPS 1 65 | #endif 66 | 67 | #if defined(CONFIG_PPC64) && KERNEL_VERSION(4, 5, 0) <= MODS_KERNEL_VERSION 68 | # define MODS_HAS_PNV_PCI_GET_NPU_DEV 1 69 | #endif 70 | 71 | #if KERNEL_VERSION(4, 12, 0) <= MODS_KERNEL_VERSION && \ 72 | KERNEL_VERSION(4, 13, 0) > MODS_KERNEL_VERSION && \ 73 | defined(CONFIG_X86) 74 | # define MODS_HAS_ASM_SET_MEMORY_HEADER 1 75 | #endif 76 | 77 | #if KERNEL_VERSION(4, 13, 0) <= MODS_KERNEL_VERSION 78 | # define MODS_HAS_SET_MEMORY_HEADER 1 79 | #endif 80 | 81 | #if KERNEL_VERSION(4, 14, 0) <= MODS_KERNEL_VERSION 82 | # define MODS_HAS_KERNEL_WRITE 83 | # define MODS_HAS_PGPROT_DECRYPTED 84 | #endif 85 | 86 | #if KERNEL_VERSION(4, 16, 0) <= MODS_KERNEL_VERSION 87 | # define MODS_HAS_POLL_T 1 88 | #endif 89 | 90 | #if defined(CONFIG_ACPI_NUMA) && KERNEL_VERSION(5, 1, 0) <= MODS_KERNEL_VERSION 91 | # define MODS_HAS_PXM_TO_NODE 1 92 | #endif 93 | 94 | #if KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION 95 | # define MODS_HAS_DMA_ALLOC_PAGES 1 96 | #endif 97 | 98 | #if KERNEL_VERSION(5, 17, 0) <= MODS_KERNEL_VERSION 99 | # define MODS_HAS_ACPI_FETCH 1 100 | #endif 101 | 102 | #if defined(MODS_HAS_TEGRA) && KERNEL_VERSION(5, 1, 0) <= MODS_KERNEL_VERSION 103 | # define MODS_ENABLE_BPMP_MRQ_API 1 104 | #endif 105 | 106 | #if (KERNEL_VERSION(5, 14, 0) > MODS_KERNEL_VERSION) 107 | # define MODS_HAS_FB_SET_SUSPEND 1 108 | #elif (KERNEL_VERSION(5, 15, 0) > MODS_KERNEL_VERSION) && !defined(CONFIG_RHEL_DIFFERENCES) 109 | # define MODS_HAS_FB_SET_SUSPEND 1 110 | #elif (KERNEL_VERSION(6, 1, 0) > MODS_KERNEL_VERSION) && \ 111 | !defined(CONFIG_CHROME_PLATFORMS) && \ 112 | !defined(CONFIG_RHEL_DIFFERENCES) 113 | # define MODS_HAS_FB_SET_SUSPEND 1 114 | #endif 115 | 116 | #ifndef IS_BUILTIN 117 | # define IS_BUILTIN(c) 0 118 | #endif 119 | 120 | #ifndef IS_MODULE 121 | # define IS_MODULE(c) 0 122 | #endif 123 | 124 | #endif /* _MODS_CONFIG_H_ */ 125 | -------------------------------------------------------------------------------- /mods_debugfs.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2014-2023, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | static struct dentry *mods_debugfs_dir; 14 | 15 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_KFUSE) 16 | #include 17 | #endif 18 | 19 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_KFUSE) 20 | static int mods_kfuse_show(struct seq_file *s, void *unused) 21 | { 22 | unsigned int buf[KFUSE_DATA_SZ / 4]; 23 | 24 | /* copy load kfuse into buffer - only needed for early Tegra parts */ 25 | int ret = tegra_kfuse_read(buf, sizeof(buf)); 26 | int i; 27 | 28 | if (ret) 29 | return ret; 30 | 31 | for (i = 0; i < KFUSE_DATA_SZ / 4; i += 4) 32 | seq_printf(s, "0x%08x 0x%08x 0x%08x 0x%08x\n", 33 | buf[i], buf[i+1], buf[i+2], buf[i+3]); 34 | 35 | return 0; 36 | } 37 | 38 | static int mods_kfuse_open(struct inode *inode, struct file *file) 39 | { 40 | return single_open(file, mods_kfuse_show, inode->i_private); 41 | } 42 | 43 | static const struct file_operations mods_kfuse_fops = { 44 | .open = mods_kfuse_open, 45 | .read = seq_read, 46 | .llseek = seq_lseek, 47 | .release = single_release, 48 | }; 49 | #endif /* MODS_HAS_TEGRA */ 50 | 51 | static int mods_debug_get(void *data, u64 *val) 52 | { 53 | *val = (u64)(mods_get_debug_level() & DEBUG_ALL); 54 | return 0; 55 | } 56 | static int mods_debug_set(void *data, u64 val) 57 | { 58 | mods_set_debug_level((int)val & DEBUG_ALL); 59 | return 0; 60 | } 61 | DEFINE_SIMPLE_ATTRIBUTE(mods_debug_fops, mods_debug_get, mods_debug_set, 62 | "0x%08llx\n"); 63 | 64 | static int mods_mi_get(void *data, u64 *val) 65 | { 66 | *val = (u64)mods_get_multi_instance(); 67 | return 0; 68 | } 69 | static int mods_mi_set(void *data, u64 val) 70 | { 71 | mods_set_multi_instance((int)val); 72 | return 0; 73 | } 74 | 75 | DEFINE_SIMPLE_ATTRIBUTE(mods_mi_fops, mods_mi_get, mods_mi_set, "%llu\n"); 76 | 77 | static int mods_ffa_get(void *data, u64 *val) 78 | { 79 | #if defined(MODS_HAS_ARM_FFA) 80 | *val = 1ULL; 81 | #else 82 | *val = 0ULL; 83 | #endif 84 | return 0; 85 | } 86 | 87 | DEFINE_SIMPLE_ATTRIBUTE(mods_ffa_fops, mods_ffa_get, NULL, "%llu\n"); 88 | 89 | void mods_remove_debugfs(void) 90 | { 91 | debugfs_remove_recursive(mods_debugfs_dir); 92 | mods_debugfs_dir = NULL; 93 | } 94 | 95 | int mods_create_debugfs(struct miscdevice *modsdev) 96 | { 97 | struct dentry *retval; 98 | int err = 0; 99 | 100 | mods_debugfs_dir = debugfs_create_dir(dev_name(modsdev->this_device), 101 | NULL); 102 | if (IS_ERR(mods_debugfs_dir)) { 103 | err = -EIO; 104 | goto remove_out; 105 | } 106 | 107 | retval = debugfs_create_file("debug", 0644, 108 | mods_debugfs_dir, NULL, &mods_debug_fops); 109 | if (IS_ERR(retval)) { 110 | err = -EIO; 111 | goto remove_out; 112 | } 113 | 114 | retval = debugfs_create_file("multi_instance", 0644, 115 | mods_debugfs_dir, NULL, &mods_mi_fops); 116 | if (IS_ERR(retval)) { 117 | err = -EIO; 118 | goto remove_out; 119 | } 120 | 121 | retval = debugfs_create_file("ffa", 0444, 122 | mods_debugfs_dir, NULL, &mods_ffa_fops); 123 | if (IS_ERR(retval)) { 124 | err = -EIO; 125 | goto remove_out; 126 | } 127 | 128 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_KFUSE) 129 | retval = debugfs_create_file("kfuse_data", 0444, 130 | mods_debugfs_dir, NULL, &mods_kfuse_fops); 131 | if (IS_ERR(retval)) { 132 | err = -EIO; 133 | goto remove_out; 134 | } 135 | #endif 136 | 137 | return 0; 138 | remove_out: 139 | dev_err(modsdev->this_device, "could not create debugfs\n"); 140 | mods_remove_debugfs(); 141 | return err; 142 | } 143 | -------------------------------------------------------------------------------- /mods_internal.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* SPDX-FileCopyrightText: Copyright (c) 2008-2025, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #ifndef _MODS_INTERNAL_H_ 5 | #define _MODS_INTERNAL_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "mods_config.h" 18 | #include "mods.h" 19 | 20 | #ifdef MODS_HAS_ASM_SET_MEMORY_HEADER 21 | #include 22 | #elif defined(MODS_HAS_SET_MEMORY_HEADER) 23 | #include 24 | #endif 25 | 26 | #ifdef MODS_ENABLE_BPMP_MRQ_API 27 | #include 28 | #endif 29 | 30 | #ifndef true 31 | #define true 1 32 | #define false 0 33 | #endif 34 | 35 | /* function return code */ 36 | #define OK 0 37 | 38 | #define IRQ_FOUND 1 39 | #define IRQ_NOT_FOUND 0 40 | 41 | #define DEV_FOUND 1 42 | #define DEV_NOT_FOUND 0 43 | 44 | #define MSI_DEV_FOUND 1 45 | #define MSI_DEV_NOT_FOUND 0 46 | 47 | struct en_dev_entry { 48 | struct list_head list; 49 | struct pci_dev *dev; 50 | struct msix_entry *msix_entries; 51 | struct completion client_completion; 52 | u32 irq_flags; 53 | u32 nvecs; 54 | #ifdef MODS_HAS_SRIOV 55 | u32 num_vfs; 56 | #endif 57 | u8 client_id; 58 | }; 59 | 60 | struct mem_type { 61 | u64 phys_addr; 62 | u64 size; 63 | u8 type; 64 | }; 65 | 66 | struct irq_q_data { 67 | u32 time; 68 | struct pci_dev *dev; 69 | u32 irq; 70 | u32 irq_index; 71 | }; 72 | 73 | struct irq_q_info { 74 | struct irq_q_data data[MODS_MAX_IRQS]; 75 | u32 head; 76 | u32 tail; 77 | }; 78 | 79 | /* The driver can be opened simultaneously multiple times, from the same or from 80 | * different processes. This structure tracks data specific to each open fd. 81 | */ 82 | struct mods_client { 83 | struct list_head irq_list; 84 | struct list_head mem_alloc_list; 85 | struct list_head mem_map_list; 86 | struct list_head free_mem_list; /* unused UC/WC chunks */ 87 | #if defined(CONFIG_PPC64) 88 | struct list_head ppc_tce_bypass_list; 89 | struct list_head nvlink_sysmem_trained_list; 90 | #endif 91 | struct list_head enabled_devices; 92 | wait_queue_head_t interrupt_event; 93 | struct irq_q_info irq_queue; 94 | spinlock_t irq_lock; 95 | struct workqueue_struct *work_queue; 96 | struct mem_type mem_type; 97 | #if defined(CONFIG_PCI) 98 | struct pci_dev *cached_dev; 99 | #endif 100 | struct mutex mtx; 101 | int mods_fb_suspended[FB_MAX]; 102 | u32 access_token; 103 | atomic_t num_allocs; 104 | atomic_t num_pages; 105 | #if defined(MODS_HAS_CONSOLE_LOCK) 106 | atomic_t console_is_locked; 107 | #endif 108 | atomic_t last_bad_dbdf; 109 | u8 client_id; 110 | }; 111 | 112 | /* Free WC or UC chunk, which can be reused */ 113 | struct MODS_FREE_PHYS_CHUNK { 114 | struct list_head list; 115 | struct page *p_page; 116 | int numa_node; 117 | u8 order; 118 | u8 cache_type : 2; 119 | u8 dma32 : 1; 120 | }; 121 | 122 | /* DMA mapping into a PCI or non-PCI device */ 123 | struct MODS_DMA_MAP { 124 | struct list_head list; 125 | struct pci_dev *pcidev; /* PCI device these mappings are for. 126 | * Can be NULL if the device the memory 127 | * was mapped to is not a PCI device. 128 | */ 129 | struct device *dev; /* device these mappings are for */ 130 | struct scatterlist sg[]; /* each entry corresponds to phys chunk 131 | * in sg array in MODS_MEM_INFO at the 132 | * same index 133 | */ 134 | }; 135 | 136 | /* system memory allocation tracking */ 137 | struct MODS_MEM_INFO { 138 | struct list_head list; 139 | 140 | /* List of DMA mappings for devices other than the default 141 | * PCI device specified by the dev field. 142 | */ 143 | struct list_head dma_map_list; 144 | 145 | u64 reservation_tag; /* zero if not reserved */ 146 | u32 num_pages; /* total number of allocated pages */ 147 | u32 num_chunks; /* number of allocated contig chunks */ 148 | int numa_node; /* numa node for the allocation */ 149 | u8 cache_type : 2; /* MODS_ALLOC_* */ 150 | u8 dma32 : 1; /* true/false */ 151 | u8 force_numa : 1; /* true/false */ 152 | u8 no_free_opt : 1; /* true/false */ 153 | u8 dma_pages : 1; /* true/false */ 154 | u8 decrypted_mmap : 1; /* true/false */ 155 | u8 large_aux : 1; /* true/false */ 156 | 157 | struct pci_dev *dev; /* (optional) pci_dev this allocation is for. */ 158 | unsigned long *wc_bitmap; /* marks which chunks use WC/UC */ 159 | struct scatterlist *sg; /* current list of chunks */ 160 | struct scatterlist contig_sg; /* contiguous merged chunk */ 161 | struct scatterlist alloc_sg[]; /* allocated memory chunks, each chunk 162 | * consists of 2^n contiguous pages 163 | */ 164 | }; 165 | 166 | /* Size for MODS_MEM_INFO struct allocation which requires vzalloc() */ 167 | #define MODS_LARGE_AUX_ALLOC_SIZE 0x10000U 168 | 169 | static inline u32 get_num_chunks(const struct MODS_MEM_INFO *p_mem_info) 170 | { 171 | if (unlikely(p_mem_info->sg == &p_mem_info->contig_sg)) 172 | return 1; 173 | 174 | return p_mem_info->num_chunks; 175 | } 176 | 177 | /* map memory tracking */ 178 | struct SYS_MAP_MEMORY { 179 | struct list_head list; 180 | 181 | /* used for offset lookup, NULL for device memory */ 182 | struct MODS_MEM_INFO *p_mem_info; 183 | 184 | struct mods_client *client; 185 | atomic_t usage_count; 186 | phys_addr_t phys_addr; 187 | unsigned long virtual_addr; 188 | unsigned long mapping_offs; /* mapped offset from the beginning of the allocation */ 189 | unsigned long mapping_length; /* how many bytes were mapped */ 190 | }; 191 | 192 | struct mods_smmu_dev { 193 | struct device *dev; 194 | #ifdef MODS_ENABLE_BPMP_MRQ_API 195 | struct tegra_bpmp *bpmp; //bpmp node for mrq 196 | int cid; //pcie ctrl id 197 | #endif 198 | char dev_name[MAX_DT_SIZE]; 199 | }; 200 | 201 | /* functions used to avoid global debug variables */ 202 | void mods_set_debug_level(int mask); 203 | int mods_get_debug_level(void); 204 | int mods_check_debug_level(int mask); 205 | int mods_get_multi_instance(void); 206 | void mods_set_multi_instance(int mi); 207 | u32 mods_get_access_token(void); 208 | 209 | #if defined(CONFIG_PPC64) 210 | void mods_set_ppc_tce_bypass(int bypass); 211 | int mods_get_ppc_tce_bypass(void); 212 | 213 | /* PPC TCE bypass tracking */ 214 | struct PPC_TCE_BYPASS { 215 | struct pci_dev *dev; 216 | u64 dma_mask; 217 | struct list_head list; 218 | }; 219 | 220 | int has_npu_dev(struct pci_dev *dev, int index); 221 | 222 | int mods_is_nvlink_sysmem_trained(struct mods_client *client, 223 | struct pci_dev *dev); 224 | 225 | /* NvLink Trained tracking */ 226 | struct NVL_TRAINED { 227 | struct pci_dev *dev; 228 | u8 trained; 229 | struct list_head list; 230 | }; 231 | #endif 232 | 233 | #define IRQ_MAX (256+PCI_IRQ_MAX) 234 | #define PCI_IRQ_MAX 15 235 | #define MODS_MAX_CLIENTS 32 236 | 237 | #define IRQ_VAL_POISON 0xfafbfcfdU 238 | 239 | #define INVALID_CLIENT_ID 0 240 | 241 | /* debug print masks */ 242 | #define DEBUG_IOCTL 0x2 243 | #define DEBUG_PCI 0x4 244 | #define DEBUG_ACPI 0x8 245 | #define DEBUG_ISR 0x10 246 | #define DEBUG_MEM 0x20 247 | #define DEBUG_FUNC 0x40 248 | #define DEBUG_CLOCK 0x80 249 | #define DEBUG_DETAILED 0x100 250 | #define DEBUG_TEGRADMA 0x400 251 | #define DEBUG_ISR_DETAILED (DEBUG_ISR | DEBUG_DETAILED) 252 | #define DEBUG_MEM_DETAILED (DEBUG_MEM | DEBUG_DETAILED) 253 | #define DEBUG_ALL (DEBUG_IOCTL | DEBUG_PCI | DEBUG_ACPI | \ 254 | DEBUG_ISR | DEBUG_MEM | DEBUG_FUNC | DEBUG_CLOCK | DEBUG_DETAILED | \ 255 | DEBUG_TEGRADMA) 256 | 257 | #define LOG_ENT() mods_debug_printk(DEBUG_FUNC, "> %s\n", __func__) 258 | #define LOG_EXT() mods_debug_printk(DEBUG_FUNC, "< %s\n", __func__) 259 | 260 | #define mods_debug_printk(level, fmt, args...)\ 261 | ({ \ 262 | if (mods_check_debug_level(level)) \ 263 | pr_info("mods debug: " fmt, ##args); \ 264 | }) 265 | 266 | #define cl_debug(level, fmt, args...)\ 267 | ({ \ 268 | if (mods_check_debug_level(level)) \ 269 | pr_info("mods [%u] debug: " fmt, client->client_id, \ 270 | ##args); \ 271 | }) 272 | 273 | #define mods_info_printk(fmt, args...)\ 274 | pr_info("mods: " fmt, ##args) 275 | 276 | #define cl_info(fmt, args...)\ 277 | pr_info("mods [%u]: " fmt, client->client_id, ##args) 278 | 279 | #define mods_error_printk(fmt, args...)\ 280 | pr_err("mods error: " fmt, ##args) 281 | 282 | #define cl_error(fmt, args...)\ 283 | pr_err("mods [%u] error: " fmt, client->client_id, ##args) 284 | 285 | #define mods_warning_printk(fmt, args...)\ 286 | pr_notice("mods warning: " fmt, ##args) 287 | 288 | #define cl_warn(fmt, args...)\ 289 | pr_notice("mods [%u] warning: " fmt, client->client_id, ##args) 290 | 291 | #define is_valid_client_id(client_id)\ 292 | ((client_id) != INVALID_CLIENT_ID) 293 | 294 | struct irq_mask_info { 295 | void __iomem *dev_irq_mask_reg; /*IRQ mask register, read-only reg*/ 296 | void __iomem *dev_irq_state; /* IRQ status register*/ 297 | void __iomem *dev_irq_disable_reg; /* potentionally a write-only reg*/ 298 | u64 irq_and_mask; 299 | u64 irq_or_mask; 300 | u8 mask_type; 301 | }; 302 | 303 | struct dev_irq_map { 304 | u8 __iomem *dev_irq_aperture; 305 | u32 apic_irq; 306 | u32 entry; 307 | u8 type; 308 | u8 client_id; 309 | u8 mask_info_cnt; 310 | struct irq_mask_info mask_info[MODS_IRQ_MAX_MASKS]; 311 | struct pci_dev *dev; 312 | struct list_head list; 313 | }; 314 | 315 | struct mods_priv { 316 | /* Bitmap for each allocated client id. */ 317 | unsigned long client_flags; 318 | 319 | /* Client structures */ 320 | struct mods_client clients[MODS_MAX_CLIENTS]; 321 | }; 322 | 323 | #ifdef MODS_HAS_POLL_T 324 | # define POLL_TYPE __poll_t 325 | #else 326 | # define POLL_TYPE unsigned int 327 | #endif 328 | 329 | #if ((defined(CONFIG_ARM) || defined(CONFIG_ARM64)) && \ 330 | !defined(CONFIG_CPA)) || defined(CONFIG_PPC64) 331 | # define MODS_SET_MEMORY_UC(addr, pages) 0 332 | # define MODS_SET_MEMORY_WC(addr, pages) 0 333 | # define MODS_SET_MEMORY_WB(addr, pages) 0 334 | #else 335 | # define MODS_SET_MEMORY_UC(addr, pages) set_memory_uc(addr, pages) 336 | # define MODS_SET_MEMORY_WC(addr, pages) set_memory_wc(addr, pages) 337 | # define MODS_SET_MEMORY_WB(addr, pages) set_memory_wb(addr, pages) 338 | #endif 339 | 340 | #define MODS_PGPROT_UC pgprot_noncached 341 | #define MODS_PGPROT_WC pgprot_writecombine 342 | 343 | /* ACPI */ 344 | #ifdef MODS_HAS_NEW_ACPI_WALK 345 | #define MODS_ACPI_WALK_NAMESPACE(type, start_object, max_depth, user_function, \ 346 | context, return_value)\ 347 | acpi_walk_namespace(type, start_object, max_depth, user_function, NULL,\ 348 | context, return_value) 349 | #else 350 | #define MODS_ACPI_WALK_NAMESPACE acpi_walk_namespace 351 | #endif 352 | #ifdef MODS_HAS_NEW_ACPI_HANDLE 353 | #define MODS_ACPI_HANDLE(dev) ACPI_HANDLE(dev) 354 | #else 355 | #define MODS_ACPI_HANDLE(dev) DEVICE_ACPI_HANDLE(dev) 356 | #endif 357 | 358 | #if KERNEL_VERSION(3, 10, 0) <= MODS_KERNEL_VERSION 359 | # define MODS_SG_UNMARK_END(sg) sg_unmark_end(sg) 360 | #else 361 | # define MODS_SG_UNMARK_END(sg) ({(sg)->page_link &= ~2; }) 362 | #endif 363 | 364 | #if KERNEL_VERSION(5, 11, 0) <= MODS_KERNEL_VERSION 365 | # define MODS_KMAP kmap_local_page 366 | # define MODS_KUNMAP kunmap_local 367 | #else 368 | # define MODS_KMAP kmap 369 | # define MODS_KUNMAP kunmap 370 | #endif 371 | 372 | /* ************************************************************************* */ 373 | /* ** MODULE WIDE FUNCTIONS */ 374 | /* ************************************************************************* */ 375 | 376 | /* client */ 377 | struct mods_client *mods_client_from_id(u8 client_id); 378 | int mods_is_client_enabled(u8 client_id); 379 | 380 | /* irq */ 381 | void mods_init_irq(void); 382 | struct mutex *mods_get_irq_mutex(void); 383 | void mods_free_client_interrupts(struct mods_client *client); 384 | POLL_TYPE mods_irq_event_check(u8 client_id); 385 | 386 | /* mem */ 387 | const char *mods_get_prot_str(u8 mem_type); 388 | int mods_unregister_all_alloc(struct mods_client *client); 389 | struct MODS_MEM_INFO *mods_find_alloc(struct mods_client *client, 390 | u64 phys_addr); 391 | void mods_free_mem_reservations(void); 392 | 393 | #if defined(CONFIG_PPC64) 394 | /* ppc64 */ 395 | int mods_unregister_all_ppc_tce_bypass(struct mods_client *client); 396 | 397 | int mods_unregister_all_nvlink_sysmem_trained(struct mods_client *client); 398 | #endif 399 | 400 | /* pci */ 401 | #ifdef CONFIG_PCI 402 | int mods_enable_device(struct mods_client *client, 403 | struct pci_dev *dev, 404 | struct en_dev_entry **dev_entry); 405 | void mods_disable_device(struct mods_client *client, 406 | struct pci_dev *pdev); 407 | #endif 408 | 409 | #ifdef CONFIG_PCI 410 | int mods_is_pci_dev(struct pci_dev *dev, 411 | struct mods_pci_dev_2 *pcidev); 412 | int mods_find_pci_dev(struct mods_client *client, 413 | struct mods_pci_dev_2 *pcidev, 414 | struct pci_dev **retdev); 415 | #else 416 | #define mods_is_pci_dev(a, b) 0 417 | #define mods_find_pci_dev(a, b, c) (-ENODEV) 418 | #endif 419 | 420 | /* clock */ 421 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_COMMON_CLK) 422 | void mods_init_clock_api(void); 423 | void mods_shutdown_clock_api(void); 424 | #endif 425 | 426 | /* ioctl hanndlers */ 427 | 428 | /* mem */ 429 | int esc_mods_alloc_pages(struct mods_client *client, 430 | struct MODS_ALLOC_PAGES *p); 431 | int esc_mods_device_alloc_pages(struct mods_client *client, 432 | struct MODS_DEVICE_ALLOC_PAGES *p); 433 | int esc_mods_device_alloc_pages_2(struct mods_client *client, 434 | struct MODS_DEVICE_ALLOC_PAGES_2 *p); 435 | int esc_mods_alloc_pages_2(struct mods_client *client, 436 | struct MODS_ALLOC_PAGES_2 *p); 437 | int esc_mods_free_pages(struct mods_client *client, 438 | struct MODS_FREE_PAGES *p); 439 | int esc_mods_set_cache_attr(struct mods_client *client, 440 | struct MODS_SET_CACHE_ATTR *p); 441 | int esc_mods_merge_pages(struct mods_client *client, 442 | struct MODS_MERGE_PAGES *p); 443 | int esc_mods_set_mem_type(struct mods_client *client, 444 | struct MODS_MEMORY_TYPE *p); 445 | int esc_mods_get_phys_addr(struct mods_client *client, 446 | struct MODS_GET_PHYSICAL_ADDRESS *p); 447 | int esc_mods_get_phys_addr_2(struct mods_client *client, 448 | struct MODS_GET_PHYSICAL_ADDRESS_3 *p); 449 | int esc_mods_get_mapped_phys_addr(struct mods_client *client, 450 | struct MODS_GET_PHYSICAL_ADDRESS *p); 451 | int esc_mods_get_mapped_phys_addr_2(struct mods_client *client, 452 | struct MODS_GET_PHYSICAL_ADDRESS_2 *p); 453 | int esc_mods_get_mapped_phys_addr_3(struct mods_client *client, 454 | struct MODS_GET_PHYSICAL_ADDRESS_3 *p); 455 | int esc_mods_virtual_to_phys(struct mods_client *client, 456 | struct MODS_VIRTUAL_TO_PHYSICAL *p); 457 | int esc_mods_phys_to_virtual(struct mods_client *client, 458 | struct MODS_PHYSICAL_TO_VIRTUAL *p); 459 | int esc_mods_dma_map_memory(struct mods_client *client, 460 | struct MODS_DMA_MAP_MEMORY *p); 461 | int esc_mods_dma_unmap_memory(struct mods_client *client, 462 | struct MODS_DMA_MAP_MEMORY *p); 463 | int esc_mods_iommu_dma_map_memory(struct mods_client *client, 464 | struct MODS_IOMMU_DMA_MAP_MEMORY *p); 465 | int esc_mods_iommu_dma_unmap_memory(struct mods_client *client, 466 | struct MODS_IOMMU_DMA_MAP_MEMORY *p); 467 | int esc_mods_reserve_allocation(struct mods_client *client, 468 | struct MODS_RESERVE_ALLOCATION *p); 469 | int esc_mods_get_reserved_allocation(struct mods_client *client, 470 | struct MODS_RESERVE_ALLOCATION *p); 471 | int esc_mods_release_reserved_allocation(struct mods_client *client, 472 | struct MODS_RESERVE_ALLOCATION *p); 473 | 474 | #ifdef CONFIG_ARM 475 | int esc_mods_memory_barrier(struct mods_client *client); 476 | #endif 477 | 478 | #ifdef CONFIG_ARM64 479 | int esc_mods_flush_cpu_cache_range(struct mods_client *client, 480 | struct MODS_FLUSH_CPU_CACHE_RANGE *p); 481 | #endif 482 | 483 | #if defined(CONFIG_PPC64) 484 | /* ppc64 */ 485 | int esc_mods_set_ppc_tce_bypass(struct mods_client *client, 486 | struct MODS_SET_PPC_TCE_BYPASS *p); 487 | int esc_mods_get_ats_address_range(struct mods_client *client, 488 | struct MODS_GET_ATS_ADDRESS_RANGE *p); 489 | int esc_mods_set_nvlink_sysmem_trained(struct mods_client *client, 490 | struct MODS_SET_NVLINK_SYSMEM_TRAINED *p); 491 | int esc_mods_get_nvlink_line_rate(struct mods_client *client, 492 | struct MODS_GET_NVLINK_LINE_RATE *p); 493 | #endif 494 | 495 | /* acpi */ 496 | #ifdef CONFIG_ACPI 497 | int esc_mods_eval_acpi_method(struct mods_client *client, 498 | struct MODS_EVAL_ACPI_METHOD *p); 499 | int esc_mods_eval_dev_acpi_method(struct mods_client *client, 500 | struct MODS_EVAL_DEV_ACPI_METHOD *p); 501 | int esc_mods_eval_dev_acpi_method_2(struct mods_client *client, 502 | struct MODS_EVAL_DEV_ACPI_METHOD_2 *p); 503 | int esc_mods_eval_dev_acpi_method_3(struct mods_client *client, 504 | struct MODS_EVAL_DEV_ACPI_METHOD_3 *p); 505 | int esc_mods_acpi_get_ddc(struct mods_client *client, 506 | struct MODS_ACPI_GET_DDC *p); 507 | int esc_mods_acpi_get_ddc_2(struct mods_client *client, 508 | struct MODS_ACPI_GET_DDC_2 *p); 509 | int esc_mods_get_acpi_dev_children(struct mods_client *client, 510 | struct MODS_GET_ACPI_DEV_CHILDREN *p); 511 | #ifdef MODS_HAS_PXM_TO_NODE 512 | int esc_mods_proximity_to_numa_node(struct mods_client *client, 513 | struct MODS_PROXIMITY_TO_NUMA_NODE *p); 514 | #endif 515 | #endif 516 | /* pci */ 517 | #ifdef CONFIG_PCI 518 | int esc_mods_find_pci_dev(struct mods_client *client, 519 | struct MODS_FIND_PCI_DEVICE *p); 520 | int esc_mods_find_pci_dev_2(struct mods_client *client, 521 | struct MODS_FIND_PCI_DEVICE_2 *p); 522 | int esc_mods_find_pci_class_code(struct mods_client *client, 523 | struct MODS_FIND_PCI_CLASS_CODE *p); 524 | int esc_mods_find_pci_class_code_2(struct mods_client *client, 525 | struct MODS_FIND_PCI_CLASS_CODE_2 *p); 526 | int esc_mods_pci_get_bar_info(struct mods_client *client, 527 | struct MODS_PCI_GET_BAR_INFO *p); 528 | int esc_mods_pci_get_bar_info_2(struct mods_client *client, 529 | struct MODS_PCI_GET_BAR_INFO_2 *p); 530 | int esc_mods_pci_get_irq(struct mods_client *client, 531 | struct MODS_PCI_GET_IRQ *p); 532 | int esc_mods_pci_get_irq_2(struct mods_client *client, 533 | struct MODS_PCI_GET_IRQ_2 *p); 534 | int esc_mods_pci_read(struct mods_client *client, struct MODS_PCI_READ *p); 535 | int esc_mods_pci_read_2(struct mods_client *client, struct MODS_PCI_READ_2 *p); 536 | int esc_mods_pci_write(struct mods_client *client, struct MODS_PCI_WRITE *p); 537 | int esc_mods_pci_write_2(struct mods_client *client, 538 | struct MODS_PCI_WRITE_2 *p); 539 | int esc_mods_pci_bus_rescan(struct mods_client *client, 540 | struct MODS_PCI_BUS_RESCAN *p); 541 | int esc_mods_pci_bus_add_dev(struct mods_client *client, 542 | struct MODS_PCI_BUS_ADD_DEVICES *p); 543 | int esc_mods_pci_bus_remove_dev(struct mods_client *client, 544 | struct MODS_PCI_BUS_REMOVE_DEV *p); 545 | int esc_mods_pci_hot_reset(struct mods_client *client, 546 | struct MODS_PCI_HOT_RESET *p); 547 | int esc_mods_pio_read(struct mods_client *client, struct MODS_PIO_READ *p); 548 | int esc_mods_pio_write(struct mods_client *client, struct MODS_PIO_WRITE *p); 549 | int esc_mods_device_numa_info(struct mods_client *client, 550 | struct MODS_DEVICE_NUMA_INFO *p); 551 | int esc_mods_device_numa_info_2(struct mods_client *client, 552 | struct MODS_DEVICE_NUMA_INFO_2 *p); 553 | int esc_mods_device_numa_info_3(struct mods_client *client, 554 | struct MODS_DEVICE_NUMA_INFO_3 *p); 555 | int esc_mods_get_iommu_state(struct mods_client *client, 556 | struct MODS_GET_IOMMU_STATE *state); 557 | int esc_mods_get_iommu_state_2(struct mods_client *client, 558 | struct MODS_GET_IOMMU_STATE *state); 559 | int esc_mods_pci_set_dma_mask(struct mods_client *client, 560 | struct MODS_PCI_DMA_MASK *dma_mask); 561 | int esc_mods_pci_reset_function(struct mods_client *client, 562 | struct mods_pci_dev_2 *pcidev); 563 | #ifdef MODS_HAS_DEV_PROPS 564 | int esc_mods_read_dev_property(struct mods_client *client, 565 | struct MODS_READ_DEV_PROPERTY *p); 566 | #endif 567 | #endif 568 | /* irq */ 569 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) 570 | int esc_mods_map_irq(struct mods_client *client, struct MODS_DT_INFO *p); 571 | int esc_mods_map_irq_to_gpio(struct mods_client *client, 572 | struct MODS_GPIO_INFO *p); 573 | #endif 574 | int esc_mods_register_irq(struct mods_client *client, 575 | struct MODS_REGISTER_IRQ *p); 576 | int esc_mods_register_irq_2(struct mods_client *client, 577 | struct MODS_REGISTER_IRQ_2 *p); 578 | int esc_mods_register_irq_3(struct mods_client *client, 579 | struct MODS_REGISTER_IRQ_3 *p); 580 | int esc_mods_unregister_irq(struct mods_client *client, 581 | struct MODS_REGISTER_IRQ *p); 582 | int esc_mods_unregister_irq_2(struct mods_client *client, 583 | struct MODS_REGISTER_IRQ_2 *p); 584 | int esc_mods_query_irq(struct mods_client *client, struct MODS_QUERY_IRQ *p); 585 | int esc_mods_query_irq_2(struct mods_client *client, 586 | struct MODS_QUERY_IRQ_2 *p); 587 | int esc_mods_irq_handled(struct mods_client *client, 588 | struct MODS_REGISTER_IRQ *p); 589 | int esc_mods_irq_handled_2(struct mods_client *client, 590 | struct MODS_REGISTER_IRQ_2 *p); 591 | 592 | int esc_mods_register_irq_4(struct mods_client *client, 593 | struct MODS_REGISTER_IRQ_4 *p); 594 | int esc_mods_query_irq_3(struct mods_client *client, 595 | struct MODS_QUERY_IRQ_3 *p); 596 | 597 | #ifdef MODS_HAS_TEGRA 598 | /* bpmp uphy */ 599 | int esc_mods_bpmp_set_pcie_state(struct mods_client *client, 600 | struct MODS_SET_PCIE_STATE *p); 601 | int esc_mods_bpmp_init_pcie_ep_pll(struct mods_client *client, 602 | struct MODS_INIT_PCIE_EP_PLL *p); 603 | 604 | /* clock */ 605 | int esc_mods_get_clock_handle(struct mods_client *client, 606 | struct MODS_GET_CLOCK_HANDLE *p); 607 | int esc_mods_set_clock_rate(struct mods_client *client, 608 | struct MODS_CLOCK_RATE *p); 609 | int esc_mods_get_clock_rate(struct mods_client *client, 610 | struct MODS_CLOCK_RATE *p); 611 | int esc_mods_get_clock_max_rate(struct mods_client *client, 612 | struct MODS_CLOCK_RATE *p); 613 | int esc_mods_set_clock_max_rate(struct mods_client *client, 614 | struct MODS_CLOCK_RATE *p); 615 | int esc_mods_set_clock_parent(struct mods_client *client, 616 | struct MODS_CLOCK_PARENT *p); 617 | int esc_mods_get_clock_parent(struct mods_client *client, 618 | struct MODS_CLOCK_PARENT *p); 619 | int esc_mods_enable_clock(struct mods_client *client, 620 | struct MODS_CLOCK_HANDLE *p); 621 | int esc_mods_disable_clock(struct mods_client *client, 622 | struct MODS_CLOCK_HANDLE *p); 623 | int esc_mods_is_clock_enabled(struct mods_client *client, 624 | struct MODS_CLOCK_ENABLED *p); 625 | int esc_mods_clock_reset_assert(struct mods_client *client, 626 | struct MODS_CLOCK_HANDLE *p); 627 | int esc_mods_clock_reset_deassert(struct mods_client *client, 628 | struct MODS_CLOCK_HANDLE *p); 629 | int esc_mods_reset_assert(struct mods_client *client, 630 | struct MODS_RESET_HANDLE *p); 631 | int esc_mods_get_rst_handle(struct mods_client *client, 632 | struct MODS_GET_RESET_HANDLE *p); 633 | int esc_mods_dma_alloc_coherent(struct mods_client *client, 634 | struct MODS_DMA_COHERENT_MEM_HANDLE *p); 635 | int esc_mods_dma_free_coherent(struct mods_client *client, 636 | struct MODS_DMA_COHERENT_MEM_HANDLE *p); 637 | int esc_mods_dma_copy_to_user(struct mods_client *client, 638 | struct MODS_DMA_COPY_TO_USER *p); 639 | 640 | /* oist */ 641 | int esc_mods_oist_status(struct mods_client *client, 642 | struct MODS_TEGRA_OIST_STATUS *p); 643 | 644 | #ifdef CONFIG_DMA_ENGINE 645 | int mods_init_dma(void); 646 | void mods_exit_dma(void); 647 | int esc_mods_dma_request_channel(struct mods_client *client, 648 | struct MODS_DMA_HANDLE *p); 649 | int esc_mods_dma_request_channel_2(struct mods_client *client, 650 | struct MODS_DMA_HANDLE_2 *p_handle_2); 651 | int esc_mods_dma_release_channel(struct mods_client *client, 652 | struct MODS_DMA_HANDLE *p); 653 | int esc_mods_dma_set_config(struct mods_client *client, 654 | struct MODS_DMA_CHANNEL_CONFIG *p); 655 | int esc_mods_dma_wait(struct mods_client *client, struct MODS_DMA_WAIT_DESC *p); 656 | int esc_mods_dma_submit_request(struct mods_client *client, 657 | struct MODS_DMA_TX_DESC *p); 658 | int esc_mods_dma_async_issue_pending(struct mods_client *client, 659 | struct MODS_DMA_HANDLE *p); 660 | #endif 661 | 662 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_NET) 663 | int esc_mods_net_force_link(struct mods_client *client, 664 | struct MODS_NET_DEVICE_NAME *p); 665 | #endif 666 | 667 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_DMA_SHARED_BUFFER) 668 | int esc_mods_dmabuf_get_phys_addr(struct mods_client *client, 669 | struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *p); 670 | #else 671 | static inline int esc_mods_dmabuf_get_phys_addr(struct mods_client *client, 672 | struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *p) 673 | { return -EINVAL; } 674 | #endif 675 | 676 | #ifdef MODS_HAS_TEGRA 677 | int esc_mods_adsp_load(struct mods_client *client, 678 | struct MODS_ADSP_INIT_INFO *p); 679 | int esc_mods_adsp_start(struct mods_client *client, 680 | struct MODS_ADSP_INIT_INFO *p); 681 | int esc_mods_adsp_stop(struct mods_client *client, 682 | struct MODS_ADSP_INIT_INFO *p); 683 | int esc_mods_adsp_run_app(struct mods_client *client, 684 | struct MODS_ADSP_RUN_APP_INFO *p); 685 | #endif 686 | 687 | #ifdef CONFIG_TRUSTY 688 | /* trustzone app call */ 689 | int esc_mods_send_trustzone_msg(struct mods_client *client, 690 | struct MODS_TZ_PARAMS *p); 691 | #endif 692 | 693 | #if IS_ENABLED(CONFIG_OPTEE) 694 | /* OP-TEE TA call */ 695 | int esc_mods_invoke_optee_ta(struct mods_client *client, 696 | struct MODS_OPTEE_PARAMS *p); 697 | #endif 698 | #endif 699 | 700 | /* MODS SP call */ 701 | #if defined(MODS_HAS_ARM_FFA) 702 | int esc_mods_arm_ffa_cmd(struct mods_client *client, 703 | struct MODS_FFA_PARAMS *p); 704 | #endif 705 | 706 | #ifdef CONFIG_DEBUG_FS 707 | int mods_create_debugfs(struct miscdevice *modsdev); 708 | void mods_remove_debugfs(void); 709 | #else 710 | static inline int mods_create_debugfs(struct miscdevice *modsdev) 711 | { 712 | return 0; 713 | } 714 | static inline void mods_remove_debugfs(void) {} 715 | #endif /* CONFIG_DEBUG_FS */ 716 | 717 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_DMA_SHARED_BUFFER) 718 | int mods_init_dmabuf(void); 719 | void mods_exit_dmabuf(void); 720 | #else 721 | static inline int mods_init_dmabuf(void) { return 0; } 722 | static inline void mods_exit_dmabuf(void) {} 723 | #endif 724 | 725 | #if defined(MODS_HAS_TEGRA) 726 | int get_mods_smmu_device_index(const char *name); 727 | struct mods_smmu_dev *get_mods_smmu_device(u32 index); 728 | int smmu_driver_init(void); 729 | void smmu_driver_exit(void); 730 | #endif 731 | 732 | #if defined(MODS_HAS_TEGRA) 733 | int esc_mods_send_ipi(struct mods_client *client, struct MODS_SEND_IPI *p); 734 | #endif 735 | 736 | #if defined(CONFIG_TEGRA_IVC) 737 | int esc_mods_bpmp_uphy_lane_eom_scan(struct mods_client *client, 738 | struct MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS *p); 739 | int mods_bpmpipc_init(struct mods_client *client, 740 | u64 db_phys_addr, 741 | u64 req_phys_addr, 742 | u64 resp_phys_addr); 743 | void mods_bpmpipc_cleanup(void); 744 | #endif 745 | 746 | #if defined(MODS_HAS_ARM_FFA) 747 | int mods_ffa_abi_register(void); 748 | void mods_ffa_abi_unregister(void); 749 | #endif 750 | 751 | #endif /* _MODS_INTERNAL_H_ */ 752 | -------------------------------------------------------------------------------- /mods_irq.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2008-2024, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #endif 18 | 19 | #define PCI_VENDOR_ID_NVIDIA 0x10de 20 | #define INDEX_IRQSTAT(irq) (irq / BITS_NUM) 21 | #define POS_IRQSTAT(irq) (irq & (BITS_NUM - 1)) 22 | 23 | /* MSI */ 24 | #define PCI_MSI_MASK_BIT 16 25 | #define MSI_CONTROL_REG(base) (base + PCI_MSI_FLAGS) 26 | #define IS_64BIT_ADDRESS(control) (!!(control & PCI_MSI_FLAGS_64BIT)) 27 | #define MSI_DATA_REG(base, is64bit) \ 28 | ((is64bit == 1) ? base + PCI_MSI_DATA_64 : base + PCI_MSI_DATA_32) 29 | #define TOP_TKE_TKEIE_WDT_MASK(i) (1 << (16 + 4 * (i))) 30 | #define TOP_TKE_TKEIE(i) (0x100 + 4 * (i)) 31 | 32 | /********************* 33 | * PRIVATE FUNCTIONS * 34 | *********************/ 35 | 36 | /* Mutex for guarding interrupt logic and PCI device enablement */ 37 | static struct mutex irq_mtx; 38 | 39 | struct mutex *mods_get_irq_mutex(void) 40 | { 41 | return &irq_mtx; 42 | } 43 | 44 | #ifdef CONFIG_PCI 45 | int mods_enable_device(struct mods_client *client, 46 | struct pci_dev *dev, 47 | struct en_dev_entry **dev_entry) 48 | { 49 | int err = OK; 50 | struct en_dev_entry *dpriv = NULL; 51 | 52 | WARN_ON(!mutex_is_locked(&irq_mtx)); 53 | 54 | dpriv = pci_get_drvdata(dev); 55 | if (!dpriv) { 56 | cl_error( 57 | "driver data is not set for %04x:%02x:%02x.%x\n", 58 | pci_domain_nr(dev->bus), 59 | dev->bus->number, 60 | PCI_SLOT(dev->devfn), 61 | PCI_FUNC(dev->devfn)); 62 | return -EINVAL; 63 | } 64 | 65 | 66 | /* Client already owns the device */ 67 | if (dpriv->client_id == client->client_id) { 68 | if (dev_entry) 69 | *dev_entry = dpriv; 70 | return OK; 71 | } 72 | 73 | /* Another client owns the device */ 74 | if (is_valid_client_id(dpriv->client_id)) { 75 | cl_error( 76 | "invalid client for dev %04x:%02x:%02x.%x, expected %u\n", 77 | pci_domain_nr(dev->bus), 78 | dev->bus->number, 79 | PCI_SLOT(dev->devfn), 80 | PCI_FUNC(dev->devfn), 81 | dpriv->client_id); 82 | return -EBUSY; 83 | } 84 | 85 | err = pci_enable_device(dev); 86 | if (unlikely(err)) { 87 | cl_error("failed to enable dev %04x:%02x:%02x.%x\n", 88 | pci_domain_nr(dev->bus), 89 | dev->bus->number, 90 | PCI_SLOT(dev->devfn), 91 | PCI_FUNC(dev->devfn)); 92 | return err; 93 | } 94 | 95 | cl_info("enabled dev %04x:%02x:%02x.%x\n", 96 | pci_domain_nr(dev->bus), 97 | dev->bus->number, 98 | PCI_SLOT(dev->devfn), 99 | PCI_FUNC(dev->devfn)); 100 | 101 | dpriv->client_id = client->client_id; 102 | list_add(&dpriv->list, &client->enabled_devices); 103 | #ifdef MODS_HAS_REINIT_COMPLETION 104 | reinit_completion(&dpriv->client_completion); 105 | #else 106 | INIT_COMPLETION(dpriv->client_completion); 107 | #endif 108 | 109 | if (dev_entry) 110 | *dev_entry = dpriv; 111 | return OK; 112 | } 113 | 114 | void mods_disable_device(struct mods_client *client, 115 | struct pci_dev *dev) 116 | { 117 | struct en_dev_entry *dpriv = pci_get_drvdata(dev); 118 | 119 | WARN_ON(!mutex_is_locked(&irq_mtx)); 120 | 121 | pci_disable_device(dev); 122 | 123 | if (dpriv) { 124 | dpriv->client_id = INVALID_CLIENT_ID; 125 | complete(&dpriv->client_completion); 126 | } 127 | 128 | cl_info("disabled dev %04x:%02x:%02x.%x\n", 129 | pci_domain_nr(dev->bus), 130 | dev->bus->number, 131 | PCI_SLOT(dev->devfn), 132 | PCI_FUNC(dev->devfn)); 133 | } 134 | #endif 135 | 136 | static unsigned int get_cur_time(void) 137 | { 138 | /* This is not very precise, sched_clock() would be better */ 139 | return jiffies_to_usecs(jiffies); 140 | } 141 | 142 | static u64 irq_reg_read(const struct irq_mask_info *m, void __iomem *reg) 143 | { 144 | if (m->mask_type == MODS_MASK_TYPE_IRQ_DISABLE64) 145 | return readq(reg); 146 | else 147 | return readl(reg); 148 | } 149 | 150 | static void irq_reg_write(const struct irq_mask_info *m, 151 | u64 value, 152 | void __iomem *reg) 153 | { 154 | if (m->mask_type == MODS_MASK_TYPE_IRQ_DISABLE64) 155 | writeq(value, reg); 156 | else { 157 | const u32 val32 = (value > ~0U) ? 0 : value; 158 | 159 | writel(val32, reg); 160 | } 161 | } 162 | 163 | static u64 read_irq_state(const struct irq_mask_info *m) 164 | { 165 | return irq_reg_read(m, m->dev_irq_state); 166 | } 167 | 168 | static u64 read_irq_mask(const struct irq_mask_info *m) 169 | { 170 | return irq_reg_read(m, m->dev_irq_mask_reg); 171 | } 172 | 173 | static void write_irq_disable(u64 value, const struct irq_mask_info *m) 174 | { 175 | irq_reg_write(m, value, m->dev_irq_disable_reg); 176 | } 177 | 178 | static int mods_check_interrupt(struct dev_irq_map *t) 179 | { 180 | int ii = 0; 181 | int valid = 0; 182 | 183 | /* For MSI - we always treat it as pending (must rearm later). */ 184 | /* For non-GPU devices - we can't tell. */ 185 | if (t->mask_info_cnt == 0) 186 | return true; 187 | 188 | for (ii = 0; ii < t->mask_info_cnt; ii++) { 189 | const struct irq_mask_info *const m = &t->mask_info[ii]; 190 | 191 | if (!m->dev_irq_state || !m->dev_irq_mask_reg) 192 | continue; 193 | 194 | /* GPU device */ 195 | valid |= (read_irq_state(m) && read_irq_mask(m)) != 0; 196 | } 197 | 198 | return valid; 199 | } 200 | 201 | static void mods_disable_interrupts(struct dev_irq_map *t) 202 | { 203 | u32 ii = 0; 204 | 205 | for (ii = 0; ii < t->mask_info_cnt; ii++) { 206 | const struct irq_mask_info *const m = &t->mask_info[ii]; 207 | u64 cur_mask = 0; 208 | 209 | if (!m->dev_irq_disable_reg) 210 | continue; 211 | 212 | if (m->irq_and_mask == 0) { 213 | write_irq_disable(m->irq_or_mask, m); 214 | continue; 215 | } 216 | 217 | cur_mask = read_irq_mask(m); 218 | cur_mask &= m->irq_and_mask; 219 | cur_mask |= m->irq_or_mask; 220 | write_irq_disable(cur_mask, m); 221 | } 222 | 223 | if ((ii == 0) && t->type == MODS_IRQ_TYPE_CPU) { 224 | mods_debug_printk(DEBUG_ISR, 225 | "disable_irq_nosync %u", 226 | t->apic_irq); 227 | disable_irq_nosync(t->apic_irq); 228 | } 229 | } 230 | 231 | #ifdef CONFIG_PCI 232 | static const char *mods_irq_type_name(u8 irq_type) 233 | { 234 | switch (irq_type) { 235 | case MODS_IRQ_TYPE_INT: 236 | return "INTx"; 237 | case MODS_IRQ_TYPE_MSI: 238 | return "MSI"; 239 | case MODS_IRQ_TYPE_CPU: 240 | return "CPU"; 241 | case MODS_IRQ_TYPE_MSIX: 242 | return "MSI-X"; 243 | default: 244 | return "unknown"; 245 | } 246 | } 247 | #endif 248 | 249 | static void wake_up_client(struct dev_irq_map *t) 250 | { 251 | struct mods_client *client = mods_client_from_id(t->client_id); 252 | 253 | if (client) 254 | wake_up_interruptible(&client->interrupt_event); 255 | } 256 | 257 | static int rec_irq_done(struct mods_client *client, 258 | struct dev_irq_map *t, 259 | unsigned int irq_time) 260 | { 261 | /* Get interrupt queue */ 262 | struct irq_q_info *q = &client->irq_queue; 263 | 264 | /* Don't do anything if the IRQ has already been recorded */ 265 | if (q->head != q->tail) { 266 | unsigned int i; 267 | 268 | for (i = q->head; i != q->tail; i++) { 269 | struct irq_q_data *pd 270 | = q->data+(i & (MODS_MAX_IRQS - 1)); 271 | 272 | if ((pd->irq == t->apic_irq) && 273 | (!t->dev || (pd->dev == t->dev))) 274 | return false; 275 | } 276 | } 277 | 278 | /* Print an error if the queue is full */ 279 | /* This is deadly! */ 280 | if (q->tail - q->head == MODS_MAX_IRQS) { 281 | mods_error_printk("IRQ queue is full\n"); 282 | return false; 283 | } 284 | 285 | /* Record the device which generated the IRQ in the queue */ 286 | q->data[q->tail & (MODS_MAX_IRQS - 1)].dev = t->dev; 287 | q->data[q->tail & (MODS_MAX_IRQS - 1)].irq = t->apic_irq; 288 | q->data[q->tail & (MODS_MAX_IRQS - 1)].irq_index = t->entry; 289 | q->data[q->tail & (MODS_MAX_IRQS - 1)].time = irq_time; 290 | q->tail++; 291 | 292 | #ifdef CONFIG_PCI 293 | if (t->dev) { 294 | mods_debug_printk(DEBUG_ISR_DETAILED, 295 | "dev %04x:%02x:%02x.%x %s IRQ 0x%x time=%uus\n", 296 | pci_domain_nr(t->dev->bus), 297 | t->dev->bus->number, 298 | PCI_SLOT(t->dev->devfn), 299 | PCI_FUNC(t->dev->devfn), 300 | mods_irq_type_name(t->type), 301 | t->apic_irq, 302 | irq_time); 303 | } else 304 | #endif 305 | mods_debug_printk(DEBUG_ISR_DETAILED, 306 | "CPU IRQ 0x%x, time=%uus\n", 307 | t->apic_irq, 308 | irq_time); 309 | 310 | return true; 311 | } 312 | 313 | /* mods_irq_handle - interrupt function */ 314 | static irqreturn_t mods_irq_handle(int irq, void *data) 315 | { 316 | struct dev_irq_map *t = (struct dev_irq_map *)data; 317 | int serviced = false; 318 | 319 | if (irq < 0) 320 | return IRQ_RETVAL(false); 321 | 322 | if (unlikely(!t)) 323 | mods_error_printk("received irq %d, but no context for it\n", 324 | irq); 325 | else if (unlikely(t->apic_irq != irq)) 326 | mods_error_printk("received irq %d which doesn't match registered irq %d\n", 327 | irq, t->apic_irq); 328 | else { 329 | unsigned long flags = 0; 330 | int recorded = false; 331 | unsigned int irq_time = get_cur_time(); 332 | struct mods_client *client = mods_client_from_id(t->client_id); 333 | 334 | spin_lock_irqsave(&client->irq_lock, flags); 335 | 336 | /* Check if the interrupt is still pending (shared INTA) */ 337 | if (mods_check_interrupt(t)) { 338 | 339 | /* Disable interrupts on this device to avoid interrupt 340 | * storm 341 | */ 342 | mods_disable_interrupts(t); 343 | 344 | /* Record IRQ for MODS and wake MODS up */ 345 | recorded = rec_irq_done(client, t, irq_time); 346 | 347 | serviced = true; 348 | } 349 | 350 | spin_unlock_irqrestore(&client->irq_lock, flags); 351 | 352 | if (recorded) 353 | wake_up_client(t); 354 | } 355 | 356 | return IRQ_RETVAL(serviced); 357 | } 358 | 359 | static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq) 360 | { 361 | u8 client_idx; 362 | int ret = IRQ_NOT_FOUND; 363 | 364 | LOG_ENT(); 365 | 366 | for (client_idx = 1; client_idx <= MODS_MAX_CLIENTS; client_idx++) { 367 | struct dev_irq_map *t = NULL; 368 | struct dev_irq_map *next = NULL; 369 | 370 | if (!mods_is_client_enabled(client_idx)) 371 | continue; 372 | 373 | list_for_each_entry_safe(t, 374 | next, 375 | &mods_client_from_id(client_idx)->irq_list, 376 | list) { 377 | 378 | if (t->apic_irq == irq) { 379 | ret = (!client_id || client_id == client_idx) 380 | ? IRQ_FOUND : IRQ_NOT_FOUND; 381 | 382 | /* Break out of the outer loop */ 383 | client_idx = MODS_MAX_CLIENTS; 384 | break; 385 | } 386 | } 387 | } 388 | 389 | LOG_EXT(); 390 | return ret; 391 | } 392 | 393 | #ifdef CONFIG_PCI 394 | static int is_nvidia_gpu(struct pci_dev *dev) 395 | { 396 | unsigned short class_code, vendor_id; 397 | 398 | pci_read_config_word(dev, PCI_CLASS_DEVICE, &class_code); 399 | pci_read_config_word(dev, PCI_VENDOR_ID, &vendor_id); 400 | if (((class_code == PCI_CLASS_DISPLAY_VGA) || 401 | (class_code == PCI_CLASS_DISPLAY_3D)) && (vendor_id == 0x10DE)) { 402 | return true; 403 | } 404 | return false; 405 | } 406 | 407 | static void setup_mask_info(struct dev_irq_map *newmap, 408 | struct MODS_REGISTER_IRQ_4 *p, 409 | struct pci_dev *dev) 410 | { 411 | /* account for legacy adapters */ 412 | u8 __iomem *bar = newmap->dev_irq_aperture; 413 | u32 ii = 0; 414 | 415 | if ((p->mask_info_cnt == 0) && is_nvidia_gpu(dev)) { 416 | struct irq_mask_info *const m = &newmap->mask_info[0]; 417 | 418 | newmap->mask_info_cnt = 1; 419 | m->dev_irq_mask_reg = (void __iomem *)(bar + 0x140); 420 | m->dev_irq_disable_reg = (void __iomem *)(bar + 0x140); 421 | m->dev_irq_state = (void __iomem *)(bar + 0x100); 422 | m->irq_and_mask = 0; 423 | m->irq_or_mask = 0; 424 | return; 425 | } 426 | 427 | /* setup for new adapters */ 428 | newmap->mask_info_cnt = p->mask_info_cnt; 429 | for (ii = 0; ii < p->mask_info_cnt; ii++) { 430 | struct irq_mask_info *const m = &newmap->mask_info[ii]; 431 | const struct mods_mask_info2 *const in_m = &p->mask_info[ii]; 432 | 433 | const u32 pend_offs = in_m->irq_pending_offset; 434 | const u32 stat_offs = in_m->irq_enabled_offset; 435 | const u32 dis_offs = in_m->irq_disable_offset; 436 | 437 | m->dev_irq_state = (void __iomem *)(bar + pend_offs); 438 | m->dev_irq_mask_reg = (void __iomem *)(bar + stat_offs); 439 | m->dev_irq_disable_reg = (void __iomem *)(bar + dis_offs); 440 | m->irq_and_mask = in_m->and_mask; 441 | m->irq_or_mask = in_m->or_mask; 442 | m->mask_type = in_m->mask_type; 443 | } 444 | } 445 | #endif 446 | 447 | static int add_irq_map(struct mods_client *client, 448 | struct pci_dev *dev, 449 | struct MODS_REGISTER_IRQ_4 *p, 450 | u32 irq, 451 | u32 entry) 452 | { 453 | struct dev_irq_map *newmap = NULL; 454 | u64 valid_mask = IRQF_TRIGGER_NONE; 455 | u64 irq_flags = MODS_IRQ_FLAG_FROM_FLAGS(p->irq_flags); 456 | u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); 457 | 458 | LOG_ENT(); 459 | 460 | /* Get the flags based on the interrupt type*/ 461 | switch (irq_type) { 462 | case MODS_IRQ_TYPE_INT: 463 | irq_flags = IRQF_SHARED; 464 | break; 465 | 466 | case MODS_IRQ_TYPE_CPU: 467 | valid_mask = IRQF_TRIGGER_RISING | 468 | IRQF_TRIGGER_FALLING | 469 | IRQF_SHARED; 470 | 471 | /* Either use a valid flag bit or no flags */ 472 | if (irq_flags & ~valid_mask) { 473 | cl_error("invalid device interrupt flag %llx\n", 474 | (long long)irq_flags); 475 | return -EINVAL; 476 | } 477 | break; 478 | 479 | default: 480 | irq_flags = IRQF_TRIGGER_NONE; 481 | break; 482 | } 483 | 484 | /* Allocate memory for the new entry */ 485 | newmap = kzalloc(sizeof(*newmap), GFP_KERNEL | __GFP_NORETRY); 486 | if (unlikely(!newmap)) { 487 | LOG_EXT(); 488 | return -ENOMEM; 489 | } 490 | atomic_inc(&client->num_allocs); 491 | 492 | /* Fill out the new entry */ 493 | newmap->apic_irq = irq; 494 | newmap->dev = dev; 495 | newmap->client_id = client->client_id; 496 | newmap->dev_irq_aperture = NULL; 497 | newmap->mask_info_cnt = 0; 498 | newmap->type = irq_type; 499 | newmap->entry = entry; 500 | 501 | /* Enable IRQ for this device in the kernel */ 502 | if (request_irq(irq, 503 | &mods_irq_handle, 504 | irq_flags, 505 | "nvidia mods", 506 | newmap)) { 507 | cl_error("unable to enable IRQ 0x%x with flags %llx\n", 508 | irq, 509 | (long long)irq_flags); 510 | kfree(newmap); 511 | atomic_dec(&client->num_allocs); 512 | LOG_EXT(); 513 | return -EPERM; 514 | } 515 | 516 | /* Add the new entry to the list of all registered interrupts */ 517 | list_add(&newmap->list, &client->irq_list); 518 | 519 | #ifdef CONFIG_PCI 520 | /* Map BAR0 to be able to disable interrupts */ 521 | if ((irq_type == MODS_IRQ_TYPE_INT) && 522 | (p->aperture_addr != 0) && 523 | (p->aperture_size != 0)) { 524 | u8 __iomem *bar = ioremap(p->aperture_addr, p->aperture_size); 525 | 526 | if (!bar) { 527 | cl_debug(DEBUG_ISR, 528 | "failed to remap aperture: 0x%llx size=0x%x\n", 529 | p->aperture_addr, 530 | p->aperture_size); 531 | LOG_EXT(); 532 | return -EPERM; 533 | } 534 | 535 | newmap->dev_irq_aperture = bar; 536 | setup_mask_info(newmap, p, dev); 537 | } 538 | 539 | if (dev) 540 | pci_dev_get(dev); 541 | #endif 542 | 543 | /* Print out successful registration string */ 544 | if (irq_type == MODS_IRQ_TYPE_CPU) 545 | cl_debug(DEBUG_ISR, 546 | "registered CPU IRQ 0x%x with flags %llx\n", 547 | irq, 548 | (long long)irq_flags); 549 | #ifdef CONFIG_PCI 550 | else if ((irq_type == MODS_IRQ_TYPE_INT) || 551 | (irq_type == MODS_IRQ_TYPE_MSI) || 552 | (irq_type == MODS_IRQ_TYPE_MSIX)) { 553 | int dom = dev ? pci_domain_nr(dev->bus) : 0; 554 | 555 | if (dom < 0) 556 | dom = 0; 557 | 558 | cl_debug(DEBUG_ISR, 559 | "dev %04x:%02x:%02x.%x registered %s IRQ 0x%x\n", 560 | dom, 561 | dev ? dev->bus->number : 0U, 562 | dev ? PCI_SLOT(dev->devfn) : 0U, 563 | dev ? PCI_FUNC(dev->devfn) : 0U, 564 | mods_irq_type_name(irq_type), 565 | irq); 566 | } 567 | #endif 568 | #ifdef CONFIG_PCI_MSI 569 | else if (irq_type == MODS_IRQ_TYPE_MSI) { 570 | u16 control; 571 | u16 data; 572 | int cap_pos = pci_find_capability(dev, PCI_CAP_ID_MSI); 573 | 574 | pci_read_config_word(dev, MSI_CONTROL_REG(cap_pos), &control); 575 | if (IS_64BIT_ADDRESS(control)) 576 | pci_read_config_word(dev, 577 | MSI_DATA_REG(cap_pos, 1), 578 | &data); 579 | else 580 | pci_read_config_word(dev, 581 | MSI_DATA_REG(cap_pos, 0), 582 | &data); 583 | cl_debug(DEBUG_ISR, 584 | "dev %04x:%02x:%02x.%x registered MSI IRQ 0x%x data:0x%02x\n", 585 | pci_domain_nr(dev->bus), 586 | dev->bus->number, 587 | PCI_SLOT(dev->devfn), 588 | PCI_FUNC(dev->devfn), 589 | irq, 590 | data); 591 | } else if (irq_type == MODS_IRQ_TYPE_MSIX) { 592 | cl_debug(DEBUG_ISR, 593 | "dev %04x:%02x:%02x.%x registered MSI-X IRQ 0x%x\n", 594 | pci_domain_nr(dev->bus), 595 | dev->bus->number, 596 | PCI_SLOT(dev->devfn), 597 | PCI_FUNC(dev->devfn), 598 | irq); 599 | } 600 | #endif 601 | 602 | LOG_EXT(); 603 | return OK; 604 | } 605 | 606 | static void mods_free_map(struct mods_client *client, 607 | struct dev_irq_map *del) 608 | { 609 | unsigned long flags = 0; 610 | 611 | LOG_ENT(); 612 | 613 | WARN_ON(client->client_id != del->client_id); 614 | 615 | /* Disable interrupts on the device */ 616 | spin_lock_irqsave(&client->irq_lock, flags); 617 | mods_disable_interrupts(del); 618 | spin_unlock_irqrestore(&client->irq_lock, flags); 619 | 620 | /* Unhook interrupts in the kernel */ 621 | free_irq(del->apic_irq, del); 622 | 623 | /* Unmap aperture used for masking irqs */ 624 | if (del->dev_irq_aperture) 625 | iounmap(del->dev_irq_aperture); 626 | 627 | #ifdef CONFIG_PCI 628 | pci_dev_put(del->dev); 629 | #endif 630 | 631 | /* Free memory */ 632 | kfree(del); 633 | atomic_dec(&client->num_allocs); 634 | 635 | LOG_EXT(); 636 | } 637 | 638 | void mods_init_irq(void) 639 | { 640 | LOG_ENT(); 641 | 642 | mutex_init(&irq_mtx); 643 | 644 | LOG_EXT(); 645 | } 646 | 647 | POLL_TYPE mods_irq_event_check(u8 client_id) 648 | { 649 | struct irq_q_info *q; 650 | 651 | if (!mods_is_client_enabled(client_id)) 652 | return POLLERR; /* client has quit */ 653 | 654 | q = &mods_client_from_id(client_id)->irq_queue; 655 | 656 | if (q->head != q->tail) 657 | return POLLIN; /* irq generated */ 658 | 659 | return 0; 660 | } 661 | 662 | static int mods_free_irqs(struct mods_client *client, 663 | struct pci_dev *dev) 664 | { 665 | #ifdef CONFIG_PCI 666 | struct dev_irq_map *del = NULL; 667 | struct dev_irq_map *next; 668 | struct en_dev_entry *dpriv; 669 | unsigned int irq_type; 670 | 671 | LOG_ENT(); 672 | 673 | if (unlikely(mutex_lock_interruptible(&irq_mtx))) { 674 | LOG_EXT(); 675 | return -EINTR; 676 | } 677 | 678 | dpriv = pci_get_drvdata(dev); 679 | 680 | if (!dpriv) { 681 | cl_error( 682 | "driver data is not set for %04x:%02x:%02x.%x\n", 683 | pci_domain_nr(dev->bus), 684 | dev->bus->number, 685 | PCI_SLOT(dev->devfn), 686 | PCI_FUNC(dev->devfn)); 687 | mutex_unlock(&irq_mtx); 688 | LOG_EXT(); 689 | return -EINVAL; 690 | } 691 | 692 | if (!is_valid_client_id(dpriv->client_id)) { 693 | cl_error( 694 | "no client owns dev %04x:%02x:%02x.%x\n", 695 | pci_domain_nr(dev->bus), 696 | dev->bus->number, 697 | PCI_SLOT(dev->devfn), 698 | PCI_FUNC(dev->devfn)); 699 | mutex_unlock(&irq_mtx); 700 | LOG_EXT(); 701 | return -EINVAL; 702 | } 703 | 704 | if (dpriv->client_id != client->client_id) { 705 | cl_error( 706 | "invalid client for dev %04x:%02x:%02x.%x, expected %u\n", 707 | pci_domain_nr(dev->bus), 708 | dev->bus->number, 709 | PCI_SLOT(dev->devfn), 710 | PCI_FUNC(dev->devfn), 711 | dpriv->client_id); 712 | mutex_unlock(&irq_mtx); 713 | LOG_EXT(); 714 | return -EINVAL; 715 | } 716 | 717 | cl_debug(DEBUG_ISR_DETAILED, 718 | "free IRQ for dev %04x:%02x:%02x.%x irq_flags=0x%x nvecs=%d\n", 719 | pci_domain_nr(dev->bus), 720 | dev->bus->number, 721 | PCI_SLOT(dev->devfn), 722 | PCI_FUNC(dev->devfn), 723 | dpriv->irq_flags, 724 | dpriv->nvecs); 725 | 726 | /* Delete device interrupts from the list */ 727 | list_for_each_entry_safe(del, next, &client->irq_list, list) { 728 | if (dev == del->dev) { 729 | u8 type = del->type; 730 | 731 | list_del(&del->list); 732 | cl_debug(DEBUG_ISR, 733 | "unregistered %s IRQ 0x%x dev %04x:%02x:%02x.%x\n", 734 | mods_irq_type_name(type), 735 | del->apic_irq, 736 | pci_domain_nr(dev->bus), 737 | dev->bus->number, 738 | PCI_SLOT(dev->devfn), 739 | PCI_FUNC(dev->devfn)); 740 | mods_free_map(client, del); 741 | 742 | WARN_ON(MODS_IRQ_TYPE_FROM_FLAGS(dpriv->irq_flags) 743 | != type); 744 | if (type != MODS_IRQ_TYPE_MSIX) 745 | break; 746 | } 747 | } 748 | 749 | cl_debug(DEBUG_ISR_DETAILED, "before disable\n"); 750 | #ifdef CONFIG_PCI_MSI 751 | irq_type = MODS_IRQ_TYPE_FROM_FLAGS(dpriv->irq_flags); 752 | 753 | if (irq_type == MODS_IRQ_TYPE_MSIX) { 754 | pci_disable_msix(dev); 755 | kfree(dpriv->msix_entries); 756 | if (dpriv->msix_entries) 757 | atomic_dec(&client->num_allocs); 758 | dpriv->msix_entries = NULL; 759 | } else if (irq_type == MODS_IRQ_TYPE_MSI) { 760 | pci_disable_msi(dev); 761 | } 762 | #endif 763 | 764 | dpriv->nvecs = 0; 765 | cl_debug(DEBUG_ISR_DETAILED, "irqs freed\n"); 766 | #endif 767 | 768 | mutex_unlock(&irq_mtx); 769 | LOG_EXT(); 770 | return 0; 771 | } 772 | 773 | void mods_free_client_interrupts(struct mods_client *client) 774 | { 775 | struct list_head *head = &client->enabled_devices; 776 | struct list_head *iter; 777 | struct en_dev_entry *dpriv; 778 | 779 | LOG_ENT(); 780 | 781 | /* Release all interrupts */ 782 | list_for_each(iter, head) { 783 | dpriv = list_entry(iter, struct en_dev_entry, list); 784 | mods_free_irqs(client, dpriv->dev); 785 | } 786 | 787 | LOG_EXT(); 788 | } 789 | 790 | #ifdef CONFIG_PCI 791 | static int mods_allocate_irqs(struct mods_client *client, 792 | struct pci_dev *dev, 793 | u32 nvecs, 794 | u32 flags) 795 | { 796 | struct en_dev_entry *dpriv; 797 | unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(flags); 798 | int err = OK; 799 | 800 | LOG_ENT(); 801 | 802 | cl_debug(DEBUG_ISR_DETAILED, 803 | "allocate %u IRQs on dev %04x:%02x:%02x.%x, flags=0x%x\n", 804 | nvecs, 805 | pci_domain_nr(dev->bus), 806 | dev->bus->number, 807 | PCI_SLOT(dev->devfn), 808 | PCI_FUNC(dev->devfn), 809 | flags); 810 | 811 | /* Determine if the device supports requested interrupt type */ 812 | #ifdef CONFIG_PCI_MSI 813 | if (irq_type == MODS_IRQ_TYPE_MSI) { 814 | if (pci_find_capability(dev, PCI_CAP_ID_MSI) == 0) { 815 | cl_error("dev %04x:%02x:%02x.%x does not support MSI\n", 816 | pci_domain_nr(dev->bus), 817 | dev->bus->number, 818 | PCI_SLOT(dev->devfn), 819 | PCI_FUNC(dev->devfn)); 820 | LOG_EXT(); 821 | return -ENOENT; 822 | } 823 | } else if (irq_type == MODS_IRQ_TYPE_MSIX) { 824 | #ifdef MODS_HAS_MSIX_RANGE 825 | int cnt; 826 | #endif 827 | 828 | if (pci_find_capability(dev, PCI_CAP_ID_MSIX) == 0) { 829 | cl_error( 830 | "dev %04x:%02x:%02x.%x does not support MSI-X\n", 831 | pci_domain_nr(dev->bus), 832 | dev->bus->number, 833 | PCI_SLOT(dev->devfn), 834 | PCI_FUNC(dev->devfn)); 835 | LOG_EXT(); 836 | return -ENOENT; 837 | } 838 | 839 | #ifdef MODS_HAS_MSIX_RANGE 840 | cnt = pci_msix_vec_count(dev); 841 | if (cnt < 0) { 842 | cl_error("MSI-X is not available on dev %04x:%02x:%02x.%x\n", 843 | pci_domain_nr(dev->bus), 844 | dev->bus->number, 845 | PCI_SLOT(dev->devfn), 846 | PCI_FUNC(dev->devfn)); 847 | LOG_EXT(); 848 | return cnt; 849 | } 850 | if (nvecs > (u32)cnt) { 851 | cl_error("cannot allocate %u MSI-X vectors on dev %04x:%02x:%02x.%x, only %d supported\n", 852 | nvecs, 853 | pci_domain_nr(dev->bus), 854 | dev->bus->number, 855 | PCI_SLOT(dev->devfn), 856 | PCI_FUNC(dev->devfn), 857 | cnt); 858 | LOG_EXT(); 859 | return -EINVAL; 860 | } 861 | #endif 862 | } 863 | #else 864 | if (irq_type == MODS_IRQ_TYPE_MSI || irq_type == MODS_IRQ_TYPE_MSIX) { 865 | cl_error("the kernel does not support MSI\n"); 866 | LOG_EXT(); 867 | return -EINVAL; 868 | } 869 | #endif 870 | 871 | /* Enable device on the PCI bus */ 872 | err = mods_enable_device(client, dev, &dpriv); 873 | if (err) { 874 | LOG_EXT(); 875 | return err; 876 | } 877 | 878 | if (irq_type == MODS_IRQ_TYPE_INT) { 879 | /* use legacy irq */ 880 | if (nvecs != 1) { 881 | cl_error( 882 | "INTA: only 1 INTA vector supported, requested %u\n", 883 | nvecs); 884 | LOG_EXT(); 885 | return -EINVAL; 886 | } 887 | dpriv->nvecs = 1; 888 | } 889 | /* Enable MSI */ 890 | #ifdef CONFIG_PCI_MSI 891 | else if (irq_type == MODS_IRQ_TYPE_MSI) { 892 | if (nvecs != 1) { 893 | cl_error( 894 | "MSI: only 1 MSI vector supported, requested %u\n", 895 | nvecs); 896 | LOG_EXT(); 897 | return -EINVAL; 898 | } 899 | err = pci_enable_msi(dev); 900 | if (err) { 901 | cl_error( 902 | "unable to enable MSI on dev %04x:%02x:%02x.%x\n", 903 | pci_domain_nr(dev->bus), 904 | dev->bus->number, 905 | PCI_SLOT(dev->devfn), 906 | PCI_FUNC(dev->devfn)); 907 | LOG_EXT(); 908 | return err; 909 | } 910 | dpriv->nvecs = 1; 911 | } else if (irq_type == MODS_IRQ_TYPE_MSIX) { 912 | struct msix_entry *entries; 913 | int i; 914 | int cnt; 915 | 916 | entries = kcalloc(nvecs, sizeof(struct msix_entry), 917 | GFP_KERNEL | __GFP_NORETRY); 918 | 919 | if (!entries) { 920 | cl_error("could not allocate memory\n"); 921 | LOG_EXT(); 922 | return -ENOMEM; 923 | } 924 | atomic_inc(&client->num_allocs); 925 | 926 | for (i = 0; i < nvecs; i++) 927 | entries[i].entry = (uint16_t)i; 928 | 929 | #ifdef MODS_HAS_MSIX_RANGE 930 | cnt = pci_enable_msix_range(dev, entries, nvecs, nvecs); 931 | 932 | if (cnt < 0) { 933 | cl_error("failed to allocate %u MSI-X vectors on dev %04x:%02x:%02x.%x\n", 934 | nvecs, 935 | pci_domain_nr(dev->bus), 936 | dev->bus->number, 937 | PCI_SLOT(dev->devfn), 938 | PCI_FUNC(dev->devfn)); 939 | kfree(entries); 940 | atomic_dec(&client->num_allocs); 941 | LOG_EXT(); 942 | return cnt; 943 | } 944 | 945 | nvecs = (u32)cnt; 946 | #else 947 | cnt = pci_enable_msix(dev, entries, nvecs); 948 | 949 | if (cnt) { 950 | /* A return of < 0 indicates a failure. 951 | * A return of > 0 indicates that driver request is 952 | * exceeding the number of irqs or MSI-X 953 | * vectors available 954 | */ 955 | cl_error("failed to allocate %u MSI-X vectors on dev %04x:%02x:%02x.%x\n", 956 | nvecs, 957 | pci_domain_nr(dev->bus), 958 | dev->bus->number, 959 | PCI_SLOT(dev->devfn), 960 | PCI_FUNC(dev->devfn)); 961 | kfree(entries); 962 | atomic_dec(&client->num_allocs); 963 | LOG_EXT(); 964 | if (cnt > 0) 965 | cnt = -ENOSPC; 966 | return cnt; 967 | } 968 | #endif 969 | 970 | cl_debug(DEBUG_ISR, 971 | "allocated %d irq's of type %s(%d)\n", 972 | nvecs, 973 | mods_irq_type_name(irq_type), 974 | irq_type); 975 | 976 | for (i = 0; i < nvecs; i++) 977 | cl_debug(DEBUG_ISR, 978 | "vec %d %x\n", 979 | entries[i].entry, 980 | entries[i].vector); 981 | 982 | dpriv->nvecs = nvecs; 983 | dpriv->msix_entries = entries; 984 | } 985 | #endif 986 | else { 987 | cl_error("unsupported irq_type %u dev %04x:%02x:%02x.%x\n", 988 | irq_type, 989 | pci_domain_nr(dev->bus), 990 | dev->bus->number, 991 | PCI_SLOT(dev->devfn), 992 | PCI_FUNC(dev->devfn)); 993 | LOG_EXT(); 994 | return -EINVAL; 995 | } 996 | 997 | WARN_ON(dpriv->client_id != client->client_id); 998 | dpriv->irq_flags = flags; 999 | LOG_EXT(); 1000 | return OK; 1001 | } 1002 | 1003 | static int mods_register_pci_irq(struct mods_client *client, 1004 | struct MODS_REGISTER_IRQ_4 *p) 1005 | { 1006 | int err = OK; 1007 | unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); 1008 | struct pci_dev *dev; 1009 | struct en_dev_entry *dpriv; 1010 | int i; 1011 | 1012 | LOG_ENT(); 1013 | 1014 | if (unlikely(!p->irq_count)) { 1015 | cl_error("no irq's requested\n"); 1016 | LOG_EXT(); 1017 | return -EINVAL; 1018 | } 1019 | 1020 | /* Get the PCI device structure for the specified device from kernel */ 1021 | err = mods_find_pci_dev(client, &p->dev, &dev); 1022 | if (unlikely(err)) { 1023 | if (err == -ENODEV) 1024 | cl_error("dev %04x:%02x:%02x.%x not found\n", 1025 | p->dev.domain, 1026 | p->dev.bus, 1027 | p->dev.device, 1028 | p->dev.function); 1029 | LOG_EXT(); 1030 | return err; 1031 | } 1032 | 1033 | if (unlikely(mutex_lock_interruptible(&irq_mtx))) { 1034 | pci_dev_put(dev); 1035 | LOG_EXT(); 1036 | return -EINTR; 1037 | } 1038 | 1039 | dpriv = pci_get_drvdata(dev); 1040 | if (!dpriv) { 1041 | cl_error( 1042 | "driver data is not set for %04x:%02x:%02x.%x\n", 1043 | pci_domain_nr(dev->bus), 1044 | dev->bus->number, 1045 | PCI_SLOT(dev->devfn), 1046 | PCI_FUNC(dev->devfn)); 1047 | mutex_unlock(&irq_mtx); 1048 | pci_dev_put(dev); 1049 | LOG_EXT(); 1050 | return -EINVAL; 1051 | } 1052 | 1053 | if (is_valid_client_id(dpriv->client_id)) { 1054 | if (dpriv->client_id != client->client_id) { 1055 | cl_error( 1056 | "dev %04x:%02x:%02x.%x already owned by client %u\n", 1057 | p->dev.domain, 1058 | p->dev.bus, 1059 | p->dev.device, 1060 | p->dev.function, 1061 | dpriv->client_id); 1062 | mutex_unlock(&irq_mtx); 1063 | pci_dev_put(dev); 1064 | LOG_EXT(); 1065 | return -EBUSY; 1066 | } 1067 | if (dpriv->nvecs) { 1068 | cl_error( 1069 | "interrupt for dev %04x:%02x:%02x.%x already registered\n", 1070 | p->dev.domain, 1071 | p->dev.bus, 1072 | p->dev.device, 1073 | p->dev.function); 1074 | mutex_unlock(&irq_mtx); 1075 | pci_dev_put(dev); 1076 | LOG_EXT(); 1077 | return -EINVAL; 1078 | } 1079 | } 1080 | 1081 | err = mods_allocate_irqs(client, dev, p->irq_count, 1082 | p->irq_flags); 1083 | if (err) { 1084 | cl_error("could not allocate irqs for irq_type %d\n", 1085 | irq_type); 1086 | mutex_unlock(&irq_mtx); 1087 | pci_dev_put(dev); 1088 | LOG_EXT(); 1089 | return err; 1090 | } 1091 | 1092 | dpriv = pci_get_drvdata(dev); 1093 | 1094 | for (i = 0; i < p->irq_count; i++) { 1095 | u32 irq = ((irq_type == MODS_IRQ_TYPE_INT) || 1096 | (irq_type == MODS_IRQ_TYPE_MSI)) ? dev->irq : 1097 | dpriv->msix_entries[i].vector; 1098 | 1099 | err = add_irq_map(client, dev, p, irq, i); 1100 | if (unlikely(err)) { 1101 | #ifdef CONFIG_PCI_MSI 1102 | if (irq_type == MODS_IRQ_TYPE_MSI) 1103 | pci_disable_msi(dev); 1104 | else if (irq_type == MODS_IRQ_TYPE_MSIX) 1105 | pci_disable_msix(dev); 1106 | #endif 1107 | break; 1108 | } 1109 | } 1110 | 1111 | mutex_unlock(&irq_mtx); 1112 | pci_dev_put(dev); 1113 | LOG_EXT(); 1114 | return err; 1115 | } 1116 | #endif /* CONFIG_PCI */ 1117 | 1118 | static int mods_register_cpu_irq(struct mods_client *client, 1119 | struct MODS_REGISTER_IRQ_4 *p) 1120 | { 1121 | u32 irq = p->dev.bus; 1122 | int err; 1123 | 1124 | LOG_ENT(); 1125 | 1126 | if (unlikely(mutex_lock_interruptible(&irq_mtx))) { 1127 | LOG_EXT(); 1128 | return -EINTR; 1129 | } 1130 | 1131 | /* Determine if the interrupt is already hooked */ 1132 | if (mods_lookup_cpu_irq(0, irq) == IRQ_FOUND) { 1133 | cl_error("CPU IRQ 0x%x has already been registered\n", irq); 1134 | mutex_unlock(&irq_mtx); 1135 | LOG_EXT(); 1136 | return -EINVAL; 1137 | } 1138 | 1139 | /* Register interrupt */ 1140 | err = add_irq_map(client, NULL, p, irq, 0); 1141 | 1142 | mutex_unlock(&irq_mtx); 1143 | LOG_EXT(); 1144 | return err; 1145 | } 1146 | 1147 | #ifdef CONFIG_PCI 1148 | static int mods_unregister_pci_irq(struct mods_client *client, 1149 | struct MODS_REGISTER_IRQ_2 *p) 1150 | { 1151 | struct pci_dev *dev; 1152 | int err = OK; 1153 | 1154 | LOG_ENT(); 1155 | 1156 | /* Get the PCI device structure for the specified device from kernel */ 1157 | err = mods_find_pci_dev(client, &p->dev, &dev); 1158 | if (unlikely(err)) { 1159 | LOG_EXT(); 1160 | return err; 1161 | } 1162 | 1163 | err = mods_free_irqs(client, dev); 1164 | 1165 | pci_dev_put(dev); 1166 | LOG_EXT(); 1167 | return err; 1168 | } 1169 | #endif 1170 | 1171 | static int mods_unregister_cpu_irq(struct mods_client *client, 1172 | struct MODS_REGISTER_IRQ_2 *p) 1173 | { 1174 | struct dev_irq_map *del = NULL; 1175 | struct dev_irq_map *next; 1176 | unsigned int irq; 1177 | 1178 | LOG_ENT(); 1179 | 1180 | irq = p->dev.bus; 1181 | 1182 | if (unlikely(mutex_lock_interruptible(&irq_mtx))) { 1183 | LOG_EXT(); 1184 | return -EINTR; 1185 | } 1186 | 1187 | /* Determine if the interrupt is already hooked by this client */ 1188 | if (mods_lookup_cpu_irq(client->client_id, irq) == IRQ_NOT_FOUND) { 1189 | cl_error("IRQ 0x%x not hooked, can't unhook\n", irq); 1190 | mutex_unlock(&irq_mtx); 1191 | LOG_EXT(); 1192 | return -EINVAL; 1193 | } 1194 | 1195 | /* Delete device interrupt from the list */ 1196 | list_for_each_entry_safe(del, next, &client->irq_list, list) { 1197 | if ((irq == del->apic_irq) && (del->dev == NULL)) { 1198 | if (del->type != p->type) { 1199 | cl_error("wrong IRQ type passed\n"); 1200 | mutex_unlock(&irq_mtx); 1201 | LOG_EXT(); 1202 | return -EINVAL; 1203 | } 1204 | list_del(&del->list); 1205 | cl_debug(DEBUG_ISR, 1206 | "unregistered CPU IRQ 0x%x\n", 1207 | irq); 1208 | mods_free_map(client, del); 1209 | break; 1210 | } 1211 | } 1212 | 1213 | mutex_unlock(&irq_mtx); 1214 | LOG_EXT(); 1215 | return OK; 1216 | } 1217 | 1218 | /************************* 1219 | * ESCAPE CALL FUNCTIONS * 1220 | *************************/ 1221 | 1222 | int esc_mods_register_irq_4(struct mods_client *client, 1223 | struct MODS_REGISTER_IRQ_4 *p) 1224 | { 1225 | u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); 1226 | 1227 | if (irq_type == MODS_IRQ_TYPE_CPU) 1228 | return mods_register_cpu_irq(client, p); 1229 | #ifdef CONFIG_PCI 1230 | return mods_register_pci_irq(client, p); 1231 | #else 1232 | cl_error("PCI not available\n"); 1233 | return -EINVAL; 1234 | #endif 1235 | } 1236 | 1237 | int esc_mods_register_irq_3(struct mods_client *client, 1238 | struct MODS_REGISTER_IRQ_3 *p) 1239 | { 1240 | struct MODS_REGISTER_IRQ_4 irq_data = { {0} }; 1241 | u32 ii = 0; 1242 | 1243 | irq_data.dev = p->dev; 1244 | irq_data.aperture_addr = p->aperture_addr; 1245 | irq_data.aperture_size = p->aperture_size; 1246 | irq_data.mask_info_cnt = p->mask_info_cnt; 1247 | for (ii = 0; ii < p->mask_info_cnt; ii++) { 1248 | irq_data.mask_info[ii].mask_type = p->mask_info[ii].mask_type; 1249 | irq_data.mask_info[ii].irq_pending_offset = 1250 | p->mask_info[ii].irq_pending_offset; 1251 | irq_data.mask_info[ii].irq_enabled_offset = 1252 | p->mask_info[ii].irq_enabled_offset; 1253 | irq_data.mask_info[ii].irq_enable_offset = 1254 | p->mask_info[ii].irq_enable_offset; 1255 | irq_data.mask_info[ii].irq_disable_offset = 1256 | p->mask_info[ii].irq_disable_offset; 1257 | irq_data.mask_info[ii].and_mask = 1258 | p->mask_info[ii].and_mask; 1259 | irq_data.mask_info[ii].or_mask = 1260 | p->mask_info[ii].or_mask; 1261 | } 1262 | irq_data.irq_count = 1; 1263 | irq_data.irq_flags = p->irq_type; 1264 | 1265 | return esc_mods_register_irq_4(client, &irq_data); 1266 | } 1267 | 1268 | int esc_mods_register_irq_2(struct mods_client *client, 1269 | struct MODS_REGISTER_IRQ_2 *p) 1270 | { 1271 | struct MODS_REGISTER_IRQ_4 irq_data = { {0} }; 1272 | #ifdef CONFIG_PCI 1273 | struct pci_dev *dev; 1274 | int err; 1275 | resource_size_t size; 1276 | #endif 1277 | 1278 | irq_data.dev = p->dev; 1279 | irq_data.irq_count = 1; 1280 | irq_data.irq_flags = p->type; 1281 | 1282 | #ifdef CONFIG_PCI 1283 | /* Get the PCI device structure */ 1284 | err = mods_find_pci_dev(client, &p->dev, &dev); 1285 | if (unlikely(err)) 1286 | return err; 1287 | 1288 | irq_data.aperture_addr = pci_resource_start(dev, 0); 1289 | 1290 | size = pci_resource_len(dev, 0); 1291 | 1292 | pci_dev_put(dev); 1293 | 1294 | if (size > ~0U) { 1295 | cl_error("BAR0 size of device %04x:%02x:%02x.%x is %llx and exceeds 32 bits\n", 1296 | p->dev.domain, 1297 | p->dev.bus, 1298 | p->dev.device, 1299 | p->dev.function, 1300 | (unsigned long long)size); 1301 | return -EINVAL; 1302 | } 1303 | 1304 | irq_data.aperture_size = (unsigned int)size; 1305 | #endif 1306 | 1307 | return esc_mods_register_irq_4(client, &irq_data); 1308 | } 1309 | 1310 | int esc_mods_register_irq(struct mods_client *client, 1311 | struct MODS_REGISTER_IRQ *p) 1312 | { 1313 | struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; 1314 | 1315 | register_irq.dev.domain = 0; 1316 | register_irq.dev.bus = p->dev.bus; 1317 | register_irq.dev.device = p->dev.device; 1318 | register_irq.dev.function = p->dev.function; 1319 | register_irq.type = p->type; 1320 | 1321 | return esc_mods_register_irq_2(client, ®ister_irq); 1322 | } 1323 | 1324 | int esc_mods_unregister_irq_2(struct mods_client *client, 1325 | struct MODS_REGISTER_IRQ_2 *p) 1326 | { 1327 | if (p->type == MODS_IRQ_TYPE_CPU) 1328 | return mods_unregister_cpu_irq(client, p); 1329 | #ifdef CONFIG_PCI 1330 | return mods_unregister_pci_irq(client, p); 1331 | #else 1332 | return -EINVAL; 1333 | #endif 1334 | } 1335 | 1336 | int esc_mods_unregister_irq(struct mods_client *client, 1337 | struct MODS_REGISTER_IRQ *p) 1338 | { 1339 | struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; 1340 | 1341 | register_irq.dev.domain = 0; 1342 | register_irq.dev.bus = p->dev.bus; 1343 | register_irq.dev.device = p->dev.device; 1344 | register_irq.dev.function = p->dev.function; 1345 | register_irq.type = p->type; 1346 | 1347 | return esc_mods_unregister_irq_2(client, ®ister_irq); 1348 | } 1349 | 1350 | int esc_mods_query_irq_3(struct mods_client *client, 1351 | struct MODS_QUERY_IRQ_3 *p) 1352 | { 1353 | struct irq_q_info *q = NULL; 1354 | int err = OK; 1355 | unsigned int i = 0; 1356 | unsigned long flags = 0; 1357 | unsigned int cur_time = get_cur_time(); 1358 | 1359 | LOG_ENT(); 1360 | 1361 | /* Clear return array */ 1362 | memset(p->irq_list, 0xFF, sizeof(p->irq_list)); 1363 | 1364 | /* Lock IRQ queue */ 1365 | spin_lock_irqsave(&client->irq_lock, flags); 1366 | 1367 | /* Fill in return array with IRQ information */ 1368 | q = &client->irq_queue; 1369 | for (i = 0; 1370 | (q->head != q->tail) && (i < MODS_MAX_IRQS); 1371 | q->head++, i++) { 1372 | 1373 | unsigned int index = q->head & (MODS_MAX_IRQS - 1); 1374 | struct pci_dev *dev = q->data[index].dev; 1375 | 1376 | if (dev) { 1377 | const int dom = pci_domain_nr(dev->bus); 1378 | 1379 | if (dom < 0 || dom > 0xFFFF) { 1380 | cl_error("unsupported domain %d\n", dom); 1381 | err = -EINVAL; 1382 | goto error; 1383 | } 1384 | p->irq_list[i].dev.domain = dom; 1385 | p->irq_list[i].dev.bus = dev->bus->number; 1386 | p->irq_list[i].dev.device = PCI_SLOT(dev->devfn); 1387 | p->irq_list[i].dev.function = PCI_FUNC(dev->devfn); 1388 | } else { 1389 | if (q->data[index].irq > 0xFFFFU) { 1390 | cl_error("unsupported IRQ %u\n", 1391 | q->data[index].irq); 1392 | err = -EINVAL; 1393 | goto error; 1394 | } 1395 | p->irq_list[i].dev.domain = 0; 1396 | p->irq_list[i].dev.bus = q->data[index].irq; 1397 | p->irq_list[i].dev.device = 0xFFU; 1398 | p->irq_list[i].dev.function = 0xFFU; 1399 | } 1400 | p->irq_list[i].irq_index = q->data[index].irq_index; 1401 | p->irq_list[i].delay = cur_time - q->data[index].time; 1402 | 1403 | /* Print info about IRQ status returned */ 1404 | if (dev) { 1405 | cl_debug(DEBUG_ISR_DETAILED, 1406 | "retrieved IRQ index=%d dev %04x:%02x:%02x.%x, time=%uus, delay=%uus\n", 1407 | p->irq_list[i].irq_index, 1408 | p->irq_list[i].dev.domain, 1409 | p->irq_list[i].dev.bus, 1410 | p->irq_list[i].dev.device, 1411 | p->irq_list[i].dev.function, 1412 | q->data[index].time, 1413 | p->irq_list[i].delay); 1414 | } else { 1415 | cl_debug(DEBUG_ISR_DETAILED, 1416 | "retrieved IRQ 0x%x, time=%uus, delay=%uus\n", 1417 | (unsigned int)p->irq_list[i].dev.bus, 1418 | q->data[index].time, 1419 | p->irq_list[i].delay); 1420 | } 1421 | } 1422 | 1423 | /* Indicate if there are more IRQs pending */ 1424 | if (q->head != q->tail) 1425 | p->more = 1; 1426 | 1427 | error: 1428 | /* Unlock IRQ queue */ 1429 | spin_unlock_irqrestore(&client->irq_lock, flags); 1430 | 1431 | LOG_EXT(); 1432 | return err; 1433 | } 1434 | 1435 | int esc_mods_query_irq_2(struct mods_client *client, 1436 | struct MODS_QUERY_IRQ_2 *p) 1437 | { 1438 | int retval, i; 1439 | struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; 1440 | 1441 | retval = esc_mods_query_irq_3(client, &query_irq); 1442 | if (retval) 1443 | return retval; 1444 | 1445 | for (i = 0; i < MODS_MAX_IRQS; i++) { 1446 | p->irq_list[i].dev = query_irq.irq_list[i].dev; 1447 | p->irq_list[i].delay = query_irq.irq_list[i].delay; 1448 | } 1449 | p->more = query_irq.more; 1450 | return OK; 1451 | } 1452 | 1453 | int esc_mods_query_irq(struct mods_client *client, 1454 | struct MODS_QUERY_IRQ *p) 1455 | { 1456 | int retval, i; 1457 | struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; 1458 | 1459 | retval = esc_mods_query_irq_3(client, &query_irq); 1460 | if (retval) 1461 | return retval; 1462 | 1463 | for (i = 0; i < MODS_MAX_IRQS; i++) { 1464 | struct mods_irq_3 *entry = &query_irq.irq_list[i]; 1465 | 1466 | if (entry->dev.device > 0xFFU) { 1467 | cl_error("unsupported device %02x\n", 1468 | entry->dev.device); 1469 | return -EINVAL; 1470 | } 1471 | if (entry->dev.function > 0xFFU) { 1472 | cl_error("unsupported function %02x\n", 1473 | entry->dev.function); 1474 | return -EINVAL; 1475 | } 1476 | p->irq_list[i].dev.bus = entry->dev.bus; 1477 | p->irq_list[i].dev.device = entry->dev.device; 1478 | p->irq_list[i].dev.function = entry->dev.function; 1479 | p->irq_list[i].delay = entry->delay; 1480 | } 1481 | p->more = query_irq.more; 1482 | return OK; 1483 | } 1484 | 1485 | int esc_mods_irq_handled_2(struct mods_client *client, 1486 | struct MODS_REGISTER_IRQ_2 *p) 1487 | { 1488 | unsigned long flags = 0; 1489 | u32 irq = p->dev.bus; 1490 | struct dev_irq_map *t = NULL; 1491 | struct dev_irq_map *next = NULL; 1492 | int err = -EINVAL; 1493 | 1494 | if (p->type != MODS_IRQ_TYPE_CPU) 1495 | return -EINVAL; 1496 | 1497 | LOG_ENT(); 1498 | 1499 | /* Print info */ 1500 | cl_debug(DEBUG_ISR_DETAILED, "mark CPU IRQ 0x%x handled\n", irq); 1501 | 1502 | /* Lock IRQ queue */ 1503 | spin_lock_irqsave(&client->irq_lock, flags); 1504 | 1505 | list_for_each_entry_safe(t, next, &client->irq_list, list) { 1506 | if (t->apic_irq == irq) { 1507 | if (t->type != p->type) { 1508 | cl_error( 1509 | "IRQ type doesn't match registered IRQ\n"); 1510 | } else { 1511 | enable_irq(irq); 1512 | err = OK; 1513 | } 1514 | break; 1515 | } 1516 | } 1517 | 1518 | /* Unlock IRQ queue */ 1519 | spin_unlock_irqrestore(&client->irq_lock, flags); 1520 | 1521 | LOG_EXT(); 1522 | return err; 1523 | } 1524 | 1525 | int esc_mods_irq_handled(struct mods_client *client, 1526 | struct MODS_REGISTER_IRQ *p) 1527 | { 1528 | struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; 1529 | 1530 | register_irq.dev.domain = 0; 1531 | register_irq.dev.bus = p->dev.bus; 1532 | register_irq.dev.device = p->dev.device; 1533 | register_irq.dev.function = p->dev.function; 1534 | register_irq.type = p->type; 1535 | 1536 | return esc_mods_irq_handled_2(client, ®ister_irq); 1537 | } 1538 | 1539 | #if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF_IRQ) && defined(CONFIG_OF) 1540 | int esc_mods_map_irq(struct mods_client *client, 1541 | struct MODS_DT_INFO *p) 1542 | { 1543 | /* irq parameters */ 1544 | struct of_phandle_args oirq; 1545 | /* platform device handle */ 1546 | struct platform_device *pdev = NULL; 1547 | struct device_node *np; 1548 | int err = 0; 1549 | /* the physical irq */ 1550 | int hwirq; 1551 | 1552 | /* Make sure strings from userspace are null-terminated */ 1553 | p->dt_name[sizeof(p->dt_name) - 1] = 0; 1554 | p->full_name[sizeof(p->full_name) - 1] = 0; 1555 | 1556 | /* Search for the node by device tree name */ 1557 | np = of_find_node_by_name(NULL, p->dt_name); 1558 | 1559 | if (!np) { 1560 | cl_error("node %s is not valid\n", p->full_name); 1561 | err = -EINVAL; 1562 | goto error; 1563 | } 1564 | 1565 | /* Can be multiple nodes that share the same dt name, */ 1566 | /* make sure you get the correct node matched by the device's full */ 1567 | /* name in device tree (i.e. watchdog@30c0000 as opposed */ 1568 | /* to watchdog) */ 1569 | while (of_node_cmp(np->full_name, p->full_name)) { 1570 | np = of_find_node_by_name(np, p->dt_name); 1571 | if (!np) { 1572 | cl_error("node %s is not valid\n", p->full_name); 1573 | err = -EINVAL; 1574 | goto error; 1575 | } 1576 | } 1577 | 1578 | if (p->index > 0x7FFFFFFFU) { 1579 | cl_error("unsupported index %u\n", p->index); 1580 | err = -EINVAL; 1581 | goto error; 1582 | } 1583 | p->irq = irq_of_parse_and_map(np, p->index); 1584 | err = of_irq_parse_one(np, p->index, &oirq); 1585 | if (err) { 1586 | cl_error("could not parse IRQ\n"); 1587 | goto error; 1588 | } 1589 | if ((int)oirq.args[1] < 0) { 1590 | cl_error("unsupported IRQ %d\n", (int)oirq.args[1]); 1591 | err = -EINVAL; 1592 | goto error; 1593 | } 1594 | 1595 | hwirq = (int)oirq.args[1]; 1596 | 1597 | /* Get the platform device handle */ 1598 | pdev = of_find_device_by_node(np); 1599 | 1600 | if (of_node_cmp(p->dt_name, "watchdog") == 0) { 1601 | /* Enable and unmask interrupt for watchdog */ 1602 | struct resource *res_src = 1603 | platform_get_resource(pdev, IORESOURCE_MEM, 0); 1604 | struct resource *res_tke = 1605 | platform_get_resource(pdev, IORESOURCE_MEM, 2); 1606 | void __iomem *wdt_tke = NULL; 1607 | unsigned int wdt_index = 0; 1608 | 1609 | if (res_tke && res_src) { 1610 | wdt_tke = devm_ioremap(&pdev->dev, res_tke->start, 1611 | resource_size(res_tke)); 1612 | wdt_index = (unsigned int) 1613 | (((res_src->start >> 16) & 0xFU) - 0xCU); 1614 | } 1615 | 1616 | if (wdt_tke) { 1617 | writel(TOP_TKE_TKEIE_WDT_MASK(wdt_index), 1618 | wdt_tke + TOP_TKE_TKEIE(hwirq)); 1619 | } 1620 | } 1621 | 1622 | error: 1623 | of_node_put(np); 1624 | /* enable the interrupt */ 1625 | return err; 1626 | } 1627 | 1628 | int esc_mods_map_irq_to_gpio(struct mods_client *client, 1629 | struct MODS_GPIO_INFO *p) 1630 | { 1631 | //TODO: Make sure you are allocating gpio properly 1632 | struct device_node *np; 1633 | int gpio_handle; 1634 | int irq; 1635 | int err = 0; 1636 | 1637 | /* Make sure strings from userspace are null-terminated */ 1638 | p->dt_name[sizeof(p->dt_name) - 1] = 0; 1639 | p->full_name[sizeof(p->full_name) - 1] = 0; 1640 | 1641 | np = of_find_node_by_name(NULL, p->dt_name); 1642 | 1643 | if (!np) { 1644 | cl_error("node %s is not valid\n", p->full_name); 1645 | err = -EINVAL; 1646 | goto error; 1647 | } 1648 | 1649 | while (of_node_cmp(np->full_name, p->full_name)) { 1650 | np = of_find_node_by_name(np, p->dt_name); 1651 | if (!np) { 1652 | cl_error("node %s is not valid\n", p->full_name); 1653 | err = -EINVAL; 1654 | goto error; 1655 | } 1656 | } 1657 | 1658 | gpio_handle = of_get_named_gpio(np, p->name, 0); 1659 | if (!gpio_is_valid(gpio_handle)) { 1660 | cl_error("gpio %s is missing\n", p->name); 1661 | err = gpio_handle; 1662 | goto error; 1663 | } 1664 | 1665 | err = gpio_direction_input(gpio_handle); 1666 | if (err < 0) { 1667 | cl_error("pex_rst_gpio input direction change failed\n"); 1668 | goto error; 1669 | } 1670 | 1671 | irq = gpio_to_irq(gpio_handle); 1672 | if (irq < 0) { 1673 | cl_error("Unable to get irq for pex_rst_gpio\n"); 1674 | err = -EINVAL; 1675 | goto error; 1676 | } 1677 | p->irq = irq; 1678 | 1679 | error: 1680 | of_node_put(np); 1681 | return err; 1682 | } 1683 | #endif 1684 | -------------------------------------------------------------------------------- /mods_pci.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2008-2023, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | 6 | #include 7 | #if defined(MODS_HAS_DMA_OPS) 8 | #include 9 | #endif 10 | #include 11 | #include 12 | #include 13 | #if KERNEL_VERSION(2, 32, 0) <= MODS_KERNEL_VERSION 14 | #include 15 | #endif 16 | #if KERNEL_VERSION(3, 19, 0) <= MODS_KERNEL_VERSION 17 | #include 18 | #endif 19 | 20 | /* Address in config space is 8bit for base caps and 12bit for extended caps */ 21 | #define MODS_MAX_PCI_CFG_ADDR 0xFFFU 22 | 23 | int mods_is_pci_dev(struct pci_dev *dev, 24 | struct mods_pci_dev_2 *pcidev) 25 | { 26 | unsigned int devfn = PCI_DEVFN(pcidev->device, pcidev->function); 27 | 28 | return dev && 29 | pci_domain_nr(dev->bus) == pcidev->domain && 30 | dev->bus->number == pcidev->bus && 31 | dev->devfn == devfn; 32 | } 33 | 34 | int mods_find_pci_dev(struct mods_client *client, 35 | struct mods_pci_dev_2 *pcidev, 36 | struct pci_dev **retdev) 37 | { 38 | struct pci_dev *dev; 39 | int err; 40 | 41 | if (unlikely(mutex_lock_interruptible(&client->mtx))) 42 | return -EINTR; 43 | 44 | dev = client->cached_dev; 45 | 46 | if (mods_is_pci_dev(dev, pcidev)) { 47 | *retdev = pci_dev_get(dev); 48 | mutex_unlock(&client->mtx); 49 | return OK; 50 | } 51 | 52 | mutex_unlock(&client->mtx); 53 | 54 | dev = NULL; 55 | 56 | #ifdef MODS_HAS_NEW_ACPI_WALK 57 | dev = pci_get_domain_bus_and_slot(pcidev->domain, 58 | pcidev->bus, 59 | PCI_DEVFN(pcidev->device, 60 | pcidev->function)); 61 | #else 62 | while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev))) 63 | if (mods_is_pci_dev(dev, pcidev)) 64 | break; 65 | #endif 66 | 67 | if (dev) { 68 | if (unlikely(mutex_lock_interruptible(&client->mtx))) { 69 | pci_dev_put(dev); 70 | return -EINTR; 71 | } 72 | 73 | if (dev != client->cached_dev) { 74 | pci_dev_put(client->cached_dev); 75 | client->cached_dev = pci_dev_get(dev); 76 | } 77 | 78 | mutex_unlock(&client->mtx); 79 | 80 | err = OK; 81 | } else 82 | err = -ENODEV; 83 | 84 | *retdev = dev; 85 | return err; 86 | } 87 | 88 | static int find_pci_dev_impl(struct mods_client *client, 89 | struct MODS_FIND_PCI_DEVICE_2 *p, 90 | int enum_non_zero_dom) 91 | { 92 | struct pci_dev *dev = NULL; 93 | int index = -1; 94 | 95 | LOG_ENT(); 96 | 97 | if (p->index > 0xFFFFU) { 98 | cl_error("invalid device index %u\n", p->index); 99 | LOG_EXT(); 100 | return -EINVAL; 101 | } 102 | 103 | cl_debug(DEBUG_PCI, 104 | "find pci dev %04x:%04x, index %u\n", 105 | p->vendor_id, 106 | p->device_id, 107 | p->index); 108 | 109 | do { 110 | dev = pci_get_device(p->vendor_id, p->device_id, dev); 111 | if (!dev) { 112 | LOG_EXT(); 113 | return -EINVAL; 114 | } 115 | 116 | if (enum_non_zero_dom || !pci_domain_nr(dev->bus)) 117 | ++index; 118 | } while (index < (int)(p->index)); 119 | 120 | p->pci_device.domain = pci_domain_nr(dev->bus); 121 | p->pci_device.bus = dev->bus->number; 122 | p->pci_device.device = PCI_SLOT(dev->devfn); 123 | p->pci_device.function = PCI_FUNC(dev->devfn); 124 | 125 | pci_dev_put(dev); 126 | LOG_EXT(); 127 | return OK; 128 | } 129 | 130 | int esc_mods_find_pci_dev_2(struct mods_client *client, 131 | struct MODS_FIND_PCI_DEVICE_2 *p) 132 | { 133 | return find_pci_dev_impl(client, p, 1); 134 | } 135 | 136 | int esc_mods_find_pci_dev(struct mods_client *client, 137 | struct MODS_FIND_PCI_DEVICE *p) 138 | { 139 | struct MODS_FIND_PCI_DEVICE_2 p2; 140 | int err; 141 | 142 | p2.device_id = p->device_id; 143 | p2.vendor_id = p->vendor_id; 144 | p2.index = p->index; 145 | 146 | err = find_pci_dev_impl(client, &p2, 0); 147 | 148 | if (!err) { 149 | p->bus_number = p2.pci_device.bus; 150 | p->device_number = p2.pci_device.device; 151 | p->function_number = p2.pci_device.function; 152 | } 153 | 154 | return err; 155 | } 156 | 157 | static int mods_find_pci_class_code(struct mods_client *client, 158 | struct MODS_FIND_PCI_CLASS_CODE_2 *p, 159 | int enum_non_zero_dom) 160 | { 161 | struct pci_dev *dev = NULL; 162 | int index = -1; 163 | 164 | LOG_ENT(); 165 | 166 | if (p->index > 0xFFFFU) { 167 | cl_error("invalid device index %u\n", p->index); 168 | LOG_EXT(); 169 | return -EINVAL; 170 | } 171 | 172 | cl_debug(DEBUG_PCI, 173 | "find pci class code %04x, index %u\n", 174 | p->class_code, 175 | p->index); 176 | 177 | do { 178 | dev = pci_get_class(p->class_code, dev); 179 | if (!dev) { 180 | LOG_EXT(); 181 | return -EINVAL; 182 | } 183 | 184 | if (enum_non_zero_dom || !pci_domain_nr(dev->bus)) 185 | ++index; 186 | } while (index < (int)(p->index)); 187 | 188 | p->pci_device.domain = pci_domain_nr(dev->bus); 189 | p->pci_device.bus = dev->bus->number; 190 | p->pci_device.device = PCI_SLOT(dev->devfn); 191 | p->pci_device.function = PCI_FUNC(dev->devfn); 192 | 193 | pci_dev_put(dev); 194 | LOG_EXT(); 195 | return OK; 196 | } 197 | 198 | int esc_mods_find_pci_class_code_2(struct mods_client *client, 199 | struct MODS_FIND_PCI_CLASS_CODE_2 *p) 200 | { 201 | return mods_find_pci_class_code(client, p, 1); 202 | } 203 | 204 | int esc_mods_find_pci_class_code(struct mods_client *client, 205 | struct MODS_FIND_PCI_CLASS_CODE *p) 206 | { 207 | struct MODS_FIND_PCI_CLASS_CODE_2 p2; 208 | int err; 209 | 210 | p2.class_code = p->class_code; 211 | p2.index = p->index; 212 | 213 | err = mods_find_pci_class_code(client, &p2, 0); 214 | 215 | if (!err) { 216 | p->bus_number = p2.pci_device.bus; 217 | p->device_number = p2.pci_device.device; 218 | p->function_number = p2.pci_device.function; 219 | } 220 | 221 | return err; 222 | } 223 | 224 | int esc_mods_pci_get_bar_info_2(struct mods_client *client, 225 | struct MODS_PCI_GET_BAR_INFO_2 *p) 226 | { 227 | struct pci_dev *dev; 228 | unsigned int bar_resource_offset; 229 | unsigned int i; 230 | int err; 231 | #if !defined(MODS_HAS_IORESOURCE_MEM_64) 232 | __u32 temp; 233 | #endif 234 | 235 | LOG_ENT(); 236 | 237 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 238 | if (unlikely(err)) { 239 | LOG_EXT(); 240 | return err; 241 | } 242 | 243 | cl_debug(DEBUG_PCI, 244 | "pci get bar info dev %04x:%02x:%02x:%x, bar index %u\n", 245 | p->pci_device.domain, 246 | p->pci_device.bus, 247 | p->pci_device.device, 248 | p->pci_device.function, 249 | p->bar_index); 250 | 251 | #if defined(CONFIG_PPC64) 252 | if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex()))) { 253 | pci_dev_put(dev); 254 | LOG_EXT(); 255 | return -EINTR; 256 | } 257 | 258 | /* Enable device on the PCI bus */ 259 | err = mods_enable_device(client, dev, NULL); 260 | if (err) { 261 | cl_error("unable to enable dev %04x:%02x:%02x.%x\n", 262 | p->pci_device.domain, 263 | p->pci_device.bus, 264 | p->pci_device.device, 265 | p->pci_device.function); 266 | mutex_unlock(mods_get_irq_mutex()); 267 | pci_dev_put(dev); 268 | LOG_EXT(); 269 | return err; 270 | } 271 | 272 | mutex_unlock(mods_get_irq_mutex()); 273 | #endif 274 | 275 | bar_resource_offset = 0; 276 | for (i = 0; i < p->bar_index; i++) { 277 | #if defined(MODS_HAS_IORESOURCE_MEM_64) 278 | if (pci_resource_flags(dev, bar_resource_offset) 279 | & IORESOURCE_MEM_64) { 280 | #else 281 | pci_read_config_dword(dev, 282 | (PCI_BASE_ADDRESS_0 283 | + (bar_resource_offset * 4)), 284 | &temp); 285 | if (temp & PCI_BASE_ADDRESS_MEM_TYPE_64) { 286 | #endif 287 | bar_resource_offset += 2; 288 | } else { 289 | bar_resource_offset += 1; 290 | } 291 | } 292 | p->base_address = pci_resource_start(dev, bar_resource_offset); 293 | p->bar_size = pci_resource_len(dev, bar_resource_offset); 294 | 295 | pci_dev_put(dev); 296 | LOG_EXT(); 297 | return OK; 298 | } 299 | 300 | int esc_mods_pci_get_bar_info(struct mods_client *client, 301 | struct MODS_PCI_GET_BAR_INFO *p) 302 | { 303 | int err; 304 | struct MODS_PCI_GET_BAR_INFO_2 get_bar_info = { {0} }; 305 | 306 | get_bar_info.pci_device.domain = 0; 307 | get_bar_info.pci_device.bus = p->pci_device.bus; 308 | get_bar_info.pci_device.device = p->pci_device.device; 309 | get_bar_info.pci_device.function = p->pci_device.function; 310 | get_bar_info.bar_index = p->bar_index; 311 | 312 | err = esc_mods_pci_get_bar_info_2(client, &get_bar_info); 313 | 314 | if (likely(!err)) { 315 | p->base_address = get_bar_info.base_address; 316 | p->bar_size = get_bar_info.bar_size; 317 | } 318 | 319 | return err; 320 | } 321 | 322 | int esc_mods_pci_get_irq_2(struct mods_client *client, 323 | struct MODS_PCI_GET_IRQ_2 *p) 324 | { 325 | struct pci_dev *dev; 326 | int err; 327 | 328 | LOG_ENT(); 329 | 330 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 331 | if (unlikely(err)) { 332 | LOG_EXT(); 333 | return err; 334 | } 335 | 336 | cl_debug(DEBUG_PCI, 337 | "pci get irq dev %04x:%02x:%02x:%x irq=%u\n", 338 | p->pci_device.domain, 339 | p->pci_device.bus, 340 | p->pci_device.device, 341 | p->pci_device.function, 342 | dev->irq); 343 | 344 | p->irq = dev->irq; 345 | 346 | pci_dev_put(dev); 347 | LOG_EXT(); 348 | return OK; 349 | } 350 | 351 | int esc_mods_pci_get_irq(struct mods_client *client, 352 | struct MODS_PCI_GET_IRQ *p) 353 | { 354 | int err; 355 | struct MODS_PCI_GET_IRQ_2 get_irq = { {0} }; 356 | 357 | get_irq.pci_device.domain = 0; 358 | get_irq.pci_device.bus = p->pci_device.bus; 359 | get_irq.pci_device.device = p->pci_device.device; 360 | get_irq.pci_device.function = p->pci_device.function; 361 | 362 | err = esc_mods_pci_get_irq_2(client, &get_irq); 363 | 364 | if (likely(!err)) 365 | p->irq = get_irq.irq; 366 | 367 | return err; 368 | } 369 | 370 | int esc_mods_pci_read_2(struct mods_client *client, struct MODS_PCI_READ_2 *p) 371 | { 372 | struct pci_dev *dev; 373 | int err; 374 | int dbdf; 375 | 376 | LOG_ENT(); 377 | 378 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 379 | if (unlikely(err)) { 380 | LOG_EXT(); 381 | return err; 382 | } 383 | 384 | if (p->address > MODS_MAX_PCI_CFG_ADDR) { 385 | cl_error("invalid pci config space address 0x%x\n", p->address); 386 | LOG_EXT(); 387 | return -EINVAL; 388 | } 389 | 390 | p->data = 0; 391 | switch (p->data_size) { 392 | case 1: { 393 | u8 value; 394 | 395 | pci_read_config_byte(dev, p->address, &value); 396 | p->data = value; 397 | } 398 | break; 399 | case 2: { 400 | u16 value; 401 | 402 | pci_read_config_word(dev, p->address, &value); 403 | p->data = value; 404 | } 405 | break; 406 | case 4: 407 | pci_read_config_dword(dev, p->address, (u32 *) &p->data); 408 | break; 409 | default: 410 | err = -EINVAL; 411 | break; 412 | } 413 | 414 | cl_debug(DEBUG_PCI | DEBUG_DETAILED, 415 | "pci read dev %04x:%02x:%02x.%x, addr 0x%04x, size %u, data 0x%x\n", 416 | p->pci_device.domain, 417 | p->pci_device.bus, 418 | p->pci_device.device, 419 | p->pci_device.function, 420 | p->address, 421 | p->data_size, 422 | p->data); 423 | 424 | dbdf = (int)(((u32)p->pci_device.domain << 16) | 425 | ((u32)(p->pci_device.bus & 0xFFU) << 8) | 426 | (u32)(p->pci_device.device & 0xFFU)); 427 | 428 | /* Usually one of the first reads from PCI config space occurs 429 | * at address 0 and or 2 to read PCI device vendor/id. 430 | * If this reads all Fs, the device probably fell off the bus. 431 | */ 432 | if (p->address <= 4 && (p->data == ~0U || p->data == 0xFFFFU)) { 433 | if (dbdf != atomic_read(&client->last_bad_dbdf)) 434 | cl_warn("pci read dev %04x:%02x:%02x.%x, addr 0x%04x, size %u, data 0x%x\n", 435 | p->pci_device.domain, 436 | p->pci_device.bus, 437 | p->pci_device.device, 438 | p->pci_device.function, 439 | p->address, 440 | p->data_size, 441 | p->data); 442 | atomic_set(&client->last_bad_dbdf, dbdf); 443 | } else if (dbdf == atomic_read(&client->last_bad_dbdf)) 444 | atomic_set(&client->last_bad_dbdf, -1); 445 | 446 | pci_dev_put(dev); 447 | LOG_EXT(); 448 | return err; 449 | } 450 | 451 | int esc_mods_pci_read(struct mods_client *client, struct MODS_PCI_READ *p) 452 | { 453 | int err; 454 | struct MODS_PCI_READ_2 pci_read = { {0} }; 455 | 456 | if (p->bus_number > 0xFFU) { 457 | cl_error("invalid bus number 0x%x\n", p->bus_number); 458 | return -EINVAL; 459 | } 460 | if (p->device_number > 0xFFU) { 461 | cl_error("invalid device number 0x%x\n", p->device_number); 462 | return -EINVAL; 463 | } 464 | if (p->function_number > 0xFU) { 465 | cl_error("invalid function number 0x%x\n", p->function_number); 466 | return -EINVAL; 467 | } 468 | 469 | pci_read.pci_device.domain = 0; 470 | pci_read.pci_device.bus = p->bus_number; 471 | pci_read.pci_device.device = p->device_number; 472 | pci_read.pci_device.function = p->function_number; 473 | pci_read.address = p->address; 474 | pci_read.data_size = p->data_size; 475 | 476 | err = esc_mods_pci_read_2(client, &pci_read); 477 | 478 | if (likely(!err)) 479 | p->data = pci_read.data; 480 | 481 | return err; 482 | } 483 | 484 | int esc_mods_pci_write_2(struct mods_client *client, struct MODS_PCI_WRITE_2 *p) 485 | { 486 | struct pci_dev *dev; 487 | int err; 488 | 489 | LOG_ENT(); 490 | 491 | cl_debug(DEBUG_PCI | DEBUG_DETAILED, 492 | "pci write dev %04x:%02x:%02x.%x, addr 0x%04x, size %u, data 0x%x\n", 493 | p->pci_device.domain, 494 | p->pci_device.bus, 495 | p->pci_device.device, 496 | p->pci_device.function, 497 | p->address, 498 | p->data_size, 499 | p->data); 500 | 501 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 502 | if (unlikely(err)) { 503 | if (err == -ENODEV) 504 | cl_error("dev %04x:%02x:%02x.%x not found\n", 505 | p->pci_device.domain, 506 | p->pci_device.bus, 507 | p->pci_device.device, 508 | p->pci_device.function); 509 | LOG_EXT(); 510 | return err; 511 | } 512 | 513 | if (p->address > MODS_MAX_PCI_CFG_ADDR) { 514 | cl_error("invalid pci config space address 0x%x\n", p->address); 515 | LOG_EXT(); 516 | return -EINVAL; 517 | } 518 | 519 | switch (p->data_size) { 520 | case 1: 521 | if (p->data > 0xFFU) { 522 | cl_error("invalid byte data 0x%x\n", p->data); 523 | err = -EINVAL; 524 | } else 525 | pci_write_config_byte(dev, p->address, p->data); 526 | break; 527 | case 2: 528 | if (p->data > 0xFFFFU) { 529 | cl_error("invalid word data 0x%x\n", p->data); 530 | err = -EINVAL; 531 | } else 532 | pci_write_config_word(dev, p->address, p->data); 533 | break; 534 | case 4: 535 | pci_write_config_dword(dev, p->address, p->data); 536 | break; 537 | default: 538 | cl_error("invalid data size %u\n", p->data_size); 539 | err = -EINVAL; 540 | break; 541 | } 542 | 543 | pci_dev_put(dev); 544 | LOG_EXT(); 545 | return err; 546 | } 547 | 548 | int esc_mods_pci_write(struct mods_client *client, 549 | struct MODS_PCI_WRITE *p) 550 | { 551 | struct MODS_PCI_WRITE_2 pci_write = { {0} }; 552 | 553 | if (p->bus_number > 0xFFU) { 554 | cl_error("invalid bus number 0x%x\n", p->bus_number); 555 | return -EINVAL; 556 | } 557 | if (p->device_number > 0xFFU) { 558 | cl_error("invalid device number 0x%x\n", p->device_number); 559 | return -EINVAL; 560 | } 561 | if (p->function_number > 0xFU) { 562 | cl_error("invalid function number 0x%x\n", p->function_number); 563 | return -EINVAL; 564 | } 565 | 566 | pci_write.pci_device.domain = 0; 567 | pci_write.pci_device.bus = p->bus_number; 568 | pci_write.pci_device.device = p->device_number; 569 | pci_write.pci_device.function = p->function_number; 570 | pci_write.address = p->address; 571 | pci_write.data = p->data; 572 | pci_write.data_size = p->data_size; 573 | 574 | return esc_mods_pci_write_2(client, &pci_write); 575 | } 576 | 577 | int esc_mods_pci_bus_add_dev(struct mods_client *client, 578 | struct MODS_PCI_BUS_ADD_DEVICES *scan) 579 | { 580 | struct MODS_PCI_BUS_RESCAN rescan = { 0, 0 }; 581 | 582 | if (scan->bus > 0xFFU) { 583 | cl_error("invalid bus number 0x%x\n", scan->bus); 584 | return -EINVAL; 585 | } 586 | 587 | rescan.bus = (u16)scan->bus; 588 | 589 | return esc_mods_pci_bus_rescan(client, &rescan); 590 | } 591 | 592 | int esc_mods_pci_bus_rescan(struct mods_client *client, 593 | struct MODS_PCI_BUS_RESCAN *rescan) 594 | { 595 | #ifndef MODS_HASNT_PCI_RESCAN_BUS 596 | struct pci_bus *bus; 597 | int err = OK; 598 | 599 | LOG_ENT(); 600 | 601 | cl_info("scanning pci bus %04x:%02x\n", rescan->domain, rescan->bus); 602 | 603 | bus = pci_find_bus(rescan->domain, rescan->bus); 604 | 605 | if (likely(bus)) { 606 | #ifndef MODS_HASNT_PCI_LOCK_RESCAN_REMOVE 607 | pci_lock_rescan_remove(); 608 | #endif 609 | pci_rescan_bus(bus); 610 | #ifndef MODS_HASNT_PCI_LOCK_RESCAN_REMOVE 611 | pci_unlock_rescan_remove(); 612 | #endif 613 | } else { 614 | cl_error("bus %04x:%02x not found\n", 615 | rescan->domain, 616 | rescan->bus); 617 | err = -EINVAL; 618 | } 619 | 620 | LOG_EXT(); 621 | 622 | return err; 623 | #else 624 | return -EINVAL; 625 | #endif 626 | } 627 | 628 | int esc_mods_pci_bus_remove_dev(struct mods_client *client, 629 | struct MODS_PCI_BUS_REMOVE_DEV *p) 630 | { 631 | #if !defined(MODS_HASNT_PCI_BUS_REMOVE_DEV) 632 | struct pci_dev *dev; 633 | int err; 634 | 635 | LOG_ENT(); 636 | 637 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 638 | if (unlikely(err)) { 639 | if (err == -ENODEV) 640 | cl_error( 641 | "pci_remove cannot find dev %04x:%02x:%02x.%x\n", 642 | p->pci_device.domain, 643 | p->pci_device.bus, 644 | p->pci_device.device, 645 | p->pci_device.function); 646 | LOG_EXT(); 647 | return err; 648 | } 649 | 650 | cl_debug(DEBUG_PCI, 651 | "pci remove on dev %04x:%02x:%02x.%x\n", 652 | p->pci_device.domain, 653 | p->pci_device.bus, 654 | p->pci_device.device, 655 | p->pci_device.function); 656 | 657 | pci_stop_and_remove_bus_device(dev); 658 | LOG_EXT(); 659 | return err; 660 | #else 661 | return -EINVAL; 662 | #endif 663 | } 664 | 665 | /************************ 666 | * PIO ESCAPE FUNCTIONS * 667 | ************************/ 668 | 669 | int esc_mods_pio_read(struct mods_client *client, struct MODS_PIO_READ *p) 670 | { 671 | LOG_ENT(); 672 | switch (p->data_size) { 673 | case 1: 674 | p->data = inb(p->port); 675 | break; 676 | case 2: 677 | p->data = inw(p->port); 678 | break; 679 | case 4: 680 | p->data = inl(p->port); 681 | break; 682 | default: 683 | return -EINVAL; 684 | } 685 | LOG_EXT(); 686 | return OK; 687 | } 688 | 689 | int esc_mods_pio_write(struct mods_client *client, struct MODS_PIO_WRITE *p) 690 | { 691 | int err = OK; 692 | 693 | LOG_ENT(); 694 | 695 | switch (p->data_size) { 696 | case 1: 697 | if (p->data > 0xFFU) { 698 | cl_error("invalid byte data 0x%x\n", p->data); 699 | err = -EINVAL; 700 | } else 701 | outb(p->data, p->port); 702 | break; 703 | case 2: 704 | if (p->data > 0xFFFFU) { 705 | cl_error("invalid word data 0x%x\n", p->data); 706 | err = -EINVAL; 707 | } else 708 | outw(p->data, p->port); 709 | break; 710 | case 4: 711 | outl(p->data, p->port); 712 | break; 713 | default: 714 | cl_error("invalid data size %u\n", p->data_size); 715 | err = -EINVAL; 716 | } 717 | 718 | LOG_EXT(); 719 | return err; 720 | } 721 | 722 | int esc_mods_device_numa_info_3(struct mods_client *client, 723 | struct MODS_DEVICE_NUMA_INFO_3 *p) 724 | { 725 | struct pci_dev *dev; 726 | int err; 727 | 728 | LOG_ENT(); 729 | 730 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 731 | if (unlikely(err)) { 732 | if (err == -ENODEV) 733 | cl_error("dev %04x:%02x:%02x.%x not found\n", 734 | p->pci_device.domain, 735 | p->pci_device.bus, 736 | p->pci_device.device, 737 | p->pci_device.function); 738 | LOG_EXT(); 739 | return err; 740 | } 741 | 742 | p->node = dev_to_node(&dev->dev); 743 | if (p->node != -1) { 744 | u32 first_offset = ~0U; 745 | unsigned int i; 746 | const unsigned long *maskp; 747 | 748 | maskp = cpumask_bits(cpumask_of_node(p->node)); 749 | 750 | memset(&p->node_cpu_mask, 0, sizeof(p->node_cpu_mask)); 751 | 752 | for (i = 0; i < nr_cpumask_bits; i += 32) { 753 | 754 | const u32 word = i / BITS_PER_LONG; 755 | const u32 bit = i % BITS_PER_LONG; 756 | const u32 cur_mask = (u32)(maskp[word] >> bit); 757 | u32 mask_idx; 758 | 759 | if (first_offset == ~0U) { 760 | if (cur_mask) { 761 | first_offset = i / 32; 762 | p->first_cpu_mask_offset = first_offset; 763 | } else 764 | continue; 765 | } 766 | 767 | mask_idx = (i / 32) - first_offset; 768 | 769 | if (cur_mask && mask_idx >= MAX_CPU_MASKS_3) { 770 | 771 | cl_error("too many CPUs (%d) for mask bits\n", 772 | nr_cpumask_bits); 773 | pci_dev_put(dev); 774 | LOG_EXT(); 775 | return -EINVAL; 776 | } 777 | 778 | if (mask_idx < MAX_CPU_MASKS_3) 779 | p->node_cpu_mask[mask_idx] = cur_mask; 780 | } 781 | 782 | if (first_offset == ~0U) 783 | p->first_cpu_mask_offset = 0; 784 | } 785 | p->node_count = num_possible_nodes(); 786 | p->cpu_count = num_possible_cpus(); 787 | 788 | pci_dev_put(dev); 789 | LOG_EXT(); 790 | return OK; 791 | } 792 | 793 | int esc_mods_device_numa_info_2(struct mods_client *client, 794 | struct MODS_DEVICE_NUMA_INFO_2 *p) 795 | { 796 | int err; 797 | struct MODS_DEVICE_NUMA_INFO_3 numa_info = { {0} }; 798 | 799 | numa_info.pci_device = p->pci_device; 800 | 801 | err = esc_mods_device_numa_info_3(client, &numa_info); 802 | 803 | if (likely(!err)) { 804 | int i; 805 | 806 | p->node = numa_info.node; 807 | p->node_count = numa_info.node_count; 808 | p->cpu_count = numa_info.cpu_count; 809 | 810 | memset(&p->node_cpu_mask, 0, sizeof(p->node_cpu_mask)); 811 | 812 | for (i = 0; i < MAX_CPU_MASKS_3; i++) { 813 | 814 | const u32 cur_mask = numa_info.node_cpu_mask[i]; 815 | const u32 dst = i + 816 | numa_info.first_cpu_mask_offset; 817 | 818 | if (cur_mask && dst >= MAX_CPU_MASKS) { 819 | cl_error("too many CPUs (%d) for mask bits\n", 820 | nr_cpumask_bits); 821 | err = -EINVAL; 822 | break; 823 | } 824 | 825 | if (dst < MAX_CPU_MASKS) 826 | p->node_cpu_mask[dst] 827 | = numa_info.node_cpu_mask[i]; 828 | } 829 | } 830 | 831 | return err; 832 | } 833 | 834 | int esc_mods_device_numa_info(struct mods_client *client, 835 | struct MODS_DEVICE_NUMA_INFO *p) 836 | { 837 | int err; 838 | struct MODS_DEVICE_NUMA_INFO_3 numa_info = { {0} }; 839 | 840 | numa_info.pci_device.domain = 0; 841 | numa_info.pci_device.bus = p->pci_device.bus; 842 | numa_info.pci_device.device = p->pci_device.device; 843 | numa_info.pci_device.function = p->pci_device.function; 844 | 845 | err = esc_mods_device_numa_info_3(client, &numa_info); 846 | 847 | if (likely(!err)) { 848 | int i; 849 | 850 | p->node = numa_info.node; 851 | p->node_count = numa_info.node_count; 852 | p->cpu_count = numa_info.cpu_count; 853 | 854 | memset(&p->node_cpu_mask, 0, sizeof(p->node_cpu_mask)); 855 | 856 | for (i = 0; i < MAX_CPU_MASKS_3; i++) { 857 | 858 | const u32 cur_mask = numa_info.node_cpu_mask[i]; 859 | const u32 dst = i + 860 | numa_info.first_cpu_mask_offset; 861 | 862 | if (cur_mask && dst >= MAX_CPU_MASKS) { 863 | cl_error("too many CPUs (%d) for mask bits\n", 864 | nr_cpumask_bits); 865 | err = -EINVAL; 866 | break; 867 | } 868 | 869 | if (dst < MAX_CPU_MASKS) 870 | p->node_cpu_mask[dst] 871 | = numa_info.node_cpu_mask[i]; 872 | } 873 | } 874 | 875 | return err; 876 | } 877 | 878 | int esc_mods_get_iommu_state(struct mods_client *client, 879 | struct MODS_GET_IOMMU_STATE *state) 880 | { 881 | int err = esc_mods_get_iommu_state_2(client, state); 882 | 883 | if (!err) 884 | state->state = (state->state == MODS_SWIOTLB_DISABLED) ? 1 : 0; 885 | 886 | return err; 887 | } 888 | 889 | int esc_mods_get_iommu_state_2(struct mods_client *client, 890 | struct MODS_GET_IOMMU_STATE *state) 891 | { 892 | #if !defined(CONFIG_SWIOTLB) 893 | state->state = MODS_SWIOTLB_DISABLED; 894 | #elif defined(MODS_HAS_DMA_OPS) 895 | 896 | const struct dma_map_ops *ops; 897 | struct pci_dev *dev; 898 | int err; 899 | 900 | LOG_ENT(); 901 | 902 | err = mods_find_pci_dev(client, &state->pci_device, &dev); 903 | if (unlikely(err)) { 904 | LOG_EXT(); 905 | return err; 906 | } 907 | 908 | ops = get_dma_ops(&dev->dev); 909 | 910 | state->state = ops->map_sg != swiotlb_map_sg_attrs 911 | ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE; 912 | 913 | pci_dev_put(dev); 914 | LOG_EXT(); 915 | 916 | #else 917 | /* No way to detect it */ 918 | state->state = MODS_SWIOTLB_INDETERMINATE; 919 | #endif 920 | return OK; 921 | } 922 | 923 | int esc_mods_pci_set_dma_mask(struct mods_client *client, 924 | struct MODS_PCI_DMA_MASK *dma_mask) 925 | { 926 | int err; 927 | struct pci_dev *dev; 928 | u64 mask; 929 | 930 | LOG_ENT(); 931 | 932 | if (unlikely(dma_mask->num_bits > 64)) { 933 | cl_error("num_bits=%u exceeds 64\n", 934 | dma_mask->num_bits); 935 | LOG_EXT(); 936 | return -EINVAL; 937 | } 938 | 939 | err = mods_find_pci_dev(client, &dma_mask->pci_device, &dev); 940 | if (unlikely(err)) { 941 | if (err == -ENODEV) 942 | cl_error("dev %04x:%02x:%02x.%x not found\n", 943 | dma_mask->pci_device.domain, 944 | dma_mask->pci_device.bus, 945 | dma_mask->pci_device.device, 946 | dma_mask->pci_device.function); 947 | LOG_EXT(); 948 | return err; 949 | } 950 | 951 | mask = dma_mask->num_bits == 64 ? ~0ULL : (1ULL<num_bits)-1; 952 | 953 | err = dma_set_mask(&dev->dev, mask); 954 | if (err) { 955 | cl_error( 956 | "failed to set dma mask 0x%llx (%u) for dev %04x:%02x:%02x.%x\n", 957 | mask, 958 | dma_mask->num_bits, 959 | dma_mask->pci_device.domain, 960 | dma_mask->pci_device.bus, 961 | dma_mask->pci_device.device, 962 | dma_mask->pci_device.function); 963 | #if defined(CONFIG_PPC64) 964 | /* Ignore error if TCE bypass is on */ 965 | if (dev->dma_mask == ~0ULL) 966 | err = OK; 967 | #endif 968 | } else { 969 | #if defined(MODS_HAS_SET_COHERENT_MASK) 970 | err = dma_set_coherent_mask(&dev->dev, mask); 971 | #else 972 | err = pci_set_consistent_dma_mask(dev, mask); 973 | #endif 974 | if (err) 975 | cl_error( 976 | "failed to set consistent dma mask 0x%llx (%u) for dev %04x:%02x:%02x.%x\n", 977 | mask, 978 | dma_mask->num_bits, 979 | dma_mask->pci_device.domain, 980 | dma_mask->pci_device.bus, 981 | dma_mask->pci_device.device, 982 | dma_mask->pci_device.function); 983 | } 984 | 985 | if (!err) 986 | cl_info("set dma mask %u for dev %04x:%02x:%02x.%x\n", 987 | dma_mask->num_bits, 988 | dma_mask->pci_device.domain, 989 | dma_mask->pci_device.bus, 990 | dma_mask->pci_device.device, 991 | dma_mask->pci_device.function); 992 | 993 | pci_dev_put(dev); 994 | LOG_EXT(); 995 | return err; 996 | } 997 | 998 | static int check_flr(struct pci_dev *dev) 999 | { 1000 | int cap_pos; 1001 | u32 cap; 1002 | 1003 | #if KERNEL_VERSION(4, 12, 0) <= MODS_KERNEL_VERSION 1004 | if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 1005 | return -ENOTTY; 1006 | #endif 1007 | 1008 | cap_pos = pci_find_capability(dev, PCI_CAP_ID_EXP); 1009 | if (!cap_pos) 1010 | return -ENOTTY; 1011 | 1012 | pci_read_config_dword(dev, cap_pos + PCI_EXP_DEVCAP, &cap); 1013 | if (!(cap & PCI_EXP_DEVCAP_FLR)) 1014 | return -ENOTTY; 1015 | 1016 | return 0; 1017 | } 1018 | 1019 | int esc_mods_pci_reset_function(struct mods_client *client, 1020 | struct mods_pci_dev_2 *pcidev) 1021 | { 1022 | struct pci_dev *dev; 1023 | int err; 1024 | 1025 | LOG_ENT(); 1026 | 1027 | err = mods_find_pci_dev(client, pcidev, &dev); 1028 | if (unlikely(err)) { 1029 | if (err == -ENODEV) 1030 | cl_error("dev %04x:%02x:%02x.%x not found\n", 1031 | pcidev->domain, 1032 | pcidev->bus, 1033 | pcidev->device, 1034 | pcidev->function); 1035 | LOG_EXT(); 1036 | return err; 1037 | } 1038 | 1039 | err = check_flr(dev); 1040 | if (unlikely(err)) { 1041 | cl_error( 1042 | "function level reset not supported on dev %04x:%02x:%02x.%x\n", 1043 | pcidev->domain, 1044 | pcidev->bus, 1045 | pcidev->device, 1046 | pcidev->function); 1047 | goto error; 1048 | } 1049 | 1050 | #if KERNEL_VERSION(2, 32, 0) <= MODS_KERNEL_VERSION 1051 | pm_runtime_get_sync(&dev->dev); 1052 | #endif 1053 | 1054 | err = pci_reset_function(dev); 1055 | 1056 | #if KERNEL_VERSION(2, 32, 0) <= MODS_KERNEL_VERSION 1057 | pm_runtime_put(&dev->dev); 1058 | #endif 1059 | 1060 | if (unlikely(err)) 1061 | cl_error("pcie_flr failed on dev %04x:%02x:%02x.%x\n", 1062 | pcidev->domain, 1063 | pcidev->bus, 1064 | pcidev->device, 1065 | pcidev->function); 1066 | else 1067 | cl_info("pcie_flr succeeded on dev %04x:%02x:%02x.%x\n", 1068 | pcidev->domain, 1069 | pcidev->bus, 1070 | pcidev->device, 1071 | pcidev->function); 1072 | 1073 | error: 1074 | pci_dev_put(dev); 1075 | LOG_EXT(); 1076 | return err; 1077 | } 1078 | 1079 | #ifdef MODS_HAS_DEV_PROPS 1080 | int esc_mods_read_dev_property(struct mods_client *client, 1081 | struct MODS_READ_DEV_PROPERTY *p) 1082 | { 1083 | struct pci_dev *dev = NULL; 1084 | int err = -EINVAL; 1085 | 1086 | LOG_ENT(); 1087 | 1088 | if (unlikely(p->type != MODS_PROP_TYPE_U64)) { 1089 | cl_error("invalid property type %u\n", p->type); 1090 | goto error; 1091 | } 1092 | 1093 | if (unlikely(sizeof(64) * p->array_size > sizeof(p->output))) { 1094 | cl_error("requested size %zu exceeds output array size %zu\n", 1095 | sizeof(u64) * p->array_size, 1096 | sizeof(p->output)); 1097 | goto error; 1098 | } 1099 | 1100 | if (unlikely(p->array_size == 0)) { 1101 | cl_error("invalid output array size 0\n"); 1102 | goto error; 1103 | } 1104 | 1105 | if (!memchr(p->prop_name, 0, sizeof(p->prop_name))) { 1106 | cl_error("invalid property name, misses terminating NUL\n"); 1107 | goto error; 1108 | } 1109 | 1110 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 1111 | if (unlikely(err)) { 1112 | if (err == -ENODEV) 1113 | cl_error("dev %04x:%02x:%02x.%x not found\n", 1114 | p->pci_device.domain, 1115 | p->pci_device.bus, 1116 | p->pci_device.device, 1117 | p->pci_device.function); 1118 | goto error; 1119 | } 1120 | 1121 | err = device_property_read_u64_array(&dev->dev, p->prop_name, 1122 | (u64 *)p->output, p->array_size); 1123 | if (unlikely(err)) 1124 | cl_info("failed to read property %s\n", p->prop_name); 1125 | 1126 | error: 1127 | pci_dev_put(dev); 1128 | LOG_EXT(); 1129 | return err; 1130 | } 1131 | #endif 1132 | -------------------------------------------------------------------------------- /mods_ppc64.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* SPDX-FileCopyrightText: Copyright (c) 2017-2023, NVIDIA CORPORATION. All rights reserved. */ 3 | 4 | #include "mods_internal.h" 5 | 6 | #include 7 | 8 | #ifdef MODS_HAS_PNV_PCI_GET_NPU_DEV 9 | static struct pci_dev *get_npu_dev(struct pci_dev *dev, int index) 10 | { 11 | return pnv_pci_get_npu_dev(dev, index); 12 | } 13 | #else 14 | #define get_npu_dev(dev, index) (NULL) 15 | #endif 16 | 17 | int has_npu_dev(struct pci_dev *dev, int index) 18 | { 19 | struct pci_dev *npu_dev = get_npu_dev(dev, index); 20 | 21 | /* We should call pci_dev_put(npu_dev), but it's currently crashing */ 22 | 23 | return npu_dev != NULL; 24 | } 25 | 26 | static struct NVL_TRAINED *mods_find_nvlink_sysmem_trained( 27 | struct mods_client *client, 28 | struct pci_dev *dev) 29 | { 30 | struct list_head *plist_head; 31 | struct list_head *plist_iter; 32 | struct NVL_TRAINED *p_nvl_trained; 33 | 34 | plist_head = &client->nvlink_sysmem_trained_list; 35 | 36 | list_for_each(plist_iter, plist_head) { 37 | p_nvl_trained = list_entry(plist_iter, 38 | struct NVL_TRAINED, 39 | list); 40 | if (dev == p_nvl_trained->dev) 41 | return p_nvl_trained; 42 | } 43 | 44 | /* The device has never had its dma mask changed */ 45 | return NULL; 46 | } 47 | 48 | static int mods_register_nvlink_sysmem_trained(struct mods_client *client, 49 | struct pci_dev *dev, 50 | u8 trained) 51 | { 52 | struct NVL_TRAINED *p_nvl_trained; 53 | 54 | p_nvl_trained = mods_find_nvlink_sysmem_trained(client, dev); 55 | if (p_nvl_trained != NULL) { 56 | p_nvl_trained->trained = trained; 57 | return OK; 58 | } 59 | 60 | if (unlikely(mutex_lock_interruptible(&client->mtx))) 61 | return -EINTR; 62 | 63 | p_nvl_trained = kzalloc(sizeof(struct NVL_TRAINED), 64 | GFP_KERNEL | __GFP_NORETRY); 65 | if (unlikely(!p_nvl_trained)) { 66 | cl_error("failed to allocate NvLink trained struct\n"); 67 | LOG_EXT(); 68 | return -ENOMEM; 69 | } 70 | atomic_inc(&client->num_allocs); 71 | 72 | p_nvl_trained->dev = pci_dev_get(dev); 73 | p_nvl_trained->trained = trained; 74 | 75 | list_add(&p_nvl_trained->list, 76 | &client->nvlink_sysmem_trained_list); 77 | 78 | cl_debug(DEBUG_MEM, 79 | "registered NvLink trained on dev %04x:%02x:%02x.%x\n", 80 | pci_domain_nr(dev->bus), 81 | dev->bus->number, 82 | PCI_SLOT(dev->devfn), 83 | PCI_FUNC(dev->devfn)); 84 | mutex_unlock(&client->mtx); 85 | return OK; 86 | } 87 | 88 | static int mods_unregister_nvlink_sysmem_trained(struct mods_client *client, 89 | struct pci_dev *dev) 90 | { 91 | struct NVL_TRAINED *p_nvl_trained; 92 | struct list_head *head = &client->nvlink_sysmem_trained_list; 93 | struct list_head *iter; 94 | 95 | LOG_ENT(); 96 | 97 | if (unlikely(mutex_lock_interruptible(&client->mtx))) 98 | return -EINTR; 99 | 100 | list_for_each(iter, head) { 101 | p_nvl_trained = 102 | list_entry(iter, struct NVL_TRAINED, list); 103 | 104 | if (p_nvl_trained->dev == dev) { 105 | list_del(iter); 106 | 107 | mutex_unlock(&client->mtx); 108 | 109 | cl_debug(DEBUG_MEM, 110 | "unregistered NvLink trained on dev %04x:%02x:%02x.%x\n", 111 | pci_domain_nr(p_nvl_trained->dev->bus), 112 | p_nvl_trained->dev->bus->number, 113 | PCI_SLOT(p_nvl_trained->dev->devfn), 114 | PCI_FUNC(p_nvl_trained->dev->devfn)); 115 | 116 | pci_dev_put(dev); 117 | 118 | kfree(p_nvl_trained); 119 | atomic_dec(&client->num_allocs); 120 | 121 | LOG_EXT(); 122 | return OK; 123 | } 124 | } 125 | 126 | mutex_unlock(&client->mtx); 127 | 128 | cl_error( 129 | "failed to unregister NvLink trained on dev %04x:%02x:%02x.%x\n", 130 | pci_domain_nr(dev->bus), 131 | dev->bus->number, 132 | PCI_SLOT(dev->devfn), 133 | PCI_FUNC(dev->devfn)); 134 | LOG_EXT(); 135 | 136 | return -EINVAL; 137 | 138 | } 139 | 140 | int mods_unregister_all_nvlink_sysmem_trained(struct mods_client *client) 141 | { 142 | struct list_head *head = &client->nvlink_sysmem_trained_list; 143 | struct list_head *iter; 144 | struct list_head *tmp; 145 | 146 | list_for_each_safe(iter, tmp, head) { 147 | struct NVL_TRAINED *p_nvl_trained; 148 | int err; 149 | 150 | p_nvl_trained = 151 | list_entry(iter, struct NVL_TRAINED, list); 152 | err = mods_unregister_nvlink_sysmem_trained(client, 153 | p_nvl_trained->dev); 154 | if (err) 155 | return err; 156 | } 157 | 158 | return OK; 159 | } 160 | 161 | int mods_is_nvlink_sysmem_trained(struct mods_client *client, 162 | struct pci_dev *dev) 163 | { 164 | struct NVL_TRAINED *p_nvl_trained; 165 | 166 | p_nvl_trained = mods_find_nvlink_sysmem_trained(client, dev); 167 | if (p_nvl_trained != NULL) 168 | return p_nvl_trained->trained; 169 | 170 | return false; 171 | } 172 | 173 | int esc_mods_set_nvlink_sysmem_trained(struct mods_client *client, 174 | struct MODS_SET_NVLINK_SYSMEM_TRAINED *p) 175 | { 176 | struct pci_dev *dev; 177 | int err; 178 | 179 | LOG_ENT(); 180 | 181 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 182 | if (unlikely(err)) { 183 | if (err == -ENODEV) 184 | cl_error("dev %04x:%02x:%02x.%x not found\n", 185 | p->pci_device.domain, 186 | p->pci_device.bus, 187 | p->pci_device.device, 188 | p->pci_device.function); 189 | LOG_EXT(); 190 | return err; 191 | } 192 | 193 | err = mods_register_nvlink_sysmem_trained(client, dev, p->trained); 194 | 195 | pci_dev_put(dev); 196 | LOG_EXT(); 197 | return err; 198 | } 199 | 200 | static struct PPC_TCE_BYPASS *mods_find_ppc_tce_bypass( 201 | struct mods_client *client, 202 | struct pci_dev *dev) 203 | { 204 | struct list_head *plist_head; 205 | struct list_head *plist_iter; 206 | struct PPC_TCE_BYPASS *p_ppc_tce_bypass; 207 | 208 | plist_head = &client->ppc_tce_bypass_list; 209 | 210 | list_for_each(plist_iter, plist_head) { 211 | p_ppc_tce_bypass = list_entry(plist_iter, 212 | struct PPC_TCE_BYPASS, 213 | list); 214 | if (dev == p_ppc_tce_bypass->dev) 215 | return p_ppc_tce_bypass; 216 | } 217 | 218 | /* The device has never had its dma mask changed */ 219 | return NULL; 220 | } 221 | 222 | static int mods_register_ppc_tce_bypass(struct mods_client *client, 223 | struct pci_dev *dev, 224 | u64 original_mask) 225 | { 226 | struct PPC_TCE_BYPASS *p_ppc_tce_bypass; 227 | 228 | /* only register the first time in order to restore the true actual dma 229 | * mask 230 | */ 231 | if (mods_find_ppc_tce_bypass(client, dev) != NULL) { 232 | cl_debug(DEBUG_MEM, 233 | "TCE bypass already registered on dev %04x:%02x:%02x.%x\n", 234 | pci_domain_nr(dev->bus), 235 | dev->bus->number, 236 | PCI_SLOT(dev->devfn), 237 | PCI_FUNC(dev->devfn)); 238 | return OK; 239 | } 240 | 241 | if (unlikely(mutex_lock_interruptible(&client->mtx))) 242 | return -EINTR; 243 | 244 | p_ppc_tce_bypass = kzalloc(sizeof(struct PPC_TCE_BYPASS), 245 | GFP_KERNEL | __GFP_NORETRY); 246 | if (unlikely(!p_ppc_tce_bypass)) { 247 | cl_error("failed to allocate TCE bypass struct\n"); 248 | LOG_EXT(); 249 | return -ENOMEM; 250 | } 251 | atomic_inc(&client->num_allocs); 252 | 253 | p_ppc_tce_bypass->dev = pci_dev_get(dev); 254 | p_ppc_tce_bypass->dma_mask = original_mask; 255 | 256 | list_add(&p_ppc_tce_bypass->list, 257 | &client->ppc_tce_bypass_list); 258 | 259 | cl_debug(DEBUG_MEM, 260 | "registered TCE bypass on dev %04x:%02x:%02x.%x\n", 261 | pci_domain_nr(dev->bus), 262 | dev->bus->number, 263 | PCI_SLOT(dev->devfn), 264 | PCI_FUNC(dev->devfn)); 265 | mutex_unlock(&client->mtx); 266 | return OK; 267 | } 268 | 269 | static int mods_unregister_ppc_tce_bypass(struct mods_client *client, 270 | struct pci_dev *dev) 271 | { 272 | struct PPC_TCE_BYPASS *p_ppc_tce_bypass; 273 | struct list_head *head = &client->ppc_tce_bypass_list; 274 | struct list_head *iter; 275 | 276 | LOG_ENT(); 277 | 278 | if (unlikely(mutex_lock_interruptible(&client->mtx))) 279 | return -EINTR; 280 | 281 | list_for_each(iter, head) { 282 | p_ppc_tce_bypass = 283 | list_entry(iter, struct PPC_TCE_BYPASS, list); 284 | 285 | if (p_ppc_tce_bypass->dev == dev) { 286 | int err = -EINVAL; 287 | 288 | list_del(iter); 289 | 290 | mutex_unlock(&client->mtx); 291 | 292 | err = pci_set_dma_mask(p_ppc_tce_bypass->dev, 293 | p_ppc_tce_bypass->dma_mask); 294 | dma_set_coherent_mask(&p_ppc_tce_bypass->dev->dev, 295 | dev->dma_mask); 296 | cl_debug(DEBUG_MEM, 297 | "restored dma_mask on dev %04x:%02x:%02x.%x to %llx\n", 298 | pci_domain_nr(p_ppc_tce_bypass->dev->bus), 299 | p_ppc_tce_bypass->dev->bus->number, 300 | PCI_SLOT(p_ppc_tce_bypass->dev->devfn), 301 | PCI_FUNC(p_ppc_tce_bypass->dev->devfn), 302 | p_ppc_tce_bypass->dma_mask); 303 | 304 | pci_dev_put(dev); 305 | 306 | kfree(p_ppc_tce_bypass); 307 | atomic_dec(&client->num_allocs); 308 | 309 | LOG_EXT(); 310 | return err; 311 | } 312 | } 313 | 314 | mutex_unlock(&client->mtx); 315 | 316 | cl_error("failed to unregister TCE bypass on dev %04x:%02x:%02x.%x\n", 317 | pci_domain_nr(dev->bus), 318 | dev->bus->number, 319 | PCI_SLOT(dev->devfn), 320 | PCI_FUNC(dev->devfn)); 321 | LOG_EXT(); 322 | 323 | return -EINVAL; 324 | 325 | } 326 | 327 | int mods_unregister_all_ppc_tce_bypass(struct mods_client *client) 328 | { 329 | int err = OK; 330 | struct list_head *head = &client->ppc_tce_bypass_list; 331 | struct list_head *iter; 332 | struct list_head *tmp; 333 | 334 | list_for_each_safe(iter, tmp, head) { 335 | struct PPC_TCE_BYPASS *p_ppc_tce_bypass; 336 | 337 | p_ppc_tce_bypass = 338 | list_entry(iter, struct PPC_TCE_BYPASS, list); 339 | err = mods_unregister_ppc_tce_bypass(client, 340 | p_ppc_tce_bypass->dev); 341 | if (err) 342 | break; 343 | } 344 | 345 | return err; 346 | } 347 | 348 | int esc_mods_set_ppc_tce_bypass(struct mods_client *client, 349 | struct MODS_SET_PPC_TCE_BYPASS *p) 350 | { 351 | int err = OK; 352 | dma_addr_t dma_addr; 353 | struct pci_dev *dev; 354 | u64 original_dma_mask; 355 | u32 bypass_mode = p->mode; 356 | u32 cur_bypass_mode = MODS_PPC_TCE_BYPASS_OFF; 357 | u64 dma_mask = DMA_BIT_MASK(64); 358 | 359 | LOG_ENT(); 360 | 361 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 362 | if (unlikely(err)) { 363 | if (err == -ENODEV) 364 | cl_error("dev %04x:%02x:%02x.%x not found\n", 365 | p->pci_device.domain, 366 | p->pci_device.bus, 367 | p->pci_device.device, 368 | p->pci_device.function); 369 | LOG_EXT(); 370 | return -EINVAL; 371 | } 372 | 373 | original_dma_mask = dev->dma_mask; 374 | 375 | if (bypass_mode == MODS_PPC_TCE_BYPASS_DEFAULT) 376 | bypass_mode = mods_get_ppc_tce_bypass(); 377 | 378 | if (original_dma_mask == DMA_BIT_MASK(64)) 379 | cur_bypass_mode = MODS_PPC_TCE_BYPASS_ON; 380 | 381 | /* 382 | * Linux on IBM POWER8 offers 2 different DMA set-ups, sometimes 383 | * referred to as "windows". 384 | * 385 | * The "default window" provides a 2GB region of PCI address space 386 | * located below the 32-bit line. The IOMMU is used to provide a 387 | * "rich" mapping--any page in system memory can be mapped at an 388 | * arbitrary address within this window. The mappings are dynamic 389 | * and pass in and out of being as pci_map*()/pci_unmap*() calls 390 | * are made. 391 | * 392 | * Dynamic DMA Windows (sometimes "Huge DDW", also PPC TCE Bypass "ON") 393 | * provides a linear 394 | * mapping of the system's entire physical address space at some 395 | * fixed offset above the 59-bit line. IOMMU is still used, and 396 | * pci_map*()/pci_unmap*() are still required, but mappings are 397 | * static. They're effectively set up in advance, and any given 398 | * system page will always map to the same PCI bus address. I.e. 399 | * physical 0x00000000xxxxxxxx => PCI 0x08000000xxxxxxxx 400 | * 401 | * Linux on POWER8 will only provide the DDW-style full linear 402 | * mapping when the driver claims support for 64-bit DMA addressing 403 | * (a pre-requisite because the PCI addresses used in this case will 404 | * be near the top of the 64-bit range). The linear mapping 405 | * is not available in all system configurations. 406 | * 407 | * Detect whether the linear mapping is present by claiming 408 | * 64-bit support and then mapping physical page 0. For historical 409 | * reasons, Linux on POWER8 will never map a page to PCI address 0x0. 410 | * In the "default window" case page 0 will be mapped to some 411 | * non-zero address below the 32-bit line. In the 412 | * DDW/linear-mapping case, it will be mapped to address 0 plus 413 | * some high-order offset. 414 | * 415 | * If the linear mapping is present and sane then return the offset 416 | * as the starting address for all DMA mappings. 417 | */ 418 | if ((bypass_mode != MODS_PPC_TCE_BYPASS_DEFAULT) && 419 | (cur_bypass_mode != bypass_mode)) { 420 | /* Set DMA mask appropriately here */ 421 | if (bypass_mode == MODS_PPC_TCE_BYPASS_OFF) 422 | dma_mask = p->device_dma_mask; 423 | 424 | err = pci_set_dma_mask(dev, dma_mask); 425 | if (unlikely(err)) { 426 | cl_error( 427 | "pci_set_dma_mask failed on dev %04x:%02x:%02x.%x\n", 428 | p->pci_device.domain, 429 | p->pci_device.bus, 430 | p->pci_device.device, 431 | p->pci_device.function); 432 | pci_dev_put(dev); 433 | LOG_EXT(); 434 | return err; 435 | } 436 | } 437 | 438 | dma_addr = pci_map_single(dev, NULL, 1, DMA_BIDIRECTIONAL); 439 | err = pci_dma_mapping_error(dev, dma_addr); 440 | if (unlikely(err)) { 441 | pci_set_dma_mask(dev, original_dma_mask); 442 | cl_error("pci_map_single failed on dev %04x:%02x:%02x.%x\n", 443 | p->pci_device.domain, 444 | p->pci_device.bus, 445 | p->pci_device.device, 446 | p->pci_device.function); 447 | pci_dev_put(dev); 448 | LOG_EXT(); 449 | return err; 450 | } 451 | pci_unmap_single(dev, dma_addr, 1, DMA_BIDIRECTIONAL); 452 | 453 | if (bypass_mode == MODS_PPC_TCE_BYPASS_ON) { 454 | int failed = false; 455 | 456 | /* 457 | * From IBM: "For IODA2, native DMA bypass or KVM TCE-based 458 | * implementation of full 64-bit DMA support will establish a 459 | * window in address-space with the high 14 bits being constant 460 | * and the bottom up-to-50 bits varying with the mapping." 461 | * 462 | * Unfortunately, we don't have any good interfaces or 463 | * definitions from the kernel to get information about the DMA 464 | * offset assigned by OS. However, we have been told that the 465 | * offset will be defined by the top 14 bits of the address, 466 | * and bits 40-49 will not vary for any DMA mappings until 1TB 467 | * of system memory is surpassed; this limitation is essential 468 | * for us to function properly since our current GPUs only 469 | * support 40 physical address bits. We are in a fragile place 470 | * where we need to tell the OS that we're capable of 64-bit 471 | * addressing, while relying on the assumption that the top 24 472 | * bits will not vary in this case. 473 | * 474 | * The way we try to compute the window, then, is mask the trial 475 | * mapping against the DMA capabilities of the device. That way, 476 | * devices with greater addressing capabilities will only take 477 | * the bits it needs to define the window. 478 | */ 479 | if ((dma_addr & DMA_BIT_MASK(32)) != 0) { 480 | /* 481 | * Huge DDW not available - page 0 mapped to non-zero 482 | * address below the 32-bit line. 483 | */ 484 | cl_warn("enabling PPC TCE bypass mode failed due to platform on dev %04x:%02x:%02x.%x\n", 485 | p->pci_device.domain, 486 | p->pci_device.bus, 487 | p->pci_device.device, 488 | p->pci_device.function); 489 | failed = true; 490 | } else if ((dma_addr & original_dma_mask) != 0) { 491 | /* 492 | * The physical window straddles our addressing limit 493 | * boundary, e.g., for an adapter that can address up to 494 | * 1TB, the window crosses the 40-bit limit so that the 495 | * lower end of the range has different bits 63:40 than 496 | * the higher end of the range. We can only handle a 497 | * single, static value for bits 63:40, so we must fall 498 | * back here. 499 | */ 500 | u64 memory_size = get_num_physpages() * PAGE_SIZE; 501 | 502 | if ((dma_addr & ~original_dma_mask) != 503 | ((dma_addr + memory_size) & ~original_dma_mask)) { 504 | 505 | cl_warn("enabling PPC TCE bypass mode failed due to memory size on dev %04x:%02x:%02x.%x\n", 506 | p->pci_device.domain, 507 | p->pci_device.bus, 508 | p->pci_device.device, 509 | p->pci_device.function); 510 | failed = true; 511 | } 512 | } 513 | if (failed) 514 | pci_set_dma_mask(dev, original_dma_mask); 515 | } 516 | 517 | cl_debug(DEBUG_MEM, 518 | "%s ppc tce bypass on dev %04x:%02x:%02x.%x with dma mask 0x%llx\n", 519 | (dev->dma_mask == DMA_BIT_MASK(64)) ? "enabled" : "disabled", 520 | p->pci_device.domain, 521 | p->pci_device.bus, 522 | p->pci_device.device, 523 | p->pci_device.function, 524 | dev->dma_mask); 525 | 526 | p->dma_base_address = dma_addr & ~(p->device_dma_mask); 527 | 528 | cl_debug(DEBUG_MEM, 529 | "dma base address 0x%0llx on dev %04x:%02x:%02x.%x\n", 530 | p->dma_base_address, 531 | p->pci_device.domain, 532 | p->pci_device.bus, 533 | p->pci_device.device, 534 | p->pci_device.function); 535 | 536 | /* Update the coherent mask to match */ 537 | dma_set_coherent_mask(&dev->dev, dev->dma_mask); 538 | 539 | if (original_dma_mask != dev->dma_mask) 540 | err = mods_register_ppc_tce_bypass(client, 541 | dev, 542 | original_dma_mask); 543 | 544 | pci_dev_put(dev); 545 | LOG_EXT(); 546 | return err; 547 | } 548 | 549 | int esc_mods_get_ats_address_range(struct mods_client *client, 550 | struct MODS_GET_ATS_ADDRESS_RANGE *p) 551 | { 552 | struct pci_dev *dev = NULL; 553 | struct pci_dev *npu_dev = NULL; 554 | struct device_node *mem_node = NULL; 555 | const __u32 *val32 = NULL; 556 | const __u64 *val64; 557 | int len; 558 | int err = -EINVAL; 559 | 560 | LOG_ENT(); 561 | 562 | cl_debug(DEBUG_PCI, 563 | "get ats addr, dev %04x:%02x:%02x:%x, npu index %d\n", 564 | p->pci_device.domain, 565 | p->pci_device.bus, 566 | p->pci_device.device, 567 | p->pci_device.function, 568 | p->npu_index); 569 | 570 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 571 | if (unlikely(err)) { 572 | if (err == -ENODEV) 573 | cl_error("dev %04x:%02x:%02x.%x not found\n", 574 | p->pci_device.domain, 575 | p->pci_device.bus, 576 | p->pci_device.device, 577 | p->pci_device.function); 578 | goto exit; 579 | } 580 | 581 | err = -ENODEV; 582 | 583 | npu_dev = get_npu_dev(dev, p->npu_index); 584 | if (unlikely(npu_dev == NULL)) { 585 | cl_error("NPU device for dev %04x:%02x:%02x.%x not found\n", 586 | p->pci_device.domain, 587 | p->pci_device.bus, 588 | p->pci_device.device, 589 | p->pci_device.function); 590 | goto exit; 591 | } 592 | 593 | p->npu_device.domain = pci_domain_nr(npu_dev->bus); 594 | p->npu_device.bus = npu_dev->bus->number; 595 | p->npu_device.device = PCI_SLOT(npu_dev->devfn); 596 | p->npu_device.function = PCI_FUNC(npu_dev->devfn); 597 | 598 | cl_debug(DEBUG_PCI, 599 | "found NPU device %04x:%02x:%02x.%x\n", 600 | p->npu_device.domain, 601 | p->npu_device.bus, 602 | p->npu_device.device, 603 | p->npu_device.function); 604 | 605 | if (npu_dev->dev.of_node) 606 | val32 = (const __u32 *)of_get_property(npu_dev->dev.of_node, 607 | "memory-region", 608 | &len); 609 | if (!val32 || len < 4) { 610 | cl_error("property memory-region for NPU not found\n"); 611 | goto exit; 612 | } 613 | 614 | mem_node = of_find_node_by_phandle(be32_to_cpu(*val32)); 615 | if (!mem_node) { 616 | cl_error("node memory-region for NPU not found\n"); 617 | goto exit; 618 | } 619 | 620 | p->numa_memory_node = of_node_to_nid(mem_node); 621 | if (p->numa_memory_node == NUMA_NO_NODE) { 622 | cl_error("NUMA node for NPU not found\n"); 623 | goto exit; 624 | } 625 | 626 | val64 = (const __u64 *)of_get_property(npu_dev->dev.of_node, 627 | "ibm,device-tgt-addr", 628 | &len); 629 | if (!val64 || len < 8) { 630 | cl_error("property ibm,device-tgt-addr for NPU not found\n"); 631 | goto exit; 632 | } 633 | 634 | p->phys_addr = be64_to_cpu(*val64); 635 | 636 | val64 = (const __u64 *)of_get_property(mem_node, "reg", &len); 637 | if (!val64 || len < 16) { 638 | cl_error("property reg for memory region not found\n"); 639 | goto exit; 640 | } 641 | 642 | p->guest_addr = be64_to_cpu(val64[0]); 643 | p->aperture_size = be64_to_cpu(val64[1]); 644 | 645 | err = OK; 646 | 647 | exit: 648 | of_node_put(mem_node); 649 | /* We should call pci_dev_put(npu_dev), but it's currently crashing */ 650 | pci_dev_put(dev); 651 | LOG_EXT(); 652 | return err; 653 | } 654 | 655 | int esc_mods_get_nvlink_line_rate(struct mods_client *client, 656 | struct MODS_GET_NVLINK_LINE_RATE *p) 657 | { 658 | struct pci_dev *dev = NULL; 659 | struct pci_dev *npu_dev = NULL; 660 | const __u32 *val32 = NULL; 661 | int len; 662 | int err = -EINVAL; 663 | 664 | LOG_ENT(); 665 | 666 | cl_debug(DEBUG_PCI, 667 | "get nvlink speed, dev %04x:%02x:%02x.%x, npu index %d\n", 668 | p->pci_device.domain, 669 | p->pci_device.bus, 670 | p->pci_device.device, 671 | p->pci_device.function, 672 | p->npu_index); 673 | 674 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 675 | if (unlikely(err)) { 676 | if (err == -ENODEV) 677 | cl_error("dev %04x:%02x:%02x.%x not found\n", 678 | p->pci_device.domain, 679 | p->pci_device.bus, 680 | p->pci_device.device, 681 | p->pci_device.function); 682 | goto exit; 683 | } 684 | 685 | err = -ENODEV; 686 | 687 | npu_dev = get_npu_dev(dev, p->npu_index); 688 | if (unlikely(npu_dev == NULL)) { 689 | cl_error("NPU device for dev %04x:%02x:%02x.%x not found\n", 690 | p->pci_device.domain, 691 | p->pci_device.bus, 692 | p->pci_device.device, 693 | p->pci_device.function); 694 | goto exit; 695 | } 696 | 697 | cl_debug(DEBUG_PCI, 698 | "found NPU device %04x:%02x:%02x.%x\n", 699 | pci_domain_nr(npu_dev->bus), 700 | npu_dev->bus->number, 701 | PCI_SLOT(npu_dev->devfn), 702 | PCI_FUNC(npu_dev->devfn)); 703 | 704 | if (npu_dev->dev.of_node) 705 | val32 = (const __u32 *)of_get_property(npu_dev->dev.of_node, 706 | "ibm,nvlink-speed", 707 | &len); 708 | if (!val32) { 709 | cl_error("property ibm,nvlink-speed for NPU not found\n"); 710 | goto exit; 711 | } 712 | 713 | p->speed = be32_to_cpup(val32); 714 | if (!p->speed) { 715 | cl_error("ibm,nvlink-speed value for NPU not valid\n"); 716 | goto exit; 717 | } 718 | 719 | err = OK; 720 | 721 | exit: 722 | /* We should call pci_dev_put(npu_dev), but it's currently crashing */ 723 | pci_dev_put(dev); 724 | LOG_EXT(); 725 | return err; 726 | } 727 | 728 | int esc_mods_pci_hot_reset(struct mods_client *client, 729 | struct MODS_PCI_HOT_RESET *p) 730 | { 731 | struct pci_dev *dev; 732 | int err; 733 | 734 | LOG_ENT(); 735 | 736 | cl_debug(DEBUG_PCI, 737 | "pci_hot_reset dev %04x:%02x:%02x.%x\n", 738 | p->pci_device.domain, 739 | p->pci_device.bus, 740 | p->pci_device.device, 741 | p->pci_device.function); 742 | 743 | err = mods_find_pci_dev(client, &p->pci_device, &dev); 744 | if (unlikely(err)) { 745 | if (err == -ENODEV) 746 | cl_error( 747 | "pci_hot_reset cannot find dev %04x:%02x:%02x.%x\n", 748 | p->pci_device.domain, 749 | p->pci_device.bus, 750 | p->pci_device.device, 751 | p->pci_device.function); 752 | LOG_EXT(); 753 | return err; 754 | } 755 | 756 | err = pci_set_pcie_reset_state(dev, pcie_hot_reset); 757 | if (unlikely(err)) 758 | cl_error("pci_hot_reset failed on dev %04x:%02x:%02x.%x\n", 759 | p->pci_device.domain, 760 | p->pci_device.bus, 761 | p->pci_device.device, 762 | p->pci_device.function); 763 | else { 764 | 765 | err = pci_set_pcie_reset_state(dev, pcie_deassert_reset); 766 | if (unlikely(err)) 767 | cl_error( 768 | "pci_hot_reset deassert failed on dev %04x:%02x:%02x.%x\n", 769 | p->pci_device.domain, 770 | p->pci_device.bus, 771 | p->pci_device.device, 772 | p->pci_device.function); 773 | } 774 | 775 | pci_dev_put(dev); 776 | LOG_EXT(); 777 | return err; 778 | } 779 | --------------------------------------------------------------------------------