├── .gitignore ├── LICENSE ├── NOTICE ├── dynmx.py ├── dynmx ├── __init__.py ├── config │ ├── aam_cuckoo.yaml │ └── aam_vmray.yaml ├── converters │ ├── __init__.py │ ├── dynmx_converter.py │ └── dynmx_harmonizer.py ├── core │ ├── __init__.py │ ├── api_call.py │ ├── file_resource.py │ ├── function_log.py │ ├── network_resource.py │ ├── pointer.py │ ├── process.py │ ├── registry_resource.py │ ├── resource.py │ └── statistics.py ├── detection │ ├── __init__.py │ ├── access_activity_model.py │ ├── detection_result.py │ ├── detection_step.py │ ├── graph.py │ └── signature.py ├── flog_parsers │ ├── __init__.py │ ├── cape_flog_parser.py │ ├── cuckoo_flog_parser.py │ ├── dynmx_flog_parser.py │ ├── parser.py │ ├── parser_library.py │ ├── vmray_flog_parser.py │ └── vmray_xml_flog_parser.py └── helpers │ ├── __init__.py │ ├── argument_helper.py │ ├── flog_parser_helper.py │ ├── logging_globals.py │ ├── logging_helper.py │ ├── multiprocessing_helper.py │ ├── output_helper.py │ ├── regex_helper.py │ └── resource_helper.py ├── readme.md └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # General 2 | .DS_Store 3 | .AppleDouble 4 | .LSOverride 5 | 6 | # Icon must end with two \r 7 | Icon 8 | 9 | # Thumbnails 10 | ._* 11 | 12 | # Files that might appear in the root of a volume 13 | .DocumentRevisions-V100 14 | .fseventsd 15 | .Spotlight-V100 16 | .TemporaryItems 17 | .Trashes 18 | .VolumeIcon.icns 19 | .com.apple.timemachine.donotpresent 20 | 21 | # Directories potentially created on remote AFP share 22 | .AppleDB 23 | .AppleDesktop 24 | Network Trash Folder 25 | Temporary Items 26 | .apdisk 27 | 28 | # PyCharm 29 | .idea 30 | 31 | # Python 32 | *.pyc 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | dynmx 2 | Copyright 2023 Simon Jansen, 0x534a <0x534a@mailbox.org> -------------------------------------------------------------------------------- /dynmx/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /dynmx/config/aam_cuckoo.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Configuration file for the Access Activity Model (Cuckoo and CAPE sandbox) 19 | 20 | # -------------------------------------------- 21 | # Filesystem 22 | # -------------------------------------------- 23 | filesystem: 24 | # Kernel system calls for filesystem 25 | - api_call_name: "^(Nt|Zw)CreateFile$" 26 | handle_returned_in: "FileHandle" 27 | path_param: "FileName" 28 | operation: "create" 29 | - api_call_name: "NtCreatePagingFile$" 30 | path_param: "PageFileName" 31 | operation: "write" 32 | - api_call_name: "^(Nt|Zw)DeleteFile$" 33 | path_param: "FileName" 34 | operation: "delete" 35 | - api_call_name: "^(Nt|Zw)FlushBuffersFile$" 36 | handle_param: "FileHandle" 37 | operation: "write" 38 | - api_call_name: "^(Nt|Zw)LockFile$" 39 | handle_param: "FileHandle" 40 | operation: "read" 41 | - api_call_name: "^(Nt|Zw)UnlockFile$" 42 | handle_param: "FileHandle" 43 | operation: "read" 44 | - api_call_name: "^(Nt|Zw)OpenFile$" 45 | handle_returned_in: "FileHandle" 46 | path_param: "FileName" 47 | operation: "read" 48 | - api_call_name: "^(Nt|Zw)OpenProcess$" 49 | path_param: "FileName" 50 | operation: "execute" 51 | - api_call_name: "^(Nt|Zw)QueryAttributesFile$" 52 | path_param: "FileName" 53 | operation: "read" 54 | - api_call_name: "^(Nt|Zw)QueryDirectoryFile$" 55 | handle_param: "FileHandle" 56 | operation: "read" 57 | - api_call_name: "^(Nt|Zw)QueryEaFile$" 58 | handle_param: "FileHandle" 59 | operation: "read" 60 | - api_call_name: "^(Nt|Zw)QueryAttributesFile$" 61 | path_param: "FileName" 62 | operation: "read" 63 | - api_call_name: "^(Nt|Zw)QueryFullAttributesFile$" 64 | path_param: "FileName" 65 | operation: "read" 66 | - api_call_name: "^(Nt|Zw)QueryInformationFile$" 67 | handle_param: "FileHandle" 68 | operation: "read" 69 | - api_call_name: "^(Nt|Zw)QueryInformationFile$" 70 | handle_param: "FileHandle" 71 | operation: "read" 72 | - api_call_name: "^(Nt|Zw)QueryQuotaInformationFile$" 73 | handle_param: "FileHandle" 74 | operation: "read" 75 | - api_call_name: "^(Nt|Zw)QueryVolumeInformationFile$" 76 | handle_param: "FileHandle" 77 | operation: "read" 78 | - api_call_name: "^(Nt|Zw)ReadFile$" 79 | handle_param: "FileHandle" 80 | operation: "read" 81 | - api_call_name: "^(Nt|Zw)ReadFileScatter$" 82 | handle_param: "FileHandle" 83 | operation: "read" 84 | - api_call_name: "^(Nt|Zw)SetEaFile$" 85 | handle_param: "FileHandle" 86 | operation: "write" 87 | - api_call_name: "^(Nt|Zw)SetInformationFile$" 88 | handle_param: "FileHandle" 89 | operation: "write" 90 | - api_call_name: "^(Nt|Zw)SetQuotaInformationFile$" 91 | handle_param: "FileHandle" 92 | operation: "write" 93 | - api_call_name: "^(Nt|Zw)SetVolumeInformationFile$" 94 | handle_param: "FileHandle" 95 | operation: "write" 96 | - api_call_name: "^(Nt|Zw)WriteFile$" 97 | handle_param: "FileHandle" 98 | operation: "write" 99 | - api_call_name: "^(Nt|Zw)WriteFileGather$" 100 | handle_param: "FileHandle" 101 | operation: "write" 102 | - api_call_name: "LdrLoadDll" 103 | path_param: "FileName" 104 | operation: "execute" 105 | - api_call_name: "^LoadLibrary(Ex|)[AW]$" 106 | path_param: "LibFileName" 107 | handle_returned_in: "return" 108 | operation: "execute" 109 | # Userland API calls for filesystem 110 | - api_call_name: "^CreateFile[AW]$" 111 | handle_returned_in: "return" 112 | path_param: "FileName" 113 | operation: "create" 114 | - api_call_name: "fopen" 115 | handle_returned_in: "return" 116 | path_param: "FileName" 117 | operation: "read" 118 | - api_call_name: "ReadFile" 119 | handle_param: "FileHandle" 120 | operation: "read" 121 | - api_call_name: "WriteFile" 122 | handle_param: "FileHandle" 123 | operation: "write" 124 | - api_call_name: "^CreateProcess[AW]$" 125 | path_param: "ApplicationName" 126 | cmd_line_param: "CommandLine" 127 | operation: "execute" 128 | - api_call_name: "^CopyFile(Ex|)[AW]$" 129 | path_param: "ExistingFileName" 130 | operation: "read" 131 | - api_call_name: "^CopyFile(Ex|)[AW]$" 132 | path_param: "NewFileName" 133 | operation: "create" 134 | - api_call_name: "^CreateFileMapping[AW]$" 135 | handle_param: "FileHandle" 136 | operation: "read" 137 | - api_call_name: "^DeleteFile[AW]$" 138 | path_param: "FileName" 139 | operation: "delete" 140 | - api_call_name: "FlushFileBuffers" 141 | handle_param: "FileHandle" 142 | operation: "write" 143 | - api_call_name: "^GetFileAttributes(Ex|)[AW]$" 144 | path_param: "FileName" 145 | operation: "read" 146 | - api_call_name: "^GetFileSize(Ex|)$" 147 | handle_param: "FileHandle" 148 | operation: "read" 149 | - api_call_name: "GetFileTime" 150 | handle_param: "FileHandle" 151 | operation: "read" 152 | - api_call_name: "GetFileType" 153 | handle_param: "FileHandle" 154 | operation: "read" 155 | - api_call_name: "^GetFileVersionInfo[AW]$" 156 | path_param: "FileName" 157 | operation: "read" 158 | - api_call_name: "^GetFileVersionInfoSize[AW]$" 159 | path_param: "FileName" 160 | operation: "read" 161 | - api_call_name: "^GetFileVersionInfoSizeEx[AW]$" 162 | path_param: "FileName" 163 | operation: "read" 164 | - api_call_name: "^GetModuleFileName(Ex|)[AW]$" 165 | path_param: "FileName" 166 | operation: "read" 167 | - api_call_name: "^GetMappedFileName[AW]$" 168 | path_param: "Filename" 169 | operation: "read" 170 | - api_call_name: "^GetPrivateProfile(Int|String)[AW]$" 171 | path_param: "Filename" 172 | operation: "read" 173 | - api_call_name: "^GetPrivateProfileSection(Names|)[AW]$" 174 | path_param: "Filename" 175 | operation: "read" 176 | - api_call_name: "^GetTempFileName[AW]$" 177 | path_param: "TempFileName" 178 | operation: "create" 179 | - api_call_name: "ICreateErrorInfo:SetHelpFile" 180 | path_param: "HelpFile" 181 | operation: "read" 182 | - api_call_name: "ICreateTypeLib:SetHelpFileName" 183 | path_param: "HelpFileName" 184 | operation: "read" 185 | - api_call_name: "IErrorInfo:GetHelpFile" 186 | path_param: "HelpFile" 187 | operation: "read" 188 | - api_call_name: "IoCreateFile" 189 | handle_returned_in: "FileHandle" 190 | path_param: "ObjectName" 191 | operation: "create" 192 | - api_call_name: "IFileOperation:CopyItem" 193 | handle_param: "Item" 194 | operation: "read" 195 | - api_call_name: "IFileOperation:CopyItem" 196 | handle_param: "DestinationFolder" 197 | file_name_param: "CopyName" 198 | operation: "create" 199 | - api_call_name: "^(Lock|Unlock)File(Ex|)$" 200 | handle_param: "FileHandle" 201 | operation: "read" 202 | - api_call_name: "SetEndOfFile" 203 | handle_param: "FileHandle" 204 | operation: "write" 205 | - api_call_name: "SetFileTime" 206 | handle_param: "FileHandle" 207 | operation: "write" 208 | - api_call_name: "^MoveFile(Ex|)[AW]$" 209 | path_param: "ExistingFileName" 210 | operation: "delete" 211 | - api_call_name: "^MoveFile(Ex|)[AW]$" 212 | path_param: "NewFileName" 213 | operation: "create" 214 | - api_call_name: "^MoveFileWithProgress[AW]$" 215 | path_param: "ExistingFileName" 216 | operation: "delete" 217 | - api_call_name: "^MoveFileWithProgress[AW]$" 218 | path_param: "NewFileName" 219 | operation: "create" 220 | - api_call_name: "NetFwAuthorizedApplication:INetFwAuthorizedApplication:put_ProcessImageFileName" 221 | path_param: "ProcessImageFileName" 222 | operation: "read" 223 | - api_call_name: "^PathFileExists[AW]$" 224 | path_param: "Path" 225 | operation: "read" 226 | - api_call_name: "^SetFile(Attributes|Security)[AW]?$" 227 | path_param: "FileName" 228 | operation: "write" 229 | - api_call_name: "SHCreateItemFromParsingName" 230 | handle_returned_in: "ppv" 231 | path_param: "Path" 232 | operation: "read" 233 | - api_call_name: "^ShellExecute(Ex|)[AW]$" 234 | path_param: "File" 235 | operation: "execute" 236 | - api_call_name: "^SHFileOperation[AW]$" 237 | path_param: "From" 238 | operation: "read" 239 | - api_call_name: "^SHFileOperation[AW]$" 240 | path_param: "To" 241 | operation: "create" 242 | - api_call_name: "^SHGetFileInfo[AW]$" 243 | path_param: "Path" 244 | operation: "read" 245 | - api_call_name: "^URLDownloadToFile[AW]$" 246 | path_param: "FileName" 247 | operation: "create" 248 | - api_call_name: "^WritePrivateProfileString[AW]$" 249 | path_param: "FileName" 250 | operation: "write" 251 | 252 | # -------------------------------------------- 253 | # Registry 254 | # -------------------------------------------- 255 | registry: 256 | # Kernel system calls for registry 257 | - api_call_name: "^(Nt|Zw)CreateKey(Transacted|)$" 258 | handle_returned_in: "KeyHandle" 259 | path_param: "ObjectAttributes" 260 | operation: "create" 261 | - api_call_name: "^(Nt|Zw)OpenKey(Transacted|)(Ex|)$" 262 | handle_returned_in: "KeyHandle" 263 | path_param: "ObjectAttributes" 264 | operation: "read" 265 | - api_call_name: "^(Nt|Zw)DeleteKey$" 266 | handle_param: "KeyHandle" 267 | operation: "delete" 268 | - api_call_name: "^(Nt|Zw)DeleteValueKey$" 269 | handle_param: "KeyHandle" 270 | path_param: "ValueName" 271 | operation: "delete" 272 | - api_call_name: "^(Nt|Zw)QueryKey$" 273 | handle_param: "KeyHandle" 274 | path_param: "KeyName" 275 | operation: "read" 276 | - api_call_name: "^(Nt|Zw)QueryKeyValue$" 277 | handle_param: "KeyHandle" 278 | path_param: "ValueName" 279 | operation: "read" 280 | - api_call_name: "^(Nt|Zw)EnumerateKey$" 281 | handle_param: "KeyHandle" 282 | path_param: "KeyName" 283 | operation: "read" 284 | - api_call_name: "^(Nt|Zw)EnumerateValueKey" 285 | handle_param: "KeyHandle" 286 | operation: "read" 287 | - api_call_name: "^(Nt|Zw)FlushKey" 288 | handle_param: "KeyHandle" 289 | operation: "write" 290 | - api_call_name: "^(Nt|Zw)SetValueKey$" 291 | handle_param: "KeyHandle" 292 | path_param: "ValueName" 293 | operation: "write" 294 | # Userland API calls for registry 295 | - api_call_name: "^RegCreateKey(Ex|)[AW]$" 296 | handle_param: "Registry" 297 | handle_returned_in: "Handle" 298 | path_param: "SubKey" 299 | operation: "create" 300 | - api_call_name: "^RegOpenKey(Ex|)[AW]$" 301 | handle_param: "Registry" 302 | handle_returned_in: "Handle" 303 | path_param: "SubKey" 304 | operation: "read" 305 | - api_call_name: "^RegDeleteKey(Ex|)[AW]$" 306 | handle_param: "Handle" 307 | path_param: "SubKey" 308 | operation: "delete" 309 | - api_call_name: "^RegDeleteValue(Ex|)[AW]$" 310 | handle_param: "Handle" 311 | path_param: "ValueName" 312 | operation: "delete" 313 | - api_call_name: "^RegEnumKey(Ex|)[AW]$" 314 | handle_param: "Handle" 315 | path_param: "Name" 316 | operation: "read" 317 | - api_call_name: "^RegEnumValue(Ex|)[AW]$" 318 | handle_param: "Handle" 319 | path_param: "ValueName" 320 | operation: "read" 321 | - api_call_name: "RegFlusHandle" 322 | handle_param: "Handle" 323 | operation: "write" 324 | - api_call_name: "^RegGetValue[AW]$" 325 | handle_param: "Handle" 326 | path_param: "SubKey" 327 | value_param: "Value" 328 | operation: "read" 329 | - api_call_name: "RegNotifyChangeKeyValue" 330 | handle_param: "Handle" 331 | operation: "read" 332 | - api_call_name: "^RegQueryInfoKey[AW]$" 333 | handle_param: "Handle" 334 | operation: "read" 335 | - api_call_name: "^RegQueryValue(Ex|)[AW]$" 336 | handle_param: "Handle" 337 | path_param: "ValueName" 338 | operation: "read" 339 | - api_call_name: "^RegSetValueEx[AW]$" 340 | handle_param: "Handle" 341 | path_param: "ValueName" 342 | operation: "write" 343 | - api_call_name: "^RegSetValue[AW]$" 344 | handle_param: "Handle" 345 | path_param: "SubKey" 346 | operation: "write" 347 | - api_call_name: "RegOpenCurrentUser" 348 | handle_id: 0x80000001 349 | handle_returned_in: "Handle" 350 | operation: "read" 351 | # -------------------------------------------- 352 | # Network 353 | # -------------------------------------------- 354 | network: 355 | # Winsock2 356 | - api_call_name: "^WSASocket[AW]$" 357 | handle_returned_in: "socket" 358 | network_type: "connect" 359 | operation: "connect" 360 | - api_call_name: "^(connect|WSAConnect)$" 361 | handle_returned_in: "Socket" 362 | host_param: "sin_addr" 363 | port_param: "sin_port" 364 | network_type: "connect" 365 | operation: "read" 366 | - api_call_name: "ConnectEx" 367 | handle_returned_in: "socket" 368 | host_param: "ip" 369 | port_param: "port" 370 | network_type: "connect" 371 | operation: "read" 372 | - api_call_name: "^(recv|WSARecv)$" 373 | handle_param: "Socket" 374 | network_type: "connect" 375 | operation: "read" 376 | - api_call_name: "^(recvfrom|WSARecvFrom)$" 377 | handle_param: "Socket" 378 | network_type: "listen" 379 | operation: "read" 380 | - api_call_name: "sendto" 381 | handle_param: "Socket" 382 | network_type: "listen" 383 | operation: "read" 384 | - api_call_name: "^(send|WSASend)$" 385 | handle_param: "Socket" 386 | network_type: "connect" 387 | operation: "write" 388 | - api_call_name: "gethostbyname" 389 | dns_name_param: "return:h_name" 390 | dns_ip_param: "return:h_addr_list" 391 | network_type: "dns" 392 | - api_call_name: "getaddrinfo" 393 | dns_name_param: "NodeName" 394 | dns_ip_param: "ppResult:ai_addr:sin_addr" 395 | network_type: "dns" 396 | # - api_call_name: "bind" 397 | # handle_returned_in: "s" 398 | # host_param: "sin_addr" 399 | # port_param: "sin_port" 400 | # network_type: "listen" 401 | # operation: "create" 402 | # Wininet 403 | - api_call_name: "^InternetConnect[AW]$" 404 | handle_returned_in: "return" 405 | host_param: "ServerName" 406 | port_param: "ServerPort" 407 | network_type: "connect" 408 | operation: "read" 409 | - api_call_name: "^InternetOpenUrl[AW]$" 410 | handle_returned_in: "return" 411 | url_param: "Url" 412 | network_type: "connect" 413 | operation: "read" 414 | - api_call_name: "^HttpOpenRequest[AW]$" 415 | handle_param: "InternetHandle" 416 | handle_returned_in: "return" 417 | request_param: "Path" 418 | network_type: "connect" 419 | operation: "read" 420 | - api_call_name: "^HttpQueryInfo[AW]$" 421 | handle_param: "RequestHandle" 422 | network_type: "connect" 423 | operation: "read" 424 | - api_call_name: "^HttpSendRequest(Ex|)[AW]$" 425 | handle_param: "RequestHandle" 426 | network_type: "connect" 427 | operation: "write" 428 | - api_call_name: "^HttpAddRequestHeaders[AW]$" 429 | handle_param: "RequestHandle" 430 | network_type: "connect" 431 | operation: "write" 432 | - api_call_name: "^InternetReadFile(Ex|)[AW]$" 433 | handle_param: "hFile" 434 | network_type: "connect" 435 | operation: "read" 436 | - api_call_name: "^InternetWriteFile(Ex|)[AW]$" 437 | handle_param: "hFile" 438 | network_type: "connect" 439 | operation: "write" 440 | # Winhttp 441 | - api_call_name: "WinHttpConnect" 442 | handle_returned_in: "return" 443 | host_param: "ServerName" 444 | port_param: "ServerPort" 445 | network_type: "connect" 446 | operation: "read" 447 | - api_call_name: "WinHttpOpenRequest" 448 | handle_param: "InternetHandle" 449 | handle_returned_in: "return" 450 | request_param: "ObjectName" 451 | network_type: "connect" 452 | operation: "read" 453 | - api_call_name: "WinHttpQueryDataAvailable" 454 | handle_param: "InternetHandle" 455 | network_type: "connect" 456 | operation: "read" 457 | - api_call_name: "WinHttpQueryHeaders" 458 | handle_param: "InternetHandle" 459 | network_type: "connect" 460 | operation: "read" 461 | - api_call_name: "WinHttpReadData" 462 | handle_param: "InternetHandle" 463 | network_type: "connect" 464 | operation: "read" 465 | - api_call_name: "WinHttpReceiveResponse" 466 | handle_param: "InternetHandle" 467 | network_type: "connect" 468 | operation: "read" 469 | - api_call_name: "WinHttpSendRequest" 470 | handle_param: "InternetHandle" 471 | network_type: "connect" 472 | operation: "write" 473 | # Urlmon 474 | - api_call_name: "^URLDownloadToFile[AW]$" 475 | url_param: "URL" 476 | network_type: "connect" 477 | operation: "read" 478 | - api_call_name: "IInternetProtocolRoot:Start" 479 | handle_param: "This" 480 | url_param: "szURL" 481 | network_type: "connect" 482 | operation: "read" 483 | - api_call_name: "IInternetProtocolRoot:Start" 484 | handle_param: "This" 485 | url_param: "szURL" 486 | network_type: "connect" 487 | operation: "read" 488 | - api_call_name: "IInternetSession:CreateBinding" 489 | handle_returned_in: "ppOInetProt" 490 | url_param: "szURL" 491 | handle_is_out: True 492 | network_type: "connect" 493 | operation: "read" 494 | 495 | # -------------------------------------------- 496 | # Handle Closing 497 | # -------------------------------------------- 498 | close: 499 | # Filesystem 500 | - api_call_name: "^(Nt|Zw)Close$" 501 | handle_param: "Handle" 502 | - api_call_name: "CloseHandle" 503 | handle_param: "Handle" 504 | - api_call_name: "fclose" 505 | handle_param: "File" 506 | - api_call_name: "IUnknown:Release$" 507 | handle_param: "This" 508 | # Registry 509 | - api_call_name: "RegCloseKey" 510 | handle_param: "Handle" 511 | -------------------------------------------------------------------------------- /dynmx/config/aam_vmray.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Configuration file for the Access Activity Model (VMRay sandbox) 19 | 20 | # -------------------------------------------- 21 | # Filesystem 22 | # -------------------------------------------- 23 | filesystem: 24 | # Kernel system calls for filesystem 25 | - api_call_name: "^(Nt|Zw)CreateFile$" 26 | handle_returned_in: "FileHandle" 27 | path_param: "ObjectAttributes:ObjectName" 28 | handle_is_out: True 29 | operation: "create" 30 | - api_call_name: "NtCreatePagingFile$" 31 | path_param: "PageFileName" 32 | operation: "write" 33 | - api_call_name: "^(Nt|Zw)DeleteFile$" 34 | path_param: "ObjectAttributes:ObjectName" 35 | operation: "delete" 36 | - api_call_name: "^(Nt|Zw)FlushBuffersFile$" 37 | handle_param: "FileHandle" 38 | operation: "write" 39 | - api_call_name: "^(Nt|Zw)LockFile$" 40 | handle_param: "FileHandle" 41 | operation: "read" 42 | - api_call_name: "^(Nt|Zw)UnlockFile$" 43 | handle_param: "FileHandle" 44 | operation: "read" 45 | - api_call_name: "^(Nt|Zw)OpenFile$" 46 | handle_returned_in: "FileHandle" 47 | handle_is_out: True 48 | path_param: "ObjectAttributes:ObjectName" 49 | operation: "read" 50 | - api_call_name: "^(Nt|Zw)OpenProcess$" 51 | path_param: "ObjectAttributes:ObjectName" 52 | operation: "execute" 53 | - api_call_name: "^(Nt|Zw)QueryAttributesFile$" 54 | path_param: "ObjectAttributes:ObjectName" 55 | operation: "read" 56 | - api_call_name: "^(Nt|Zw)QueryDirectoryFile$" 57 | handle_param: "FileHandle" 58 | operation: "read" 59 | - api_call_name: "^(Nt|Zw)QueryEaFile$" 60 | handle_param: "FileHandle" 61 | operation: "read" 62 | - api_call_name: "^(Nt|Zw)QueryAttributesFile$" 63 | path_param: "ObjectAttributes:ObjectName" 64 | operation: "read" 65 | - api_call_name: "^(Nt|Zw)QueryFullAttributesFile$" 66 | path_param: "ObjectAttributes:ObjectName" 67 | operation: "read" 68 | - api_call_name: "^(Nt|Zw)QueryInformationFile$" 69 | handle_param: "FileHandle" 70 | operation: "read" 71 | - api_call_name: "^(Nt|Zw)QueryInformationFile$" 72 | handle_param: "FileHandle" 73 | operation: "read" 74 | - api_call_name: "^(Nt|Zw)QueryQuotaInformationFile$" 75 | handle_param: "FileHandle" 76 | operation: "read" 77 | - api_call_name: "^(Nt|Zw)QueryVolumeInformationFile$" 78 | handle_param: "FileHandle" 79 | operation: "read" 80 | - api_call_name: "^(Nt|Zw)ReadFile$" 81 | handle_param: "FileHandle" 82 | operation: "read" 83 | - api_call_name: "^(Nt|Zw)ReadFileScatter$" 84 | handle_param: "FileHandle" 85 | operation: "read" 86 | - api_call_name: "^(Nt|Zw)SetEaFile$" 87 | handle_param: "FileHandle" 88 | operation: "write" 89 | - api_call_name: "^(Nt|Zw)SetInformationFile$" 90 | handle_param: "FileHandle" 91 | operation: "write" 92 | - api_call_name: "^(Nt|Zw)SetQuotaInformationFile$" 93 | handle_param: "FileHandle" 94 | operation: "write" 95 | - api_call_name: "^(Nt|Zw)SetVolumeInformationFile$" 96 | handle_param: "FileHandle" 97 | operation: "write" 98 | - api_call_name: "^(Nt|Zw)WriteFile$" 99 | handle_param: "FileHandle" 100 | operation: "write" 101 | - api_call_name: "^(Nt|Zw)WriteFileGather$" 102 | handle_param: "FileHandle" 103 | operation: "write" 104 | - api_call_name: "LdrLoadDll" 105 | path_param: "Name" 106 | operation: "execute" 107 | - api_call_name: "^LoadLibrary(Ex|)[AW]$" 108 | path_param: "lpLibFileName" 109 | handle_returned_in: "return" 110 | operation: "execute" 111 | # Userland API calls for filesystem 112 | - api_call_name: "^CreateFile[AW]$" 113 | handle_returned_in: "return" 114 | path_param: "lpFileName" 115 | operation: "create" 116 | - api_call_name: "fopen" 117 | handle_returned_in: "return" 118 | path_param: "_Filename" 119 | operation: "read" 120 | - api_call_name: "ReadFile" 121 | handle_param: "hFile" 122 | operation: "read" 123 | - api_call_name: "WriteFile" 124 | handle_param: "hFile" 125 | operation: "write" 126 | - api_call_name: "^CreateProcess[AW]$" 127 | path_param: "lpApplicationName" 128 | cmd_line_param: "lpCommandLine" 129 | operation: "execute" 130 | - api_call_name: "^CopyFile(Ex|)[AW]$" 131 | path_param: "lpExistingFileName" 132 | operation: "read" 133 | - api_call_name: "^CopyFile(Ex|)[AW]$" 134 | path_param: "lpNewFileName" 135 | operation: "create" 136 | - api_call_name: "^CreateFileMapping[AW]$" 137 | handle_param: "hFile" 138 | operation: "read" 139 | - api_call_name: "^DeleteFile[AW]$" 140 | path_param: "lpFileName" 141 | operation: "delete" 142 | - api_call_name: "FlushFileBuffers" 143 | handle_param: "hFile" 144 | operation: "write" 145 | - api_call_name: "^GetFileAttributes(Ex|)[AW]$" 146 | path_param: "lpFileName" 147 | operation: "read" 148 | - api_call_name: "^GetFileSize(Ex|)$" 149 | handle_param: "hFile" 150 | operation: "read" 151 | - api_call_name: "GetFileTime" 152 | handle_param: "hFile" 153 | operation: "read" 154 | - api_call_name: "GetFileType" 155 | handle_param: "hFile" 156 | operation: "read" 157 | - api_call_name: "^GetFileVersionInfo[AW]$" 158 | path_param: "lptstrFilename" 159 | operation: "read" 160 | - api_call_name: "^GetFileVersionInfoSize[AW]$" 161 | path_param: "lptstrFilename" 162 | operation: "read" 163 | - api_call_name: "^GetFileVersionInfoSizeEx[AW]$" 164 | path_param: "lpwstrFilename" 165 | operation: "read" 166 | - api_call_name: "^GetModuleFileName(Ex|)[AW]$" 167 | path_param: "lpFilename" 168 | path_param_is_out: True 169 | operation: "read" 170 | - api_call_name: "^GetMappedFileName[AW]$" 171 | path_param: "lpFilename" 172 | path_param_is_out: True 173 | operation: "read" 174 | - api_call_name: "^GetPrivateProfile(Int|String)[AW]$" 175 | path_param: "lpFilename" 176 | operation: "read" 177 | - api_call_name: "^GetPrivateProfileSection(Names|)[AW]$" 178 | path_param: "lpFilename" 179 | operation: "read" 180 | - api_call_name: "^GetTempFileName[AW]$" 181 | path_param: "lpTempFileName" 182 | path_param_is_out: True 183 | operation: "create" 184 | - api_call_name: "ICreateErrorInfo:SetHelpFile" 185 | path_param: "szHelpFile" 186 | operation: "read" 187 | - api_call_name: "ICreateTypeLib:SetHelpFileName" 188 | path_param: "szHelpFileName" 189 | operation: "read" 190 | - api_call_name: "IErrorInfo:GetHelpFile" 191 | path_param: "pBstrHelpFile" 192 | operation: "read" 193 | - api_call_name: "IoCreateFile" 194 | handle_returned_in: "FileHandle" 195 | handle_is_out: True 196 | path_param: "ObjectName" 197 | operation: "create" 198 | - api_call_name: "IFileOperation:CopyItem" 199 | handle_param: "psiItem" 200 | operation: "read" 201 | - api_call_name: "IFileOperation:CopyItem" 202 | handle_param: "psiDestinationFolder" 203 | file_name_param: "pszCopyName" 204 | operation: "create" 205 | - api_call_name: "^(Lock|Unlock)File(Ex|)$" 206 | handle_param: "hFile" 207 | operation: "read" 208 | - api_call_name: "SetEndOfFile" 209 | handle_param: "hFile" 210 | operation: "write" 211 | - api_call_name: "SetFileTime" 212 | handle_param: "hFile" 213 | operation: "write" 214 | - api_call_name: "^MoveFile(Ex|)[AW]$" 215 | path_param: "lpExistingFileName" 216 | operation: "delete" 217 | - api_call_name: "^MoveFile(Ex|)[AW]$" 218 | path_param: "lpNewFileName" 219 | operation: "create" 220 | - api_call_name: "^MoveFileWithProgress[AW]$" 221 | path_param: "lpExistingFileName" 222 | operation: "delete" 223 | - api_call_name: "^MoveFileWithProgress[AW]$" 224 | path_param: "lpNewFileName" 225 | operation: "create" 226 | - api_call_name: "NetFwAuthorizedApplication:INetFwAuthorizedApplication:put_ProcessImageFileName" 227 | path_param: "ProcessImageFileName" 228 | operation: "read" 229 | - api_call_name: "^PathFileExists[AW]$" 230 | path_param: "pszPath" 231 | operation: "read" 232 | - api_call_name: "^SetFile(Attributes|Security)[AW]?$" 233 | path_param: "lpFileName" 234 | operation: "write" 235 | - api_call_name: "SHCreateItemFromParsingName" 236 | handle_returned_in: "ppv" 237 | handle_is_out: True 238 | path_param: "pszPath" 239 | operation: "read" 240 | - api_call_name: "^ShellExecute(Ex|)[AW]$" 241 | path_param: "lpFile" 242 | operation: "execute" 243 | - api_call_name: "^SHFileOperation[AW]$" 244 | path_param: "pFrom" 245 | operation: "read" 246 | - api_call_name: "^SHFileOperation[AW]$" 247 | path_param: "pTo" 248 | operation: "create" 249 | - api_call_name: "^SHGetFileInfo[AW]$" 250 | path_param: "pszPath" 251 | operation: "read" 252 | - api_call_name: "^URLDownloadToFile[AW]$" 253 | path_param: "(szFileName|param_3)" 254 | operation: "create" 255 | - api_call_name: "^WritePrivateProfileString[AW]$" 256 | path_param: "lpFileName" 257 | operation: "write" 258 | 259 | # -------------------------------------------- 260 | # Registry 261 | # -------------------------------------------- 262 | registry: 263 | # Kernel system calls for registry 264 | - api_call_name: "^(Nt|Zw)CreateKey(Transacted|)$" 265 | handle_param: "RootDirectory" 266 | handle_returned_in: "KeyHandle" 267 | handle_is_out: True 268 | path_param: "ObjectName" 269 | operation: "create" 270 | - api_call_name: "^(Nt|Zw)OpenKey(Transacted|)(Ex|)$" 271 | handle_param: "RootDirectory" 272 | handle_returned_in: "KeyHandle" 273 | handle_is_out: True 274 | path_param: "ObjectName" 275 | operation: "read" 276 | - api_call_name: "^(Nt|Zw)DeleteKey$" 277 | handle_param: "KeyHandle" 278 | operation: "delete" 279 | - api_call_name: "^(Nt|Zw)DeleteValueKey$" 280 | handle_param: "KeyHandle" 281 | path_param: "ValueName" 282 | operation: "delete" 283 | - api_call_name: "^(Nt|Zw)QueryKey$" 284 | handle_param: "KeyHandle" 285 | path_param: "KeyInformationClass:KeyNameInformation:Name" 286 | operation: "read" 287 | - api_call_name: "^(Nt|Zw)QueryKeyValue$" 288 | handle_param: "KeyHandle" 289 | path_param: "KeyValueInformationClass:KeyValueBasicInformation:Name" 290 | operation: "read" 291 | - api_call_name: "^(Nt|Zw)EnumerateKey$" 292 | handle_param: "KeyHandle" 293 | path_param: "KeyInformationClass:KeyNameInformation:Name" 294 | operation: "read" 295 | - api_call_name: "^(Nt|Zw)EnumerateValueKey" 296 | handle_param: "KeyHandle" 297 | operation: "read" 298 | - api_call_name: "^(Nt|Zw)FlushKey" 299 | handle_param: "KeyHandle" 300 | operation: "write" 301 | - api_call_name: "^(Nt|Zw)SetValueKey$" 302 | handle_param: "KeyHandle" 303 | path_param: "ValueName" 304 | operation: "write" 305 | # Userland API calls for registry 306 | - api_call_name: "^RegCreateKey(Ex|)[AW]$" 307 | handle_param: "hKey" 308 | handle_returned_in: "phkResult" 309 | path_param: "lpSubKey" 310 | operation: "create" 311 | handle_is_out: True 312 | - api_call_name: "^RegOpenKey(Ex|)[AW]$" 313 | handle_param: "hKey" 314 | handle_returned_in: "phkResult" 315 | path_param: "lpSubKey" 316 | operation: "read" 317 | handle_is_out: True 318 | - api_call_name: "^RegDeleteKey(Ex|)[AW]$" 319 | handle_param: "hKey" 320 | path_param: "lpSubKey" 321 | operation: "delete" 322 | - api_call_name: "^RegDeleteValue(Ex|)[AW]$" 323 | handle_param: "hKey" 324 | path_param: "lpValueName" 325 | operation: "delete" 326 | - api_call_name: "^RegEnumKey(Ex|)[AW]$" 327 | handle_param: "hKey" 328 | path_param: "lpName" 329 | path_param_is_out: True 330 | operation: "read" 331 | - api_call_name: "^RegEnumValue(Ex|)[AW]$" 332 | handle_param: "hKey" 333 | path_param: "lpValueName" 334 | path_param_is_out: True 335 | operation: "read" 336 | - api_call_name: "RegFlushKey" 337 | handle_param: "hKey" 338 | operation: "write" 339 | - api_call_name: "^RegGetValue[AW]$" 340 | handle_param: "hKey" 341 | path_param: "lpSubKey" 342 | value_param: "lpValue" 343 | operation: "read" 344 | - api_call_name: "RegNotifyChangeKeyValue" 345 | handle_param: "hKey" 346 | operation: "read" 347 | - api_call_name: "^RegQueryInfoKey[AW]$" 348 | handle_param: "hKey" 349 | operation: "read" 350 | - api_call_name: "^RegQueryValue(Ex|)[AW]$" 351 | handle_param: "hKey" 352 | path_param: "lpValueName" 353 | operation: "read" 354 | - api_call_name: "^RegSetValueEx[AW]$" 355 | handle_param: "hKey" 356 | path_param: "lpValueName" 357 | operation: "write" 358 | - api_call_name: "^RegSetValue[AW]$" 359 | handle_param: "hKey" 360 | path_param: "lpSubKey" 361 | operation: "write" 362 | - api_call_name: "RegOpenCurrentUser" 363 | handle_id: 0x80000001 364 | handle_returned_in: "phkResult" 365 | handle_is_out: True 366 | operation: "read" 367 | # -------------------------------------------- 368 | # Network 369 | # -------------------------------------------- 370 | network: 371 | # Winsock2 372 | - api_call_name: "^(connect|WSAConnect)$" 373 | handle_returned_in: "s" 374 | host_param: "sin_addr" 375 | port_param: "sin_port" 376 | network_type: "connect" 377 | operation: "read" 378 | - api_call_name: "^(recv|WSARecv)$" 379 | handle_param: "s" 380 | network_type: "connect" 381 | operation: "read" 382 | - api_call_name: "^(recvfrom|WSARecvFrom)$" 383 | handle_param: "s" 384 | network_type: "listen" 385 | operation: "read" 386 | - api_call_name: "sendto" 387 | handle_param: "s" 388 | network_type: "listen" 389 | operation: "read" 390 | - api_call_name: "^(send|WSASend)$" 391 | handle_param: "s" 392 | network_type: "connect" 393 | operation: "write" 394 | - api_call_name: "gethostbyname" 395 | dns_name_param: "return:h_name" 396 | dns_ip_param: "return:h_addr_list" 397 | network_type: "dns" 398 | - api_call_name: "getaddrinfo" 399 | dns_name_param: "pNodeName" 400 | dns_ip_param: "ppResult:ai_addr:sin_addr" 401 | network_type: "dns" 402 | - api_call_name: "bind" 403 | handle_returned_in: "s" 404 | host_param: "sin_addr" 405 | port_param: "sin_port" 406 | network_type: "listen" 407 | operation: "create" 408 | # Wininet 409 | - api_call_name: "^InternetConnect[AW]$" 410 | handle_returned_in: "return" 411 | host_param: "lpszServerName" 412 | port_param: "nServerPort" 413 | network_type: "connect" 414 | operation: "read" 415 | - api_call_name: "^InternetOpenUrl[AW]$" 416 | handle_returned_in: "return" 417 | url_param: "lpszUrl" 418 | network_type: "connect" 419 | operation: "read" 420 | - api_call_name: "^HttpOpenRequest[AW]$" 421 | handle_param: "hConnect" 422 | handle_returned_in: "return" 423 | request_param: "lpszObjectName" 424 | network_type: "connect" 425 | operation: "read" 426 | - api_call_name: "^HttpQueryInfo[AW]$" 427 | handle_param: "hRequest" 428 | network_type: "connect" 429 | operation: "read" 430 | - api_call_name: "^HttpSendRequest(Ex|)[AW]$" 431 | handle_param: "hRequest" 432 | network_type: "connect" 433 | operation: "write" 434 | - api_call_name: "^HttpAddRequestHeaders[AW]$" 435 | handle_param: "hRequest" 436 | network_type: "connect" 437 | operation: "write" 438 | - api_call_name: "InternetReadFile" 439 | handle_param: "hFile" 440 | network_type: "connect" 441 | operation: "read" 442 | - api_call_name: "InternetWriteFile" 443 | handle_param: "hFile" 444 | network_type: "connect" 445 | operation: "write" 446 | # Winhttp 447 | - api_call_name: "WinHttpConnect" 448 | handle_returned_in: "return" 449 | host_param: "pswzServerName" 450 | port_param: "nServerPort" 451 | network_type: "connect" 452 | operation: "read" 453 | - api_call_name: "WinHttpOpenRequest" 454 | handle_param: "hConnect" 455 | handle_returned_in: "return" 456 | request_param: "pwszObjectName" 457 | network_type: "connect" 458 | operation: "read" 459 | - api_call_name: "WinHttpQueryDataAvailable" 460 | handle_param: "hRequest" 461 | network_type: "connect" 462 | operation: "read" 463 | - api_call_name: "WinHttpQueryHeaders" 464 | handle_param: "hRequest" 465 | network_type: "connect" 466 | operation: "read" 467 | - api_call_name: "WinHttpReadData" 468 | handle_param: "hRequest" 469 | network_type: "connect" 470 | operation: "read" 471 | - api_call_name: "WinHttpReceiveResponse" 472 | handle_param: "hRequest" 473 | network_type: "connect" 474 | operation: "read" 475 | - api_call_name: "WinHttpSendRequest" 476 | handle_param: "hRequest" 477 | network_type: "connect" 478 | operation: "write" 479 | # Urlmon 480 | - api_call_name: "^URLDownloadToFile[AW]$" 481 | url_param: "(szURL|param_2)" 482 | network_type: "connect" 483 | operation: "read" 484 | - api_call_name: "IInternetProtocolRoot:Start" 485 | handle_param: "This" 486 | url_param: "szURL" 487 | network_type: "connect" 488 | operation: "read" 489 | - api_call_name: "IInternetProtocolRoot:Start" 490 | handle_param: "This" 491 | url_param: "szURL" 492 | network_type: "connect" 493 | operation: "read" 494 | - api_call_name: "IInternetSession:CreateBinding" 495 | handle_returned_in: "ppOInetProt" 496 | url_param: "szURL" 497 | handle_is_out: True 498 | network_type: "connect" 499 | operation: "read" 500 | 501 | # -------------------------------------------- 502 | # Handle Closing 503 | # -------------------------------------------- 504 | close: 505 | # Filesystem 506 | - api_call_name: "^(Nt|Zw)Close$" 507 | handle_param: "Handle" 508 | - api_call_name: "CloseHandle" 509 | handle_param: "hObject" 510 | - api_call_name: "fclose" 511 | handle_param: "_File" 512 | - api_call_name: "IUnknown:Release$" 513 | handle_param: "This" 514 | # Registry 515 | - api_call_name: "RegCloseKey" 516 | handle_param: "hKey" 517 | -------------------------------------------------------------------------------- /dynmx/converters/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /dynmx/converters/dynmx_converter.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, TYPE_CHECKING 20 | import os 21 | import json 22 | import datetime 23 | import time 24 | import gzip 25 | 26 | from dynmx.converters.dynmx_harmonizer import DynmxHarmonizer 27 | from dynmx.helpers.logging_helper import LoggingHelper 28 | 29 | # Avoid cyclic imports 30 | if TYPE_CHECKING: 31 | from dynmx.core.function_log import FunctionLog 32 | 33 | 34 | class DynmxConverter: 35 | """ 36 | Converter for the dynmx flog format 37 | """ 38 | 39 | def __init__(self, api_call_db: Optional[str] = None): 40 | """ 41 | Constructor 42 | :param api_call_db: Path to the API call sqlite database used for harmonization. If no path is passed 43 | harmonization will not take place while converting the function log. 44 | """ 45 | # Load API call database 46 | self._logger = LoggingHelper.get_logger(__name__) 47 | if api_call_db: 48 | self._harmonizer = DynmxHarmonizer(api_call_db) 49 | self._logger.debug("API call database is {}. Harmonization will take place based on this database.".format( 50 | api_call_db)) 51 | else: 52 | self._harmonizer = None 53 | self._logger.debug("No API call database given. No harmonization will take place.") 54 | 55 | def convert(self, flog: FunctionLog, output_dir: str, compress: bool) -> None: 56 | """ 57 | Converts the flog to the dynmx flog format 58 | :param flog: Function log to convert 59 | :param output_dir: Directory to write converted flog to 60 | :param compress: Defines whether the converted flog should be compressed 61 | """ 62 | self._logger = LoggingHelper.get_logger(__name__, flog.file_path) 63 | if self._harmonizer: 64 | self._harmonizer.harmonize_flog(flog) 65 | self._logger.info("Converting function log '{}'".format(flog.name)) 66 | converted_flog = flog.convert() 67 | self._logger.info("Preparing JSON output of converted log") 68 | tic = time.perf_counter() 69 | json_output = json.dumps(converted_flog, indent=4, ensure_ascii=False) 70 | toc = time.perf_counter() 71 | runtime_json = toc - tic 72 | self._logger.info("JSON conversion took {:.4f}s".format(runtime_json)) 73 | output_path = self._get_converted_file_path(flog.file_path, output_dir, compress) 74 | self._logger.info("Writing converted function log '{}' to path '{}'".format(flog.name, output_path)) 75 | if compress: 76 | with gzip.open(output_path, "wb+") as output_file: 77 | header = self._get_file_header(flog.file_path) 78 | output_file.write(header.encode()) 79 | output_file.write(json_output.encode()) 80 | else: 81 | with open(output_path, "w+") as output_file: 82 | header = self._get_file_header(flog.file_path) 83 | output_file.write(header) 84 | output_file.write(json_output) 85 | 86 | @staticmethod 87 | def _get_converted_file_path(flog_path: str, output_dir: str, compress: bool) -> str: 88 | """ 89 | Returns the file path for the converted flog 90 | :param flog_path: Path of flog to convert 91 | :param output_dir: Output directory 92 | :param compress: Compress output file 93 | :return: File path for the converted flog 94 | """ 95 | file_name = os.path.basename(flog_path) 96 | if compress: 97 | converted_flog_file_name = os.path.splitext(file_name)[0] + \ 98 | "_dynmx.txt.gz" 99 | else: 100 | converted_flog_file_name = os.path.splitext(file_name)[0] + \ 101 | "_dynmx.txt" 102 | if not output_dir: 103 | output_dir = os.path.dirname(flog_path) 104 | return os.path.join(output_dir, converted_flog_file_name) 105 | 106 | @staticmethod 107 | def _get_file_header(flog_path: str) -> str: 108 | """ 109 | Returns the dynmx flog file header 110 | :param flog_path: Path of flog to convert 111 | """ 112 | header = "# dynmx generic function log\n" 113 | header += "# converted from: {}\n".format(flog_path) 114 | header += "# converted on: {}\n\n".format(datetime.datetime.now()) 115 | return header 116 | -------------------------------------------------------------------------------- /dynmx/converters/dynmx_harmonizer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, TYPE_CHECKING, List, Dict, Any 20 | import logging 21 | import sqlite3 22 | 23 | from dynmx.core.api_call import ApiCallSignature, Argument 24 | 25 | if TYPE_CHECKING: 26 | from dynmx.core.function_log import FunctionLog 27 | from dynmx.core.api_call import APICall 28 | 29 | 30 | class DynmxHarmonizer: 31 | """ 32 | Harmonization layer for the dynmx flog format based on an API call database 33 | """ 34 | 35 | def __init__(self, api_call_db: str): 36 | """ 37 | Constructor 38 | :param api_call_db: Path to API call sqlite database file 39 | """ 40 | # Load API call database 41 | self._logger = logging.getLogger(__name__) 42 | self._api_call_db = ApiCallDb(api_call_db) 43 | 44 | def harmonize_flog(self, flog: FunctionLog) -> None: 45 | """ 46 | Harmonizes all APICall objects of the function log 47 | :param flog: Function log to harmonize 48 | :return: 49 | """ 50 | # Iterate over each process and identify unique API calls 51 | unique_api_calls = list() 52 | for proc in flog.processes: 53 | unique_api_calls = self._identify_api_call_signatures(proc.api_calls, unique_api_calls) 54 | self._logger.debug("Identified {} unique API calls for function log {}".format(len(unique_api_calls), 55 | flog.file_path)) 56 | mapping = self._build_api_call_mapping(unique_api_calls) 57 | # Manipulate API calls based on mapping 58 | for proc in flog.processes: 59 | self._logger.debug("Harmonizing API calls of process ID {}: {}".format(proc.os_id, proc.name)) 60 | for api_call in proc.api_calls: 61 | self._logger.debug("Harmonizing API call {}: {}".format(api_call.index, api_call.function_name)) 62 | if api_call.function_name in mapping.keys(): 63 | self._manipulate_api_call_obj(api_call, mapping[api_call.function_name]) 64 | 65 | def harmonize_api_call(self, api_call: APICall) -> None: 66 | """ 67 | Harmonizes the APICall object 68 | :param api_call: APICall object to harmonize 69 | :return: 70 | """ 71 | function_name = api_call.function_name 72 | api_call_signature = self._find_harmonized_api_call(function_name) 73 | if api_call_signature: 74 | self._manipulate_api_call_obj(api_call, api_call_signature) 75 | 76 | def _identify_api_call_signatures(self, api_calls: List[APICall], 77 | unique_api_calls: Optional[List[APICall]] = None) -> List[APICall]: 78 | """ 79 | Identifies unique API call signatures in a list of APICall objects 80 | :param api_calls: List of APICall objects 81 | :param unique_api_calls: List of already identified unique APICall objects 82 | :return: List of unique APICall objects 83 | """ 84 | unique_api_calls = list() if not unique_api_calls else unique_api_calls 85 | for api_call in api_calls: 86 | ix = self._find_api_call_signature(unique_api_calls, api_call) 87 | if ix is None: 88 | unique_api_calls.append(api_call) 89 | return unique_api_calls 90 | 91 | @staticmethod 92 | def _find_api_call_signature(api_calls: List[APICall], api_call: APICall) -> Optional[int]: 93 | """ 94 | Finds duplicates of a APICall object in a list of APICall objects. APICall objects are considered as 95 | duplicates if the function name and the number of arguments are matching 96 | :param api_calls: List of APICall objects 97 | :param api_call: APICall object 98 | :return: Index of the duplicate APICall object or None if no duplicate was found 99 | """ 100 | if not len(api_calls): 101 | return None 102 | api_call_name = api_call.function_name 103 | num_of_args = len(api_call.arguments) 104 | for ix, a in enumerate(api_calls): 105 | if a.function_name == api_call_name: 106 | if len(a.arguments) == num_of_args: 107 | return ix 108 | return None 109 | 110 | def _build_api_call_mapping(self, api_calls: List[APICall]) -> Dict[str, ApiCallSignature]: 111 | """ 112 | Builds a mapping represented by a dictionary of APICall objects to the corresponding harmonized 113 | ApiCallSignature object 114 | :param api_calls: List APICall objects to build the mapping for 115 | :return: Dictionary consisting of the APICall function names as keys and the corresponding harmonized 116 | ApiCallSignature object as value 117 | """ 118 | mapping = {} 119 | for api_call in api_calls: 120 | harmonized_sig = self._find_harmonized_api_call(api_call.function_name) 121 | if harmonized_sig: 122 | mapping[harmonized_sig.function_name] = harmonized_sig 123 | return mapping 124 | 125 | def _find_harmonized_api_call(self, function_name: str) -> ApiCallSignature: 126 | """ 127 | Finds a harmonized ApiCallSignature in the API call database for the given function name 128 | :param function_name: Function name to find harmonized API call signature for 129 | :return: ApiCallSignature object if a harmonized API call was found. None if no harmonized API call 130 | signature was found in the API call database. 131 | """ 132 | self._logger.debug("Searching for suitable API calls for {} in API call database".format(function_name)) 133 | candidates = self._api_call_db.select_api_call(function_name) 134 | if len(candidates): 135 | if len(candidates) > 1: 136 | self._logger.debug("Found {} harmonization candidates for API call {}".format( 137 | len(candidates), function_name)) 138 | # Identify the right API call by the calling convention 139 | for candidate in candidates: 140 | if candidate["calling_convention"] == "WINAPI": 141 | self._logger.debug("Found candidate with row ID {} for harmonizing API call {}".format( 142 | candidate["id"], function_name)) 143 | return self._build_api_call_obj(candidate) 144 | # If WINAPI calling convention can not be found take first found API call from database 145 | self._logger.debug("Found candidate with row ID {} for harmonizing API call {}".format( 146 | candidates[0]["id"], function_name)) 147 | return self._build_api_call_obj(candidates[0]) 148 | else: 149 | self._logger.debug("Found candidate with row ID {} for harmonizing API call {}".format( 150 | candidates[0]["id"], function_name)) 151 | return self._build_api_call_obj(candidates[0]) 152 | else: 153 | self._logger.warning("No harmonization candidates found for API call {}".format(function_name)) 154 | return None 155 | 156 | def _build_api_call_obj(self, api_call_row: Dict[str, Any]) -> ApiCallSignature: 157 | """ 158 | Builds a ApiCallSignature object based on the row retrieved from the API call database 159 | :param api_call_row: API call row returned by the API call database 160 | :return: ApiCallSignature object containing the information retrieved from the API call database row 161 | """ 162 | api_call = ApiCallSignature() 163 | api_call.function_name = api_call_row["name"] 164 | api_call.description = api_call_row["description"] 165 | api_call.return_value_desc = api_call_row["return_value"] 166 | api_call.return_type = api_call_row["return_type"] 167 | api_call.calling_convention = api_call_row["calling_convention"] 168 | # Arguments 169 | arg_rows = self._api_call_db.select_args_by_api_call(api_call_row["id"]) 170 | api_call.arguments = self._build_arg_objects(arg_rows) 171 | return api_call 172 | 173 | @staticmethod 174 | def _build_arg_objects(arg_rows: List[Dict[str, Any]]) -> List[Argument]: 175 | """ 176 | Builds a list of Argument objects based on the given argument rows retrieved from the API call database 177 | :param arg_rows: Rows from the API call database containing argument information 178 | :return: List of Argument objects containing the information retrieved by the API call database 179 | """ 180 | args = [] 181 | for arg_row in arg_rows: 182 | arg = Argument() 183 | arg.name = arg_row["name"] 184 | arg.is_in = True if arg_row["is_in"] == 1 else False 185 | arg.is_out = True if arg_row["is_out"] == 1 else False 186 | args.append(arg) 187 | return args 188 | 189 | def _manipulate_api_call_obj(self, api_call: APICall, api_call_signature: ApiCallSignature) -> None: 190 | """ 191 | Manipulates the given APICall object in order to harmonize the object based on the ApiCallSignature object 192 | :param api_call: APICall object to manipulate 193 | :param api_call_signature: ApiCallSignature object used as harmonization baseline 194 | """ 195 | # Find duplicate arguments based on the name in the concrete API call 196 | reference_table = self._build_arg_reference_table(api_call.arguments) 197 | # Manipulate arguments of API calls and their duplicates 198 | for ix, arg in enumerate(api_call_signature.arguments): 199 | if ix < len(api_call.arguments): 200 | api_call.arguments[ix].name = arg.name 201 | api_call.arguments[ix].is_in = arg.is_in 202 | api_call.arguments[ix].is_out = arg.is_out 203 | # Check for duplicate arguments and manipulate them too 204 | if ix in reference_table.keys(): 205 | referenced_ixs = reference_table[ix] 206 | for referenced_ix in referenced_ixs: 207 | api_call.arguments[referenced_ix].name = arg.name 208 | api_call.arguments[referenced_ix].is_in = arg.is_in 209 | api_call.arguments[referenced_ix].is_out = arg.is_out 210 | 211 | @staticmethod 212 | def _build_arg_reference_table(args: List[Argument]) -> Dict[int, List[int]]: 213 | """ 214 | Finds all duplicates of an argument in a list of arguments. An argument is considered as duplicate of another 215 | argument if the name is equal. 216 | :param args: List of arguments to search for duplicates in 217 | :return: Reference table representing duplicate arguments 218 | """ 219 | reference_table = {} 220 | already_visited = set() 221 | for ix, arg in enumerate(args): 222 | if ix in already_visited: 223 | continue 224 | for ix2, arg2 in enumerate(args): 225 | if ix != ix2 and arg.name == arg2.name: 226 | if ix not in reference_table.keys(): 227 | reference_table[ix] = [ix2] 228 | else: 229 | reference_table[ix].append(ix2) 230 | already_visited.update([ix, ix2]) 231 | return reference_table 232 | 233 | 234 | class ApiCallDb: 235 | """ 236 | Interface to the API call database 237 | """ 238 | 239 | def __init__(self, api_call_db: str): 240 | """ 241 | Constructor 242 | :param api_call_db: Path to API call sqlite database file 243 | """ 244 | # Load API call database 245 | self._logger = logging.getLogger(__name__) 246 | self._api_call_db = sqlite3.connect(api_call_db) 247 | self._api_call_db.row_factory = sqlite3.Row 248 | self._db_cursor = self._api_call_db.cursor() 249 | 250 | def select_api_call(self, function_name: str) -> List[Dict[str, Any]]: 251 | """ 252 | Finds all Windows API calls with the given function name in the API call database 253 | :param function_name: Function name to search for in API call database 254 | :return: Database rows of the found API calls 255 | """ 256 | # Search for Windows API calls matching the function name 257 | self._db_cursor.execute( 258 | "SELECT * FROM api_calls WHERE name=? AND target_os=\"Windows\";", 259 | (function_name,) 260 | ) 261 | rows = self._db_cursor.fetchall() 262 | self._logger.debug("Found {} rows for API call {} in API call database".format(len(rows), function_name)) 263 | return self._transform_rows(rows) 264 | 265 | def select_args_by_api_call(self, api_call_id: int) -> List[Dict[str, Any]]: 266 | """ 267 | Finds all arguments belonging to the given API call ID 268 | :param api_call_id: ID of the API call to find the arguments for 269 | :return: Database rows of the found arguments 270 | """ 271 | # Search for arguments of the API call id 272 | stmt = "SELECT p.*, t.name " \ 273 | "FROM api_calls a, api_call_params p, types t " \ 274 | "WHERE p.api_call_id=a.id AND p.type_id=t.id AND a.id=?;" 275 | self._db_cursor.execute( 276 | stmt, 277 | (api_call_id,) 278 | ) 279 | rows = self._db_cursor.fetchall() 280 | self._logger.debug("Found {} rows for arguments of API call with ID {} in API call database".format( 281 | len(rows), api_call_id)) 282 | return self._transform_rows(rows) 283 | 284 | @staticmethod 285 | def _transform_rows(rows: List) -> List: 286 | """ 287 | Builds a list of dictionaries based on the rows. The column names become the keys of the dictionary. 288 | :param rows: Rows returned from the database cursor 289 | :return: List of dictionaries representing a returned row 290 | """ 291 | if rows: 292 | return [dict(row) for row in rows] 293 | else: 294 | return [] 295 | -------------------------------------------------------------------------------- /dynmx/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /dynmx/core/api_call.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | import re 20 | from uuid import uuid4 21 | from typing import Optional, List, Dict, Any 22 | from dynmx.core.pointer import Pointer 23 | 24 | 25 | class APICall: 26 | """ 27 | Representation of a system call 28 | """ 29 | 30 | def __init__(self, index: Optional[int] = None, function_name: Optional[str] = None, time: Optional[float] = None, 31 | return_value: Optional = None, has_in_out_args: Optional[bool] = False, 32 | flog_index: Optional[int] = None): 33 | """ 34 | Constructor 35 | :param index: Index of system call 36 | :param function_name: Function name of system call 37 | :param time: Relative time system call was executed in seconds 38 | :param return_value: Return value of system call 39 | :param has_in_out_args: Indicates whether system call has in and out 40 | arguments 41 | :param flog_index: Line number of system call in flog 42 | """ 43 | self.index = index 44 | self.function_name = function_name 45 | self.time = time 46 | self.arguments = [] 47 | self.return_value = return_value 48 | self.has_in_out_args = has_in_out_args 49 | self.flog_index = flog_index 50 | 51 | def get_argument_values(self, arg_name: str, is_in: Optional[bool] = None, is_out: Optional[bool] = None, 52 | is_regex_pattern: bool = False) -> Optional[List[Any]]: 53 | """ 54 | Returns the argument value identified by arg_name 55 | :param arg_name: Name of argument of that the value should be returned 56 | :param is_in: Indicates whether argument is inbound 57 | :param is_out: Indicates whether argument is outbound 58 | :param is_regex_pattern: Indicates whether arg_name is a regex pattern 59 | :return: List of argument values 60 | """ 61 | if ":" in arg_name: 62 | arg_vals = self._get_deep_argument(arg_name, 63 | is_in, 64 | is_out, 65 | is_regex_pattern=is_regex_pattern) 66 | else: 67 | arg_vals = self._deep_argument_search(arg_name, 68 | is_in, 69 | is_out, 70 | is_regex_pattern=is_regex_pattern) 71 | return arg_vals 72 | 73 | def get_return_value(self, arg_name: Optional[str] = None, is_regex_pattern: bool = False) -> Optional[Any]: 74 | """ 75 | Returns the return value of the API call 76 | :return: Return value 77 | """ 78 | if arg_name: 79 | arg_val = self._get_deep_return_value(arg_name, is_regex_pattern=is_regex_pattern) 80 | if isinstance(arg_val, Pointer): 81 | arg_val = arg_val.arguments 82 | return arg_val 83 | else: 84 | if isinstance(self.return_value, Pointer): 85 | if not isinstance(list, self.return_value.arguments): 86 | return self.return_value.arguments 87 | else: 88 | return self.return_value 89 | return None 90 | 91 | def get_return_value_pointer(self, arg_name: str, is_regex_pattern: bool = False) -> Optional[Any]: 92 | """ 93 | Returns the return value identified by arg_name 94 | :param arg_name: Name of argument of that the value should be returned 95 | :param is_regex_pattern: Indicates whether arg_name is a regex pattern 96 | :return: Argument value 97 | """ 98 | if isinstance(self.return_value, Pointer): 99 | arg_val = self._deep_argument_search(arg_name, 100 | None, 101 | None, 102 | is_regex_pattern=is_regex_pattern) 103 | if arg_val: 104 | return arg_val 105 | return None 106 | 107 | def has_argument(self, arg_name: str, is_regex_pattern: bool = False) -> bool: 108 | """ 109 | Return whether a system call has the attribute identified by arg_name 110 | :param arg_name: Name of argument that should be present 111 | :param is_regex_pattern: Indicates whether arg_name is a regex pattern 112 | :return: Indicates whether the argument is present 113 | """ 114 | arg_val = self.get_argument_values(arg_name, 115 | is_regex_pattern=is_regex_pattern) 116 | return (arg_val is not None) 117 | 118 | def _get_deep_argument(self, arg_name: str, is_in: bool, is_out: bool, is_regex_pattern: bool = False) \ 119 | -> Optional[List[Any]]: 120 | """ 121 | Returns an argument value directly addressed by arg_name in all child objects (Pointer objects) 122 | :param arg_name: Directly addressed argument in child object (addressed by arg:child_arg) 123 | :param is_in: Indicates whether argument is inbound 124 | :param is_out: Indicates whether argument is outbound 125 | :param is_regex_pattern: Indicates whether arg_name is a regex pattern 126 | :return: List of values of searched argument 127 | """ 128 | found_values = [] 129 | arg_parts = arg_name.split(":") 130 | if not len(arg_parts): 131 | return found_values 132 | addressed_arg_name = arg_parts[0] 133 | addressed_args = self._get_arguments_by_name(addressed_arg_name, is_regex_pattern) 134 | if not addressed_args: 135 | return None 136 | else: 137 | addressed_arg = addressed_args[0] 138 | if is_in is not None: 139 | if addressed_arg.is_in != is_in: 140 | return None 141 | if is_out is not None: 142 | if addressed_arg.is_out != is_out: 143 | return None 144 | # Value has to be a pointer to find next addressed argument 145 | if not isinstance(addressed_arg.value, Pointer): 146 | return None 147 | # Find next addressed argument 148 | if len(arg_parts) > 1: 149 | sub_arg = ":".join(arg_parts[1:]) 150 | arg_vals = addressed_arg.value.get_argument_values(sub_arg, is_regex_pattern) 151 | if arg_vals: 152 | found_values += arg_vals 153 | else: 154 | return None 155 | return found_values if len(found_values) > 0 else None 156 | 157 | def _get_deep_return_value(self, arg_name: str, is_regex_pattern: bool = False) -> Optional[Any]: 158 | """ 159 | Returns an return value directly addressed by arg_name in all child objects (Pointer objects) 160 | :param arg_name: Directly addressed argument in child object (addressed by arg:child_arg) 161 | :param is_in: Indicates whether argument is inbound 162 | :param is_out: Indicates whether argument is outbound 163 | :param args: Arguments to search in 164 | :param is_regex_pattern: Indicates whether arg_name is a regex pattern 165 | :return: List of values of searched argument 166 | """ 167 | found_values = [] 168 | if ":" not in arg_name: 169 | args = self._get_return_value_args_by_name(arg_name, is_regex_pattern) 170 | if args and len(args): 171 | return args[0].value 172 | arg_parts = arg_name.split(":") 173 | if not len(arg_parts): 174 | return found_values 175 | addressed_arg_name = arg_parts[0] 176 | addressed_args = self._get_return_value_args_by_name(addressed_arg_name, is_regex_pattern) 177 | if addressed_args: 178 | addressed_arg = addressed_args[0] 179 | else: 180 | return None 181 | # Value has to be a pointer to find next addressed argument 182 | if not isinstance(addressed_arg.value, Pointer): 183 | return None 184 | # Find next addressed argument 185 | if len(arg_parts) > 1: 186 | sub_arg = ":".join(arg_parts[1:]) 187 | arg_vals = addressed_arg.value.get_argument_values(sub_arg, is_regex_pattern) 188 | found_values += arg_vals 189 | else: 190 | return None 191 | return found_values if len(found_values) > 0 else None 192 | 193 | def _get_arguments_by_name(self, arg_name: str, is_regex_pattern: bool = False) -> Optional[List[Argument]]: 194 | """ 195 | Returns the arguments of the system call identified by the argument name 196 | :param arg_name: Argument name 197 | :param is_regex_pattern: Indicates whether the argument name is a regex pattern 198 | :return: List of arguments if arguments were identified, otherwise None 199 | """ 200 | found_args = [] 201 | for arg in self.arguments: 202 | if is_regex_pattern: 203 | if self._is_regex_matching(arg.name, arg_name): 204 | found_args.append(arg) 205 | else: 206 | if arg.name.lower() == arg_name.lower(): 207 | found_args.append(arg) 208 | return found_args if found_args else None 209 | 210 | def _get_return_value_args_by_name(self, arg_name: str, is_regex_pattern: bool = False) -> Optional[List[Argument]]: 211 | """ 212 | Returns the arguments of the system call return value identified by the argument name 213 | :param arg_name: Argument name 214 | :param is_regex_pattern: Indicates whether the argument name is a regex pattern 215 | :return: List of arguments if arguments were identified, otherwise None 216 | """ 217 | found_args = [] 218 | if isinstance(self.return_value, Pointer): 219 | for arg in self.return_value.arguments: 220 | if is_regex_pattern: 221 | if self._is_regex_matching(arg.name, arg_name): 222 | found_args.append(arg) 223 | else: 224 | if arg.name.lower() == arg_name.lower(): 225 | found_args.append(arg) 226 | return found_args if found_args else None 227 | 228 | def _deep_argument_search(self, arg_name: str, is_in: Optional[bool], is_out: Optional[bool], 229 | is_regex_pattern: bool = False) -> Optional[List[Any]]: 230 | """ 231 | Searches for argument in all child objects (Pointer objects) 232 | :param arg_name: Name of argument to search for 233 | :param is_in: Indicates whether argument is inbound 234 | :param is_out: Indicates whether argument is outbound 235 | :param is_regex_pattern: Indicates whether arg_name is a regex pattern 236 | :return: List of values of searched argument 237 | """ 238 | found_values = [] 239 | found_args = self._get_arguments_by_name(arg_name) 240 | if found_args: 241 | for arg in found_args: 242 | if is_in is not None: 243 | if arg.is_in != is_in: 244 | continue 245 | if is_out is not None: 246 | if arg.is_out != is_out: 247 | continue 248 | if isinstance(arg.value, Pointer): 249 | if not isinstance(arg.value.arguments, list): 250 | found_values.append(arg.value.arguments) 251 | continue 252 | found_values.append(arg.value) 253 | for arg in self.arguments: 254 | if isinstance(arg.value, Pointer): 255 | if isinstance(arg.value.arguments, list): 256 | arg_vals = arg.value.get_argument_values(arg_name, is_regex_pattern) 257 | if arg_vals: 258 | found_values += arg_vals 259 | return found_values if len(found_values) > 0 else None 260 | 261 | def get_as_dict(self) -> Dict[str, Any]: 262 | """ 263 | Returns the system call object as dictionary 264 | :return: SystemCall object as dictionary 265 | """ 266 | result_dict = { 267 | "api_call_index": self.index, 268 | "api_call_flog_index": self.flog_index, 269 | "api_call_function": self.function_name, 270 | "api_call_time": self.time, 271 | "api_call_args": [], 272 | "api_call_return_value": None, 273 | } 274 | for arg in self.arguments: 275 | result_dict["api_call_args"].append(arg.get_as_dict()) 276 | # Return value conversion 277 | if isinstance(self.return_value, Pointer): 278 | result_dict["api_call_return_value"] = self.return_value.convert() 279 | else: 280 | result_dict["api_call_return_value"] = self.return_value 281 | return result_dict 282 | 283 | def convert(self) -> Dict[str, Any]: 284 | """ 285 | Converts the system call to the dynmx flog format 286 | :return: Dictionary representing the SystemCall object in the dynmx flog format 287 | """ 288 | convert_result = { 289 | "flog_index": str(uuid4()), 290 | "function_name": self.function_name, 291 | "time": self.time, 292 | "arguments": [], 293 | "return_value": None, 294 | } 295 | # Argument conversion 296 | for arg in self.arguments: 297 | convert_result["arguments"].append(arg.convert()) 298 | # Return value conversion 299 | if isinstance(self.return_value, Pointer): 300 | convert_result["return_value"] = self.return_value.convert() 301 | else: 302 | convert_result["return_value"] = self.return_value 303 | return convert_result 304 | 305 | @staticmethod 306 | def _is_regex_matching(string_to_check: str, regex_pattern: str, ignore_case: bool = True) -> bool: 307 | """ 308 | Checks whether the the given string is matching the regex pattern 309 | :param string_to_check: String to check for regex pattern 310 | :param regex_pattern: Regex pattern 311 | :param ignore_case: Indicates whether to match case insensitive 312 | :return: Whether the string was matched by the regex pattern 313 | """ 314 | if ignore_case: 315 | p = re.compile(regex_pattern, re.IGNORECASE) 316 | else: 317 | p = re.compile(regex_pattern) 318 | return (p.search(string_to_check) is not None) 319 | 320 | def __eq__(self, other: APICall) -> bool: 321 | if not isinstance(other, APICall): 322 | return NotImplemented 323 | is_equal = True 324 | attributes = self.__dict__.keys() 325 | for attr in attributes: 326 | if attr == "arguments": 327 | if len(self.arguments) != len(other.arguments): 328 | is_equal = False 329 | break 330 | for ix, arg in enumerate(self.arguments): 331 | is_equal &= (arg == other.arguments[ix]) 332 | is_equal &= (getattr(self, attr) == getattr(other, attr)) 333 | return is_equal 334 | 335 | def __str__(self) -> str: 336 | return "{} ({}; {})".format(self.function_name, self.flog_index, self.index) 337 | 338 | def __repr__(self) -> str: 339 | return str(self) 340 | 341 | 342 | class ApiCallSignature: 343 | """ 344 | Representation of an API call signature without concrete values 345 | """ 346 | 347 | def __init__(self, function_name: Optional[str] = None, description: Optional[str] = None, 348 | calling_convention: Optional[str] = None, return_type: Optional[str] = None, 349 | return_value_desc: Optional[str] = None): 350 | """ 351 | Constructor 352 | :param function_name: Function name 353 | :param description: Description of the API call 354 | :param calling_convention: Calling convention 355 | :param return_type: Type of the return value 356 | :param return_value_desc: Description of the return value 357 | """ 358 | self.function_name = function_name 359 | self.description = description 360 | self.calling_convention = calling_convention 361 | self.return_type = return_type 362 | self.return_value_desc = return_value_desc 363 | self.arguments = [] 364 | 365 | 366 | class Argument: 367 | """ 368 | Representation of an API call argument 369 | """ 370 | 371 | def __init__(self, name: Optional[str] = None, value: Optional[Any] = None, is_in: bool = False, 372 | is_out: bool = False): 373 | """ 374 | Constructor 375 | :param name: Argument name 376 | :param value: Argument value 377 | :param is_in: Defines whether the argument is inbound 378 | :param is_out: Defines whether the argument is outbound 379 | """ 380 | self.name = name 381 | self.value = value 382 | self.is_in = is_in 383 | self.is_out = is_out 384 | 385 | def get_as_dict(self) -> Dict[str, Any]: 386 | """ 387 | Returns the system call argument object as dictionary 388 | :return: System call argument object as dictionary 389 | """ 390 | result_dict = { 391 | "api_call_arg_name": self.name, 392 | "api_call_arg_is_in": self.is_in, 393 | "api_call_arg_is_out": self.is_out, 394 | } 395 | # Argument conversion to dict 396 | # Is value pointer? 397 | if isinstance(self.value, Pointer): 398 | result_dict["value"] = self.value.get_as_dict() 399 | else: 400 | result_dict["value"] = self.value 401 | return result_dict 402 | 403 | def convert(self) -> Dict[str, Any]: 404 | """ 405 | Converts the Argument object to the dynmx format 406 | :return: Dictionary representing the converted Argument object 407 | """ 408 | convert_result = { 409 | "name": self.name, 410 | "is_in": self.is_in, 411 | "is_out": self.is_out, 412 | } 413 | # Is value pointer? 414 | if isinstance(self.value, Pointer): 415 | convert_result["value"] = self.value.convert() 416 | else: 417 | convert_result["value"] = self.value 418 | return convert_result 419 | 420 | def __eq__(self, other: Argument) -> bool: 421 | if not isinstance(other, Argument): 422 | return NotImplemented 423 | is_equal = True 424 | attributes = self.__dict__.keys() 425 | for attr in attributes: 426 | is_equal &= (getattr(self, attr) == getattr(other, attr)) 427 | return is_equal 428 | -------------------------------------------------------------------------------- /dynmx/core/file_resource.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Dict, Any 20 | 21 | from dynmx.core.resource import Resource, AccessType 22 | 23 | 24 | class FileResource(Resource): 25 | """ 26 | Representation of an OS file resource 27 | """ 28 | 29 | def __init__(self, access: Optional[AccessType] = None, path: Optional[str] = None): 30 | """ 31 | Constructor 32 | """ 33 | self.path = path 34 | Resource.__init__(self, access) 35 | 36 | def get_location(self) -> str: 37 | return self.path 38 | 39 | def get_as_dict(self) -> Dict[str, Any]: 40 | result_dict = { 41 | "path": self.path, 42 | "access_operations": [] 43 | } 44 | for op in self.access_operations: 45 | result_dict["access_operations"].append(op.name) 46 | return result_dict 47 | 48 | def __str__(self) -> str: 49 | return self.path 50 | -------------------------------------------------------------------------------- /dynmx/core/function_log.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Dict, Any, TYPE_CHECKING 20 | from datetime import datetime 21 | from enum import Flag, auto 22 | 23 | if TYPE_CHECKING: 24 | from dynmx.core.process import Process 25 | 26 | 27 | class FunctionLog: 28 | """ 29 | Representation of a function log 30 | """ 31 | 32 | def __init__(self, flog_type: FunctionLogType, name: Optional[str] = None, file_path: Optional[str] = None, 33 | version: Optional[str] = None, sandbox: Optional[str] = None, sandbox_version: Optional[str] = None, 34 | analysis_ts: Optional[datetime] = None): 35 | """ 36 | Constructor 37 | :param name: Name of function log 38 | :param file_path: Path to function log file 39 | """ 40 | self.flog_type = flog_type 41 | self.name = name 42 | self.file_path = file_path 43 | self.version = version 44 | self.sandbox = sandbox 45 | self.sandbox_version = sandbox_version 46 | self.analysis_ts = analysis_ts 47 | self.processes = [] 48 | 49 | def add_process(self, process: Process) -> None: 50 | process.flog_path = self.file_path 51 | self.processes.append(process) 52 | 53 | def get_as_dict(self, include_processes: bool = False, include_api_calls: bool = False) -> Dict[str, Any]: 54 | """ 55 | Returns the function log object as dict 56 | :param include_processes: Decides whether to include the processes 57 | :param include_api_calls: Decides whether to include the system calls 58 | :return: Function log object as dict 59 | """ 60 | result_dict = { 61 | "flog_name": self.name, 62 | "flog_file_path": self.file_path, 63 | } 64 | if include_processes: 65 | result_dict["flog_processes"] = [] 66 | for p in self.processes: 67 | p_dict = p.get_as_dict(include_api_calls) 68 | result_dict["flog_processes"].append(p_dict) 69 | return result_dict 70 | 71 | def convert(self) -> Dict[str, Any]: 72 | """ 73 | Converts the function log object and inherited objects to the dynmx 74 | function log format 75 | :return: Function log object in the dynmx flog format 76 | """ 77 | convert_result = { 78 | "flog": { 79 | "version": "1.0", 80 | "sandbox": self.sandbox, 81 | "sandbox_version": self.sandbox_version, 82 | "analysis_ts": datetime.strftime(self.analysis_ts, "%d.%m.%Y %H:%M:%S.%f"), 83 | "processes": [] 84 | } 85 | } 86 | for p in self.processes: 87 | converted_p = p.convert() 88 | convert_result["flog"]["processes"].append(converted_p) 89 | return convert_result 90 | 91 | def extract_resources(self): 92 | for p in self.processes: 93 | p.extract_resources(self.flog_type) 94 | 95 | def __eq__(self, other: FunctionLog) -> bool: 96 | if not isinstance(other, FunctionLog): 97 | return NotImplemented 98 | is_equal = True 99 | attributes = self.__dict__.keys() 100 | for attr in attributes: 101 | if attr == "processes": 102 | if len(self.processes) != len(other.processes): 103 | is_equal = False 104 | break 105 | for ix, proc in enumerate(self.processes): 106 | is_equal &= (proc == other.processes[ix]) 107 | is_equal &= (getattr(self, attr) == getattr(other, attr)) 108 | return is_equal 109 | 110 | 111 | class FunctionLogType(Flag): 112 | VMRAY = auto() 113 | CUCKOO = auto() 114 | CAPE = auto() 115 | DYNMX = auto() 116 | -------------------------------------------------------------------------------- /dynmx/core/network_resource.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Dict, Any 20 | from enum import Flag 21 | from urllib.parse import urlparse 22 | import ipaddress 23 | from dynmx.core.resource import Resource, AccessType 24 | from dynmx.helpers.resource_helper import ResourceHelper 25 | 26 | 27 | class NetworkResource(Resource): 28 | """ 29 | Representation of a Registry resource 30 | """ 31 | 32 | def __init__(self, access: Optional[AccessType] = None, url: Optional[str] = None, 33 | network_type: Optional[NetworkType] = None, ip_address: Optional[Any] = None, 34 | port: Optional[int] = None, dns_name: Optional[str] = None): 35 | """ 36 | Constructor 37 | """ 38 | self.url = None 39 | if url: 40 | self.set_url(url) 41 | if network_type: 42 | self.network_type = network_type 43 | else: 44 | self.network_type = NetworkType.UNDEFINED 45 | self.ip_address = ip_address 46 | self.port = port 47 | self.dns_name = dns_name 48 | Resource.__init__(self, access) 49 | 50 | def get_as_dict(self) -> Dict[str, Any]: 51 | result_dict = { 52 | "access_operations": [] 53 | } 54 | for op in self.access_operations: 55 | result_dict["access_operations"].append(op.name) 56 | if self.url: 57 | result_dict["url"] = self.url 58 | if self.port: 59 | result_dict["port"] = self.port 60 | if self.dns_name: 61 | result_dict["dns_name"] = self.dns_name 62 | if self.ip_address: 63 | result_dict["ip_address"] = str(self.ip_address) 64 | return result_dict 65 | 66 | def get_location(self) -> str: 67 | return self.get_url() 68 | 69 | def set_url(self, url: str) -> None: 70 | if ResourceHelper.is_url(url): 71 | self.url = url 72 | parsed_url = urlparse(url) 73 | net_location = parsed_url.netloc 74 | if ":" in net_location: 75 | location_parts = net_location.split(":") 76 | host = location_parts[0] 77 | self.set_host(host) 78 | self.port = location_parts[1] 79 | else: 80 | self.set_host(net_location) 81 | 82 | def set_host(self, host: str) -> None: 83 | if ResourceHelper.is_ip_address(host): 84 | self.ip_address = ipaddress.ip_address(host) 85 | else: 86 | self.dns_name = host 87 | 88 | def get_host(self) -> str: 89 | if self.dns_name: 90 | host = self.dns_name 91 | else: 92 | host = str(self.ip_address) 93 | if self.port: 94 | host += ":" + str(self.port) 95 | return host 96 | 97 | def get_url(self) -> str: 98 | if self.url: 99 | return self.url 100 | else: 101 | return self.get_host() 102 | 103 | def is_host(self, host: str, port: Optional[int] = None) -> bool: 104 | if ResourceHelper.is_ip_address(host): 105 | is_host = str(self.ip_address) == host 106 | else: 107 | is_host = self.dns_name == host 108 | if is_host and port: 109 | if self.port == port: 110 | is_host = True 111 | return is_host 112 | 113 | def has_ip(self) -> bool: 114 | return self.ip_address is not None 115 | 116 | def __str__(self) -> str: 117 | if self.url: 118 | return str(self.url) 119 | else: 120 | if self.dns_name: 121 | host = self.dns_name 122 | else: 123 | host = str(self.ip_address) 124 | if self.port: 125 | host += ":" + str(self.port) 126 | return host 127 | 128 | 129 | class NetworkType(Flag): 130 | UNDEFINED = 1 131 | CONNECT = 2 132 | DNS = 3 133 | LISTEN = 4 134 | -------------------------------------------------------------------------------- /dynmx/core/pointer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, List, Dict, Any, TYPE_CHECKING 20 | import re 21 | 22 | # Avoid circular import 23 | if TYPE_CHECKING: 24 | from dynmx.core.api_call import Argument 25 | 26 | 27 | class Pointer: 28 | """ 29 | Representation of a pointer 30 | """ 31 | 32 | def __init__(self, address: Optional[int] = None): 33 | """ 34 | Constructor 35 | :param address: Address of the pointer 36 | """ 37 | self.address = address 38 | self.arguments = [] 39 | 40 | def get_argument_values(self, arg_name: str, is_regex_pattern: bool = False) -> List[Any]: 41 | """ 42 | Returns the value of the argument identified by arg_name 43 | :param arg_name: Name of argument to return value from 44 | :param is_regex_pattern: Decides whether arg_name should be handled as 45 | regex pattern 46 | :return: List of values of arg_name 47 | """ 48 | found_values = [] 49 | # Does the pointer reference another pointer? 50 | if isinstance(self.arguments, Pointer): 51 | values = self.arguments.get_argument_values(arg_name) 52 | if values: 53 | found_values += values 54 | # Has the pointer multiple arguments? 55 | elif isinstance(self.arguments, list): 56 | # Argument is directly addressed 57 | if ":" in arg_name: 58 | arg_parts = arg_name.split(":") 59 | if not len(arg_parts): 60 | return found_values 61 | addressed_arg_name = arg_parts[0] 62 | args = self._get_arguments_by_name(addressed_arg_name, is_regex_pattern) 63 | if args and len(args): 64 | addressed_arg = args[0] 65 | else: 66 | addressed_arg = None 67 | if addressed_arg: 68 | if len(arg_parts) > 1: 69 | sub_arg = ":".join(arg_parts[1:]) 70 | arg_vals = addressed_arg.value.get_argument_values(sub_arg, is_regex_pattern) 71 | if arg_vals: 72 | found_values += arg_vals 73 | else: 74 | # Search for arg_name as argument name 75 | values = self._search_argument_values(arg_name, is_regex_pattern) 76 | if values: 77 | found_values += values 78 | return found_values if found_values else None 79 | 80 | def _get_arguments_by_name(self, arg_name: str, is_regex_pattern: bool = False) -> List[Argument]: 81 | """ 82 | Returns the arguments of the pointer identified by the argument name 83 | :param arg_name: Argument name 84 | :param is_regex_pattern: Indicates whether the argument name is a regex pattern 85 | :return: List of arguments if arguments were identified, otherwise None 86 | """ 87 | found_args = [] 88 | for arg in self.arguments: 89 | if is_regex_pattern: 90 | if self._is_regex_matching(arg.name, arg_name): 91 | found_args.append(arg) 92 | else: 93 | if arg.name.lower() == arg_name.lower(): 94 | found_args.append(arg) 95 | return found_args if found_args else None 96 | 97 | def _search_argument_values(self, arg_name: str, is_regex_pattern: bool = False) -> Optional[List[Any]]: 98 | # Avoid import loop 99 | from dynmx.core.api_call import Argument 100 | found_values = [] 101 | if isinstance(self.arguments, list): 102 | for arg in self.arguments: 103 | if not isinstance(arg, Argument): 104 | break 105 | if isinstance(arg.value, Pointer): 106 | ptr_arg_vals = arg.value.get_argument_values(arg_name, is_regex_pattern) 107 | if ptr_arg_vals: 108 | found_values += ptr_arg_vals 109 | else: 110 | if is_regex_pattern: 111 | if self._is_regex_matching(arg.name, arg_name): 112 | # Argument found 113 | found_values.append(arg.value) 114 | else: 115 | if arg.name == arg_name: 116 | # Argument found 117 | found_values.append(arg.value) 118 | return found_values if found_values else None 119 | 120 | def get_as_dict(self) -> Dict[str, Any]: 121 | """ 122 | Returns the pointer object as dict 123 | :return: Pointer object as dict 124 | """ 125 | # Avoid import loop 126 | from dynmx.core.api_call import Argument 127 | result_dict = { 128 | "pointer_address": self.address, 129 | "pointer_args": [], 130 | } 131 | if isinstance(self.arguments, list): 132 | for arg in self.arguments: 133 | if isinstance(arg, Argument): 134 | result_dict["pointer_args"].append(arg.get_as_dict()) 135 | else: 136 | result_dict["pointer_args"].append(arg) 137 | else: 138 | result_dict["pointer_args"] = self.arguments 139 | return result_dict 140 | 141 | def convert(self) -> Dict[str, Any]: 142 | """ 143 | Converts the pointer object to the dynmx flog format 144 | :return: Pointer object in dynmx flog format 145 | """ 146 | # Avoid import loop 147 | from dynmx.core.api_call import Argument 148 | convert_result = { 149 | "address": self.address, 150 | "arguments": [], 151 | } 152 | # Argument of pointer is a pointer 153 | if isinstance(self.arguments, Pointer): 154 | convert_result["arguments"] = self.arguments.convert() 155 | elif isinstance(self.arguments, list): 156 | for arg in self.arguments: 157 | if isinstance(arg, Argument) or isinstance(arg, Pointer): 158 | convert_result["arguments"].append(arg.convert()) 159 | else: 160 | convert_result["arguments"].append(arg) 161 | else: 162 | convert_result["arguments"] = self.arguments 163 | return convert_result 164 | 165 | def _get_arguments_by_name(self, arg_name, is_regex_pattern=False) -> Optional[List[Argument]]: 166 | found_args = [] 167 | for arg in self.arguments: 168 | if is_regex_pattern: 169 | if self._is_regex_matching(arg.name, arg_name): 170 | found_args.append(arg) 171 | else: 172 | if arg.name == arg_name: 173 | found_args.append(arg) 174 | return found_args if found_args else None 175 | 176 | @staticmethod 177 | def _is_regex_matching(string_to_check: str, regex_pattern: str, ignore_case: bool = True) -> bool: 178 | """ 179 | Returns whether regex_pattern matches string_to_check 180 | :param string_to_check: String that should be matched to regex_pattern 181 | :param regex_pattern: Regex pattern 182 | :param ignore_case: Decides whether to ignore case while matching 183 | :return: Indicates whether regex_pattern has matched 184 | """ 185 | if ignore_case: 186 | p = re.compile(regex_pattern, re.IGNORECASE) 187 | else: 188 | p = re.compile(regex_pattern) 189 | return (p.search(string_to_check) is not None) 190 | 191 | def __eq__(self, other: Pointer) -> bool: 192 | if not isinstance(other, Pointer): 193 | return NotImplemented 194 | is_equal = (self.address == other.address) 195 | if isinstance(self.arguments, list): 196 | if not isinstance(other.arguments, list): 197 | return False 198 | if len(self.arguments) != len(other.arguments): 199 | return False 200 | for ix, arg in enumerate(self.arguments): 201 | is_equal &= (arg == other.arguments[ix]) 202 | else: 203 | is_equal &= (self.arguments == other.arguments) 204 | return is_equal 205 | -------------------------------------------------------------------------------- /dynmx/core/process.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, List, Dict, Set, Any, TYPE_CHECKING 20 | 21 | from dynmx.helpers.regex_helper import RegexHelper 22 | from dynmx.detection.access_activity_model import AccessActivityModel 23 | 24 | if TYPE_CHECKING: 25 | from dynmx.core.api_call import APICall 26 | from dynmx.core.function_log import FunctionLogType 27 | 28 | 29 | class Process: 30 | """ 31 | Representation of a process 32 | """ 33 | 34 | def __init__(self, flog_path: Optional[str] = None, os_id: Optional[int] = None, name: Optional[str] = None, 35 | file_path: Optional[str] = None, cmd_line: Optional[str] = None, owner: Optional[str] = None): 36 | """ 37 | Constructor 38 | :param os_id: OS given PID of process 39 | :param name: Name of process 40 | :param file_path: Path to executable of process 41 | :param cmd_line: Command line the process was started with 42 | :param owner: Owner of process 43 | """ 44 | self.flog_path = flog_path 45 | self.os_id = os_id 46 | self.name = name 47 | self.file_path = file_path 48 | self.cmd_line = cmd_line 49 | self.owner = owner 50 | self.api_calls = list() 51 | self.api_call_lookup_table = dict() 52 | self.aam = None 53 | 54 | def get_as_dict(self, include_api_calls: bool = False) -> Dict[str, Any]: 55 | """ 56 | Returns the process object as dict 57 | :param include_api_calls: Decides whether to include API calls 58 | :return: Process object as dict 59 | """ 60 | result_dict = { 61 | "proc_os_id": self.os_id, 62 | "proc_name": self.name, 63 | "proc_file_path": self.file_path, 64 | "proc_cmd_line": self.cmd_line, 65 | "proc_owner": self.owner, 66 | } 67 | if include_api_calls: 68 | result_dict["api_calls"] = [] 69 | for api_call in self.api_calls: 70 | s_dict = api_call.get_as_dict() 71 | result_dict["api_calls"].append(s_dict) 72 | return result_dict 73 | 74 | def convert(self) -> Dict[str, Any]: 75 | """ 76 | Converts the process object to the dynmx flog format 77 | :return: Process object in the dynmx flog format 78 | """ 79 | convert_result = { 80 | "os_id": self.os_id, 81 | "name": self.name, 82 | "file_path": self.file_path, 83 | "cmd_line": self.cmd_line, 84 | "owner": self.owner, 85 | "api_calls": [], 86 | } 87 | for api_call in self.api_calls: 88 | converted_api_call = api_call.convert() 89 | convert_result["api_calls"].append(converted_api_call) 90 | return convert_result 91 | 92 | def add_api_call(self, api_call: APICall) -> None: 93 | """ 94 | Adds an API call to the process 95 | :param api_call: APICall object to add 96 | """ 97 | self.api_calls.append(api_call) 98 | api_call_name = api_call.function_name 99 | if api_call_name in self.api_call_lookup_table.keys(): 100 | self.api_call_lookup_table[api_call_name].append(api_call.index) 101 | else: 102 | self.api_call_lookup_table[api_call_name] = list() 103 | self.api_call_lookup_table[api_call_name].append(api_call.index) 104 | 105 | def get_api_calls_by_name(self, function_name: str, is_regex_pattern: bool = False) -> List[APICall]: 106 | """ 107 | Returns a list of API calls identified by the function name 108 | :param function_name: Function name 109 | :param is_regex_pattern: Is function_name regular expression? 110 | :return: List of identified API calls 111 | """ 112 | api_calls = list() 113 | api_call_indices = list() 114 | if not is_regex_pattern: 115 | if function_name not in self.api_call_lookup_table.keys(): 116 | return [] 117 | api_call_indices = self.api_call_lookup_table[function_name] 118 | else: 119 | for fname in self.api_call_lookup_table: 120 | if RegexHelper.is_regex_matching(fname, function_name): 121 | api_call_indices += self.api_call_lookup_table[fname] 122 | uniq_indices = set(api_call_indices) 123 | if uniq_indices: 124 | for ix in uniq_indices: 125 | api_calls.append(self.api_calls[ix]) 126 | return api_calls 127 | 128 | def get_api_call_function_names(self) -> Set[str]: 129 | """ 130 | Returns the function names of all API calls that are part of the process 131 | :return: Set of function names the process's API calls 132 | """ 133 | return set(self.api_call_lookup_table.keys()) 134 | 135 | def has_api_call_function_name(self, function_name: str, is_regex_pattern: bool = False) -> bool: 136 | """ 137 | Indicates whether the process has API calls with the given function name 138 | :param function_name: Function name of the API call 139 | :param is_regex_pattern: Indicates whether function_name is a regex pattern 140 | :return: 141 | """ 142 | if is_regex_pattern: 143 | for api_call in self.get_api_call_function_names(): 144 | if RegexHelper.is_regex_matching(api_call, function_name): 145 | return True 146 | return False 147 | else: 148 | return function_name in self.get_api_call_function_names() 149 | 150 | def extract_resources(self, flog_type: FunctionLogType) -> None: 151 | self.aam = AccessActivityModel(flog_type) 152 | self.aam.build(self) 153 | 154 | def __eq__(self, other: Process) -> bool: 155 | if not isinstance(other, Process): 156 | return NotImplemented 157 | is_equal = True 158 | attributes = self.__dict__.keys() 159 | for attr in attributes: 160 | if attr == "api_calls": 161 | if len(self.api_calls) != len(other.api_calls): 162 | is_equal = False 163 | break 164 | for ix, api_call in enumerate(self.api_calls): 165 | is_equal &= (api_call == other.api_calls[ix]) 166 | is_equal &= (getattr(self, attr) == getattr(other, attr)) 167 | return is_equal 168 | -------------------------------------------------------------------------------- /dynmx/core/registry_resource.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Dict, Any 20 | 21 | from dynmx.core.resource import Resource, AccessType 22 | 23 | 24 | class RegistryResource(Resource): 25 | """ 26 | Representation of a Registry resource 27 | """ 28 | 29 | def __init__(self, access: Optional[AccessType] = None, path: Optional[str] = None): 30 | """ 31 | Constructor 32 | """ 33 | self.path = path 34 | Resource.__init__(self, access) 35 | 36 | def get_as_dict(self) -> Dict[str, Any]: 37 | result_dict = { 38 | "path": self.path, 39 | "access_operations": [] 40 | } 41 | for op in self.access_operations: 42 | result_dict["access_operations"].append(op.name) 43 | return result_dict 44 | 45 | def get_location(self) -> str: 46 | return self.path 47 | 48 | def __str__(self) -> str: 49 | return self.path 50 | -------------------------------------------------------------------------------- /dynmx/core/resource.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Dict, Any 20 | from enum import Flag, auto 21 | from uuid import uuid4 22 | 23 | 24 | class Resource: 25 | """ 26 | Representation of an OS resource 27 | """ 28 | 29 | def __init__(self, access: Optional[AccessType] = None): 30 | """ 31 | Constructor 32 | """ 33 | self.id = uuid4() 34 | if access: 35 | self.access_operations = set([access]) 36 | else: 37 | self.access_operations = set() 38 | 39 | # Needs to be implemented in the specialized class 40 | def get_location(self) -> str: 41 | pass 42 | 43 | # Needs to be implemented in the specialized class 44 | def get_as_dict(self) -> Dict[str, Any]: 45 | pass 46 | 47 | def has_access_operation(self, access_operation: AccessType) -> bool: 48 | return access_operation in self.access_operations 49 | 50 | 51 | class AccessType(Flag): 52 | UNDEFINED = 1 53 | READ = 2 54 | WRITE = 3 55 | EXECUTE = 4 56 | CREATE = 5 57 | DELETE = 6 58 | 59 | @staticmethod 60 | def get_entry_by_str(str_entry: str) -> Optional[AccessType]: 61 | if str_entry.lower() == "read": 62 | return AccessType.READ 63 | elif str_entry.lower() == "write": 64 | return AccessType.WRITE 65 | elif str_entry.lower() == "execute": 66 | return AccessType.EXECUTE 67 | elif str_entry.lower() == "create": 68 | return AccessType.CREATE 69 | elif str_entry.lower() == "delete": 70 | return AccessType.DELETE 71 | else: 72 | return None 73 | -------------------------------------------------------------------------------- /dynmx/core/statistics.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | import operator 20 | 21 | from dynmx.core.function_log import FunctionLog 22 | 23 | 24 | class Statistics: 25 | """ 26 | Represents statistics of function logs 27 | """ 28 | 29 | def __init__(self, flog: FunctionLog): 30 | """ 31 | Constructor 32 | :param flog: flog object to create statistics from 33 | """ 34 | self.flog = flog 35 | self.num_of_processes = 0 36 | self.num_of_api_calls = 0 37 | self.num_of_unique_api_calls = 0 38 | self.flop_api_calls = {} 39 | self.top_api_calls = {} 40 | self.api_call_stats = {} 41 | 42 | def calculate(self) -> None: 43 | """ 44 | Calculates the statistics 45 | """ 46 | self._prepare_data() 47 | 48 | def _prepare_data(self, flop_limit: int = 10) -> None: 49 | """ 50 | Prepares the data for statistics 51 | :param flop_limit: Limit for top and flop system call count 52 | """ 53 | self.num_of_processes = len(self.flog.processes) 54 | for p in self.flog.processes: 55 | for api_call in p.api_calls: 56 | if api_call.function_name not in self.api_call_stats.keys(): 57 | self.api_call_stats[api_call.function_name] = 1 58 | self.num_of_unique_api_calls += 1 59 | else: 60 | self.api_call_stats[api_call.function_name] += 1 61 | self.num_of_api_calls += 1 62 | sorted_api_calls = sorted(self.api_call_stats.items(), 63 | key=operator.itemgetter(1)) 64 | index = 0 65 | while index < flop_limit and index < len(sorted_api_calls): 66 | self.flop_api_calls[sorted_api_calls[index][0]] = sorted_api_calls[index][1] 67 | self.top_api_calls[sorted_api_calls[::-1][index][0]] = sorted_api_calls[::-1][index][1] 68 | index += 1 69 | -------------------------------------------------------------------------------- /dynmx/detection/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /dynmx/detection/detection_result.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Dict, Any 20 | 21 | 22 | class DetectionResult: 23 | """ 24 | Representation of a detection result 25 | """ 26 | 27 | def __init__(self, flog_name: Optional[str] = None, flog_path: Optional[str] = None, 28 | signature_name: Optional[str] = None, signature_path: Optional[str] = None, detected: bool = False): 29 | """ 30 | Constructor 31 | """ 32 | self.flog_name = flog_name 33 | self.flog_path = flog_path 34 | self.signature_name = signature_name 35 | self.signature_path = signature_path 36 | self.detected = detected 37 | self.runtime_flog_parsing = None 38 | self.runtime_resource_extraction = None 39 | self.runtime_signature_detection = None 40 | self.detected_processes = [] 41 | 42 | def get_as_dict(self) -> Dict[str, Any]: 43 | """ 44 | Returns DetectionResult object as dictionary 45 | :return: Dictionary representing the DetectionResult object 46 | """ 47 | result_dict = { 48 | "flog": self.flog_name, 49 | "flog_path": self.flog_path, 50 | "signature": self.signature_name, 51 | "signature_path": self.signature_path, 52 | "detected": self.detected, 53 | "detected_processes": [], 54 | } 55 | if self.runtime_flog_parsing: 56 | result_dict["runtime_flog_parsing"] = self.runtime_flog_parsing 57 | if self.runtime_resource_extraction: 58 | result_dict["runtime_resource_extraction"] = self.runtime_resource_extraction 59 | if self.runtime_signature_detection: 60 | result_dict["runtime_signature_detection"] = self.runtime_signature_detection 61 | for p in self.detected_processes: 62 | p_dict = p.get_as_dict() 63 | result_dict["detected_processes"].append(p_dict) 64 | return result_dict 65 | 66 | 67 | class DetectedProcess: 68 | """ 69 | Representation of a detected process 70 | """ 71 | 72 | def __init__(self, os_id: Optional[int] = None, name: Optional[str] = None, file_path: Optional[str] = None, 73 | cmd_line: Optional[str] = None, owner: Optional[str] = None): 74 | """ 75 | Constructor 76 | :param process: Process that was detected 77 | """ 78 | self.process_os_id = os_id 79 | self.process_name = name 80 | self.process_file_path = file_path 81 | self.process_cmd_line = cmd_line 82 | self.process_owner = owner 83 | self.findings = [] 84 | 85 | def get_as_dict(self) -> Dict[str, Any]: 86 | """ 87 | Returns the object as dictionary 88 | :return: DetectedProcess object as dictionary 89 | """ 90 | proc_info = { 91 | "os_id": self.process_os_id, 92 | "name": self.process_name, 93 | "file_path": self.process_file_path, 94 | "cmd_line": self.process_cmd_line, 95 | "owner": self.process_owner, 96 | } 97 | result_dict = { 98 | "process": proc_info, 99 | "findings": [], 100 | } 101 | for finding in self.findings: 102 | f_dict = finding.get_as_dict() 103 | result_dict["findings"].append(f_dict) 104 | return result_dict 105 | 106 | 107 | class DetectedBlock: 108 | """ 109 | Representation of a finding 110 | """ 111 | def __init__(self, detection_block_key: Optional[str] = None): 112 | """ 113 | Constructor 114 | :param detection_block_key: Key of the detection block that was 115 | detected 116 | """ 117 | self.detection_block_key = detection_block_key 118 | self.api_calls = [] 119 | self.resources = [] 120 | 121 | def get_as_dict(self) -> Dict[str, Any]: 122 | """ 123 | Returns the object as dictionary 124 | :return: DetectedStep object as dictionary 125 | """ 126 | result_dict = { 127 | "detection_block_key": self.detection_block_key, 128 | "api_calls": [], 129 | "resources": [], 130 | } 131 | for api_call in self.api_calls: 132 | s_dict = api_call.get_as_dict() 133 | result_dict["api_calls"].append(s_dict) 134 | for resource in self.resources: 135 | r_dict = resource.get_as_dict() 136 | result_dict["resources"].append(r_dict) 137 | return result_dict 138 | -------------------------------------------------------------------------------- /dynmx/detection/graph.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, List 20 | 21 | 22 | class DirectedAcyclicGraph: 23 | """ 24 | Representation of a directed acyclic graph (DAG) 25 | """ 26 | 27 | def __init__(self): 28 | self._graph = {} 29 | self.start_nodes = list() 30 | 31 | def get_nodes(self) -> List[GraphNode]: 32 | """ 33 | Returns the nodes of the graph 34 | :return: List containing the nodes of the graph 35 | """ 36 | return list(self._graph.keys()) 37 | 38 | def get_edges(self) -> List[List[GraphNode]]: 39 | """ 40 | Returns the directed edges of the graph 41 | :return: List of tuples containing the edges of the graph. Index 0 contains the start node, index 1 the end 42 | node of the edge. 43 | """ 44 | return self._generate_edges() 45 | 46 | def get_end_nodes(self) -> List[GraphNode]: 47 | """ 48 | Returns the end nodes of the graph. End nodes are indicated by having no neighbours. 49 | :return: List of end nodes of the graph 50 | """ 51 | # End nodes have no neighbours 52 | end_nodes = list() 53 | for key, node in self._graph.items(): 54 | if not node.has_neighbours(): 55 | end_nodes.append(node) 56 | return end_nodes 57 | 58 | def add_node(self, node: GraphNode) -> None: 59 | """ 60 | Adds a node to the graph 61 | :param node: GraphNode that should be added 62 | """ 63 | if not isinstance(node, GraphNode): 64 | return 65 | # Graph is empty; set start_node and add node to graph 66 | if not self.start_nodes: 67 | self.start_nodes.append(node) 68 | self._graph[node] = node 69 | # Graph is not empty 70 | elif node not in self._graph.keys(): 71 | self._graph[node] = node 72 | 73 | def add_edge(self, edge: GraphNode) -> None: 74 | """ 75 | Adds an edge to the graph 76 | :param edge: Tuple containing two GraphNode objects. Index 0 is the start node, index 1 the end node of the edge 77 | """ 78 | if len(edge) != 2: 79 | return 80 | start_node = edge[0] 81 | end_node = edge[1] 82 | if not isinstance(start_node, GraphNode) or not isinstance(end_node, GraphNode): 83 | return 84 | if start_node == end_node: 85 | return 86 | if start_node in self._graph.keys(): 87 | self._graph[start_node].add_neighbour(end_node) 88 | else: 89 | self.add_node(start_node) 90 | self._graph[start_node].add_neighbour(end_node) 91 | if end_node not in self._graph.keys(): 92 | self.add_node(end_node) 93 | 94 | def concat(self, junction_node: GraphNode, graph: DirectedAcyclicGraph) -> None: 95 | """ 96 | Concatenates two graphs on a defined junction node 97 | :param junction_node: GraphNode object to that the graph should be concatenated 98 | :param graph: DirectedAcyclicGraph object that should be concatenated 99 | """ 100 | # Junction node is not part of graph 101 | if junction_node: 102 | if junction_node not in self._graph.keys(): 103 | return 104 | # Add edge from junction node to start node of graph that should be added 105 | for start_node in graph.start_nodes: 106 | if junction_node: 107 | self.add_edge([junction_node, start_node]) 108 | else: 109 | self.start_nodes.append(start_node) 110 | # Add all nodes of the added graph 111 | for node in graph.get_nodes(): 112 | if node not in self._graph.keys(): 113 | self._graph[node] = node 114 | 115 | def find_all_paths(self) -> List[List[GraphNode]]: 116 | """ 117 | Finds all paths from the start nodes to the end nodes of the graph 118 | :return: List containing all paths of the graph 119 | """ 120 | all_paths = [] 121 | # Iterate over all start nodes and find the paths to end nodes beginning from these start nodes recursively 122 | for start_node in self.start_nodes: 123 | paths = self._find_all_paths_recursively(start_node) 124 | if paths: 125 | all_paths += paths 126 | return all_paths 127 | 128 | def _find_all_paths_recursively(self, start_node: GraphNode, path: List[GraphNode] = []): 129 | """ 130 | Recursively finds the paths beginning from the start node 131 | :param start_node: Node to start the path search from 132 | :param path: Found path 133 | :return: Paths found beginning from the start node 134 | """ 135 | path = path.copy() + [start_node] 136 | if not start_node.has_neighbours(): 137 | return [path] 138 | if start_node not in self._graph.keys(): 139 | return [] 140 | # Visit every neighbour of the start node 141 | paths_from_start = [] 142 | for neighbour in start_node.get_neighbours(): 143 | if neighbour not in path: 144 | new_paths = self._find_all_paths_recursively(neighbour, path) 145 | for new_path in new_paths: 146 | paths_from_start.append(new_path) 147 | return paths_from_start 148 | 149 | def _generate_edges(self) -> List[List[GraphNode]]: 150 | """ 151 | Generates the edges of the path based on the nodes and their neighbours 152 | :return: List of edges 153 | """ 154 | edges = list() 155 | # Iterate over all nodes in the graph and find their neighbours 156 | # Every neighbour of a give node results in a new edge 157 | for node in self._graph: 158 | for neighbour in self._graph[node].get_neighbours(): 159 | edge = [node, neighbour] 160 | edges.append(edge) 161 | return edges 162 | 163 | 164 | class GraphNode: 165 | """ 166 | Representation of a generic graph node 167 | """ 168 | 169 | def __init__(self, neighbours: Optional[List[GraphNode]] = None): 170 | if not neighbours: 171 | neighbours = list() 172 | self._neighbours = neighbours 173 | 174 | def get_neighbours(self) -> List[GraphNode]: 175 | """ 176 | Returns the neighbours of the node 177 | :return: List of neighbours of the node 178 | """ 179 | return self._neighbours 180 | 181 | def add_neighbour(self, neighbour_node: GraphNode) -> None: 182 | """ 183 | Adds a neighbour to the node 184 | :param neighbour_node: GraphNode object that should be added as neighbour 185 | """ 186 | if neighbour_node not in self._neighbours: 187 | self._neighbours.append(neighbour_node) 188 | 189 | def has_neighbours(self) -> bool: 190 | """ 191 | Indicates whether the node has neighbours 192 | :return: Bool indicating whether the node has neighbours 193 | """ 194 | return len(self._neighbours) > 0 195 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/cape_flog_parser.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Dict, Any, List 20 | import json 21 | import ntpath 22 | import time 23 | import os 24 | 25 | from dynmx.flog_parsers.parser import Parser 26 | from dynmx.core.process import Process 27 | from dynmx.core.api_call import APICall, Argument 28 | from dynmx.core.function_log import FunctionLog, FunctionLogType 29 | from dynmx.helpers.flog_parser_helper import FlogParserHelper 30 | 31 | 32 | class CapeFlogParser(Parser): 33 | """ 34 | Parser for text-based CAPE sandbox function logs (part of report.json 35 | file) 36 | """ 37 | 38 | def __init__(self): 39 | """ 40 | Constructor 41 | """ 42 | Parser.__init__(self) 43 | 44 | def parse(self, file_path: str) -> FunctionLog: 45 | """ 46 | Parses the CAPE function log (included in the report.json file in the JSON key "behavior") 47 | :param file_path: Path to the text-based CAPE report.json file 48 | :return: FunctionLog object containing the parsed CAPE function log 49 | """ 50 | function_log = FunctionLog(FunctionLogType.CAPE) 51 | with open(file_path, "r") as flog: 52 | self._content = json.load(flog) 53 | function_log.name = ntpath.basename(file_path) 54 | function_log.file_path = os.path.abspath(file_path) 55 | function_log.sandbox = "CAPE" 56 | # Parse process information and corresponding API calls 57 | if "behavior" not in self._content.keys(): 58 | raise Exception( 59 | "Could not parse CAPE function log. Key 'behavior' not present.") 60 | if "processes" not in self._content["behavior"].keys(): 61 | raise Exception( 62 | "Could not parse CAPE function log. Key 'processes' not present.") 63 | for process in self._content["behavior"]["processes"]: 64 | p = self._parse_process_info(process) 65 | function_log.add_process(p) 66 | return function_log 67 | 68 | @staticmethod 69 | def probe(file_path: str) -> bool: 70 | """ 71 | Probes whether the file is a CAPE report.json file 72 | :param file_path: Path to the file to probe 73 | :return: Indicates whether the file is a CAPE report.json file 74 | """ 75 | result = False 76 | if not FlogParserHelper.is_gzip_compressed(file_path): 77 | with open(file_path, "r", encoding="utf-8", errors="ignore") as f: 78 | first_lines = [next(f) for x in range(5)] 79 | result = first_lines[0].strip() == "{" and \ 80 | first_lines[1].strip() == "\"statistics\": {" and \ 81 | first_lines[2].strip() == "\"processing\": [" and \ 82 | first_lines[4].strip() == "\"name\": \"CAPE\"," 83 | return result 84 | 85 | def _parse_process_info(self, process: Dict[str, Any]) -> Process: 86 | """ 87 | Parses the process information from the CAPE function log 88 | :param process: Process section of CAPE report.json 89 | :return: Process object containing the parsed process information 90 | """ 91 | # Define mapping of fields 92 | mapping = { 93 | "os_id": "process_id", 94 | "name": "process_name", 95 | "file_path": "module_path", 96 | } 97 | p = Process() 98 | for k, v in mapping.items(): 99 | setattr(p, k, process[v]) 100 | p.cmd_line = process["environ"]["CommandLine"] 101 | self._process_start_time = self._convert_to_unix_ts(process["first_seen"]) 102 | # Parse API calls of the process 103 | api_call_index = 0 104 | for api_call in process["calls"]: 105 | s = self._parse_api_call(api_call) 106 | s.index = api_call_index 107 | p.add_api_call(s) 108 | api_call_index += 1 109 | return p 110 | 111 | def _parse_api_call(self, api_call_dict: Dict[str, Any]) -> APICall: 112 | """ 113 | Parses the API call information from the CAPE function log 114 | :param api_call_dict: API call section of CAPE report.json 115 | :return: APICall object containing the parsed information 116 | """ 117 | api_call = APICall() 118 | api_call.function_name = api_call_dict["api"] 119 | api_call.return_value = FlogParserHelper.parse_value(api_call_dict["return"]) 120 | # Convert absolute time to relative time since first api call of 121 | # process 122 | api_call.time = self._convert_time_to_relative(self._convert_to_unix_ts(api_call_dict["timestamp"])) 123 | # Harmonize arguments 124 | args = self._parse_args(api_call_dict["arguments"]) 125 | api_call.arguments = args 126 | return api_call 127 | 128 | @staticmethod 129 | def _convert_to_unix_ts(ts_str: str) -> float: 130 | ts = FlogParserHelper.parse_timestamp(ts_str) 131 | return time.mktime(ts.timetuple()) + ts.microsecond/1e6 132 | 133 | def _convert_time_to_relative(self, ts: float) -> float: 134 | """ 135 | Converts the absolute time of a API call to the relative time since 136 | the start of the process 137 | :param time: Time to convert as UNIX timestamp 138 | :return: Relative time 139 | """ 140 | return (ts - self._process_start_time) 141 | 142 | @staticmethod 143 | def _parse_args(args: List[Dict[str, Any]]) -> List[Argument]: 144 | """ 145 | Parses arguments of an API call 146 | :param args: Arguments to parse 147 | :return: Parsed argument objects 148 | """ 149 | parsed_args = [] 150 | for arg in args: 151 | arg_obj = Argument() 152 | arg_obj.name = arg["name"] 153 | arg_obj.value = FlogParserHelper.parse_value(arg["value"]) 154 | parsed_args.append(arg_obj) 155 | return parsed_args 156 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/cuckoo_flog_parser.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Dict, Any, List 20 | import json 21 | import ntpath 22 | import os 23 | 24 | from dynmx.flog_parsers.parser import Parser 25 | from dynmx.core.process import Process 26 | from dynmx.core.api_call import APICall, Argument 27 | from dynmx.core.function_log import FunctionLog, FunctionLogType 28 | from dynmx.helpers.flog_parser_helper import FlogParserHelper 29 | 30 | 31 | class CuckooFlogParser(Parser): 32 | """ 33 | Parser for text-based Cuckoo sandbox function logs (part of report.json 34 | file) 35 | """ 36 | 37 | def __init__(self): 38 | """ 39 | Constructor 40 | """ 41 | Parser.__init__(self) 42 | 43 | def parse(self, file_path: str) -> FunctionLog: 44 | """ 45 | Parses the Cuckoo function log (in report.json file) 46 | :param file_path: Path to the text-based Cuckoo report.json file 47 | :return: FunctionLog object containing the parsed Cuckoo function log 48 | """ 49 | function_log = FunctionLog(FunctionLogType.CUCKOO) 50 | with open(file_path, "r") as flog: 51 | self._content = json.load(flog) 52 | function_log.name = ntpath.basename(file_path) 53 | function_log.file_path = os.path.abspath(file_path) 54 | function_log.sandbox = "Cuckoo" 55 | # Parse process information and corresponding API calls 56 | if "behavior" not in self._content.keys(): 57 | raise Exception( 58 | "Could not parse Cuckoo function log. Key 'behavior' not present.") 59 | if "processes" not in self._content["behavior"].keys(): 60 | raise Exception( 61 | "Could not parse Cuckoo function log. Key 'processes' not present.") 62 | for process in self._content["behavior"]["processes"]: 63 | p = self._parse_process_info(process) 64 | function_log.add_process(p) 65 | return function_log 66 | 67 | @staticmethod 68 | def probe(file_path: str) -> bool: 69 | """ 70 | Probes whether the file is a Cuckoo report.json file 71 | :param file_path: Path to the file to probe 72 | :return: Indicates whether the file is a Cuckoo report.json file 73 | """ 74 | result = False 75 | if not FlogParserHelper.is_gzip_compressed(file_path): 76 | with open(file_path, "r", encoding="utf-8", errors="ignore") as f: 77 | first_lines = [next(f) for x in range(2)] 78 | result = first_lines[0].strip() == "{" and \ 79 | first_lines[1].strip() == "\"info\": {" 80 | return result 81 | 82 | def _parse_process_info(self, process: Dict[str, Any]) -> Process: 83 | """ 84 | Parses the process information from the Cuckoo function log 85 | :param process: Process section of Cuckoo report.json 86 | :return: Process object containing the parsed process information 87 | """ 88 | # Define mapping of fields 89 | mapping = { 90 | "os_id": "pid", 91 | "name": "process_name", 92 | "file_path": "process_path", 93 | "cmd_line": "command_line", 94 | } 95 | p = Process() 96 | for k, v in mapping.items(): 97 | setattr(p, k, process[v]) 98 | self._process_start_time = process["first_seen"] 99 | # Parse API calls of the process 100 | api_call_index = 0 101 | for api_call in process["calls"]: 102 | s = self._parse_api_call(api_call) 103 | s.index = api_call_index 104 | p.add_api_call(s) 105 | api_call_index += 1 106 | return p 107 | 108 | def _parse_api_call(self, api_call_dict): 109 | """ 110 | Parses the API call information from the Cuckoo function log 111 | :param api_call_dict: API call section of Cuckoo report.json 112 | :return: APICall object containing the parsed information 113 | """ 114 | api_call = APICall() 115 | api_call.function_name = api_call_dict["api"] 116 | api_call.return_value = FlogParserHelper.parse_value(api_call_dict["return"]) 117 | # Convert absolute time to relative time since first api call of 118 | # process 119 | api_call.time = self._convert_time_to_relative(api_call_dict["time"]) 120 | # Harmonize arguments 121 | args = self._parse_args(api_call_dict["arguments"]) 122 | api_call.arguments = args 123 | return api_call 124 | 125 | def _convert_time_to_relative(self, time: float) -> float: 126 | """ 127 | Converts the absolute time of a API call to the relative time since 128 | the start of the process 129 | :param time: Time to convert as UNIX timestamp 130 | :return: Relative time 131 | """ 132 | return (time - self._process_start_time) 133 | 134 | @staticmethod 135 | def _parse_args(args: List[Dict[str, Any]]) -> List[Argument]: 136 | """ 137 | Parses arguments of an API call 138 | :param args: Arguments to parse 139 | :return: Parsed argument objects 140 | """ 141 | parsed_args = [] 142 | for arg in args: 143 | arg_obj = Argument() 144 | arg_obj.name = arg["name"] 145 | arg_obj.value = FlogParserHelper.parse_value(arg["value"]) 146 | parsed_args.append(arg_obj) 147 | return parsed_args 148 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/dynmx_flog_parser.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import List, Dict, Any 20 | import json 21 | import ntpath 22 | import gzip 23 | import os 24 | from datetime import datetime 25 | 26 | from dynmx.flog_parsers.parser import Parser 27 | from dynmx.core.process import Process 28 | from dynmx.core.api_call import APICall, Argument 29 | from dynmx.core.function_log import FunctionLog, FunctionLogType 30 | from dynmx.core.pointer import Pointer 31 | from dynmx.helpers.flog_parser_helper import FlogParserHelper 32 | 33 | 34 | class DynmxFlogParser(Parser): 35 | """ 36 | Parser for text-based generic dynmx function logs 37 | """ 38 | 39 | def __init__(self): 40 | """ 41 | Constructor 42 | """ 43 | Parser.__init__(self) 44 | 45 | def parse(self, file_path: str) -> FunctionLog: 46 | """ 47 | Parses the dynmx generic function log 48 | :param file_path: Path to the text-based dynmx generic function log 49 | :return: FunctionLog object containing the parsed dynmx function log 50 | """ 51 | function_log = FunctionLog(FunctionLogType.DYNMX) 52 | if FlogParserHelper.is_gzip_compressed(file_path): 53 | with gzip.open(file_path, "rb") as f: 54 | content = f.read().decode(encoding='utf-8', errors='ignore').split("\n") 55 | else: 56 | with open(file_path, "r", encoding='utf-8', errors='ignore') as flog: 57 | content = flog.readlines() 58 | for index, line in enumerate(content): 59 | if not line.startswith("#"): 60 | break 61 | self._content = json.loads("".join(content[index:])) 62 | function_log.name = ntpath.basename(file_path) 63 | function_log.file_path = os.path.abspath(file_path) 64 | # Parse process information and corresponding API calls 65 | if "flog" not in self._content.keys(): 66 | raise Exception("Could not parse dynmx function log. Key 'flog' not present.") 67 | if "processes" not in self._content["flog"].keys(): 68 | raise Exception("Could not parse dynmx function log. Key 'processes' not present.") 69 | # Parse flog information 70 | self._parse_flog_info(function_log, self._content["flog"]) 71 | # Parse processes 72 | for process in self._content["flog"]["processes"]: 73 | p = self._parse_process_info(process) 74 | p.flog_path = function_log.file_path 75 | function_log.add_process(p) 76 | return function_log 77 | 78 | @staticmethod 79 | def probe(file_path: str) -> bool: 80 | """ 81 | Probes whether the file is a dynmx generic function log 82 | :param file_path: Path to the file to probe 83 | :return: Indicates whether the file is a dynmx function log 84 | """ 85 | if FlogParserHelper.is_gzip_compressed(file_path): 86 | with gzip.open(file_path, "rb") as f: 87 | first_line = f.readline().decode(encoding="utf-8", errors="ignore") 88 | else: 89 | with open(file_path, "r") as f: 90 | first_line = f.readline() 91 | result = (first_line.strip() == "# dynmx generic function log") 92 | return result 93 | 94 | def _parse_flog_info(self, flog_obj: FunctionLog, flog_dict: Dict[str, Any]) -> None: 95 | # Define mapping of fields 96 | attributes = [ 97 | "version", 98 | "sandbox", 99 | "sandbox_version", 100 | "analysis_ts", 101 | ] 102 | self._safe_set_obj_property(flog_obj, attributes, flog_dict) 103 | if flog_obj.analysis_ts: 104 | ts = datetime.strptime(flog_obj.analysis_ts, "%d.%m.%Y %H:%M:%S.%f") 105 | flog_obj.analysis_ts = ts 106 | 107 | def _parse_process_info(self, process: Dict[str, Any]) -> Process: 108 | """ 109 | Parses the process information from the dynmx function log 110 | :param process: Process section of dynmx function log 111 | :return: Process object containing the parsed process information 112 | """ 113 | # Define mapping of fields 114 | attributes = [ 115 | "os_id", 116 | "name", 117 | "file_path", 118 | "cmd_line", 119 | "owner", 120 | ] 121 | p = Process() 122 | for k in attributes: 123 | setattr(p, k, process[k]) 124 | # Parse API calls of the process 125 | for api_call_index, api_call in enumerate(process["api_calls"]): 126 | s = self._parse_api_call(api_call) 127 | s.index = api_call_index 128 | p.add_api_call(s) 129 | return p 130 | 131 | def _parse_api_call(self, api_call_dict: Dict[str, Any]) -> APICall: 132 | """ 133 | Parses the API call information from the dynmx function log 134 | :param api_call_dict: API call section of dynmx function log 135 | :return: APICall object containing the parsed information 136 | """ 137 | # Define attributes to parse 138 | attributes = [ 139 | "flog_index", 140 | "function_name", 141 | "time", 142 | "arguments", 143 | "return_value", 144 | ] 145 | # Parse attributes 146 | api_call = APICall() 147 | for k in attributes: 148 | if k in api_call_dict.keys(): 149 | setattr(api_call, k, api_call_dict[k]) 150 | args = list() 151 | for arg in api_call.arguments: 152 | a = self._parse_argument(arg) 153 | args.append(a) 154 | if not api_call.has_in_out_args and (a.is_in or a.is_out): 155 | api_call.has_in_out_args = True 156 | api_call.arguments = args 157 | # Replace pointer structures with objects 158 | if isinstance(api_call.return_value, dict): 159 | p = self._parse_pointer(api_call.return_value) 160 | api_call.return_value = p 161 | return api_call 162 | 163 | def _parse_pointer(self, pointer: Dict[str, Any]) -> Pointer: 164 | """ 165 | Parses a pointer structure 166 | :param pointer: Pointer structure as dict 167 | :return: Pointer object with the parsed information 168 | """ 169 | # Define attributes to parse 170 | attributes = [ 171 | "address", 172 | "arguments", 173 | ] 174 | # Parse attributes 175 | p = Pointer() 176 | for k in attributes: 177 | setattr(p, k, pointer[k] if k in pointer.keys() else None) 178 | if isinstance(pointer["arguments"], list): 179 | args = list() 180 | for arg in pointer["arguments"]: 181 | if isinstance(arg, dict): 182 | args.append(self._parse_argument(arg)) 183 | else: 184 | args.append(arg) 185 | p.arguments = args 186 | return p 187 | 188 | def _parse_argument(self, arg: Dict[str, Any]) -> Argument: 189 | """ 190 | Parses an argument structure 191 | :param arg: Argument structure as dict 192 | :return: Argument object with the parsed information 193 | """ 194 | # Define attributes to parse 195 | attributes = [ 196 | "name", 197 | "value", 198 | "is_in", 199 | "is_out", 200 | ] 201 | # Parse attributes 202 | a = Argument() 203 | for k in attributes: 204 | setattr(a, k, arg[k] if k in arg.keys() else None) 205 | # Replace pointer structures with objects 206 | if isinstance(a.value, dict): 207 | p_nested = self._parse_pointer(a.value) 208 | a.value = p_nested 209 | return a 210 | 211 | @staticmethod 212 | def _safe_set_obj_property(obj: Any, attribute_names: List[str], attr_values_dict: Dict[str, Any]) -> None: 213 | """ 214 | Sets attributes of the given object based on the given value dictionary 215 | :param obj: Object of that the attributes should be set 216 | :param attribute_names: List of attribute names to set 217 | :param attr_values_dict: Dictionary containing the attribute values 218 | """ 219 | for k in attribute_names: 220 | setattr(obj, k, attr_values_dict[k] if k in attr_values_dict.keys() else None) 221 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/parser.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import TYPE_CHECKING 20 | from abc import ABC, abstractmethod 21 | 22 | if TYPE_CHECKING: 23 | from dynmx.core.function_log import FunctionLog 24 | 25 | 26 | class Parser(ABC): 27 | """ 28 | Abstract base class for an input file parser 29 | """ 30 | 31 | _content = None 32 | processes = None 33 | 34 | def __init__(self): 35 | self.processes = list() 36 | super().__init__() 37 | 38 | @abstractmethod 39 | def parse(self, file_path: str) -> FunctionLog: 40 | """ 41 | Abstract parse function. Has to be overwritten in the specialized 42 | class. 43 | :param file_path: Path to the file to parse 44 | """ 45 | pass 46 | 47 | @abstractmethod 48 | def probe(file_path: str) -> bool: 49 | """ 50 | Abstract probe function. has to be overwritten in the specialized 51 | class. 52 | :param file_path: Path to the file to probe 53 | """ 54 | pass 55 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/parser_library.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Any 20 | import pkgutil 21 | import importlib 22 | 23 | 24 | class ParserLibrary: 25 | """ 26 | Representation of a library for available flog parsers 27 | """ 28 | 29 | def __init__(self): 30 | """ 31 | Constructor 32 | """ 33 | self.parsers = [] 34 | 35 | def load(self, parser_pkg: Any) -> None: 36 | """ 37 | Loads the available function log parsers 38 | :param parser_pkg: Python package that includes the parsers 39 | """ 40 | for finder, name, is_pkg in pkgutil.iter_modules(parser_pkg.__path__, 41 | parser_pkg.__name__ + "."): 42 | if is_pkg: 43 | continue 44 | if name != "dynmx.flog_parsers.parser" and name != "dynmx.flog_parsers.parser_library": 45 | module = importlib.import_module(name) 46 | class_name = self._get_parser_class(module.__file__) 47 | cls = getattr(module, class_name) 48 | self.parsers.append(cls) 49 | 50 | @staticmethod 51 | def _get_parser_class(path: str) -> str: 52 | """ 53 | Returns the class name of a flog parser class 54 | :param path: Path to flog parser class file 55 | :return: Class name of flog parser 56 | """ 57 | with open(path) as f: 58 | content = [next(f) for x in range(50)] 59 | for line in content: 60 | if line.strip().startswith("class"): 61 | class_name = line.strip().split(" ")[1][:-1] 62 | class_name = class_name.split("(")[0] 63 | return class_name 64 | -------------------------------------------------------------------------------- /dynmx/flog_parsers/vmray_xml_flog_parser.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Any, Optional, List 20 | import ntpath 21 | from lxml import etree as ET 22 | import os 23 | from datetime import datetime 24 | 25 | from dynmx.flog_parsers.parser import Parser 26 | from dynmx.core.process import Process 27 | from dynmx.core.api_call import APICall, Argument 28 | from dynmx.core.function_log import FunctionLog, FunctionLogType 29 | from dynmx.core.pointer import Pointer 30 | 31 | 32 | class VmrayXmlFlogParser(Parser): 33 | """ 34 | Parser for XML based VMray function logs 35 | """ 36 | 37 | def __init__(self): 38 | """ 39 | Constructor 40 | """ 41 | Parser.__init__(self) 42 | 43 | def parse(self, file_path: str) -> FunctionLog: 44 | """ 45 | Parses the VMray XML function log 46 | :param file_path: Path to the text-based VMray XML function log 47 | :return: FunctionLog object containing the parsed VMray XML function log 48 | """ 49 | function_log = FunctionLog(FunctionLogType.VMRAY) 50 | xml_root = ET.parse(file_path).getroot() 51 | function_log.name = ntpath.basename(file_path) 52 | function_log.file_path = os.path.abspath(file_path) 53 | function_log.sandbox = "VMRay Analyzer" 54 | # Parse header information 55 | self._parse_header_info(function_log, xml_root) 56 | # Parse process information and corresponding API calls 57 | for proc in xml_root.findall('monitor_process'): 58 | # Parse process information 59 | p = self._parse_process_info(proc) 60 | p.flog_path = function_log.file_path 61 | # Find and parse corresponding API calls 62 | vmray_proc_id = proc.attrib["process_id"] 63 | self._parse_all_api_calls(xml_root, vmray_proc_id, p) 64 | # Append process to function log 65 | function_log.add_process(p) 66 | xml_root.clear() 67 | xml_root = None 68 | del xml_root 69 | return function_log 70 | 71 | @staticmethod 72 | def probe(file_path: str) -> bool: 73 | """ 74 | Probes whether the file is a VMray XML function log file 75 | :param file_path: Path to the file to probe 76 | :return: Indicates whether the file is a VMray XML function log file 77 | """ 78 | with open(file_path, "r") as f: 79 | first_lines = [next(f) for x in range(2)] 80 | result = first_lines[0].strip() == '' and \ 81 | first_lines[1].strip().startswith(' APICall: 138 | """ 139 | Parses an API call node ( XML tag) 140 | :param api_call_node: XML node with tag containing API call information 141 | :return: APICall object containing the parsed API call information 142 | """ 143 | api_call = APICall() 144 | api_call.function_name = api_call_node.attrib["name"] 145 | api_call.index = api_call_index 146 | api_call.time = int(api_call_node.attrib["ts"]) / 1000 147 | api_call.flog_index = api_call_node.sourceline 148 | # Return Value 149 | return_val_node = api_call_node.find("./out/param[@name='ret_val']") 150 | if return_val_node is not None: 151 | if self._node_has_children(return_val_node) and return_val_node.attrib["type"] == "ptr": 152 | # Return value is a pointer 153 | api_call.return_value = self._parse_pointer_node(return_val_node) 154 | else: 155 | if self._node_has_attribute(return_val_node, "value"): 156 | ret_val_str = return_val_node.attrib["value"] 157 | api_call.return_value = self._parse_arg_value(ret_val_str) 158 | else: 159 | # API call returns void 160 | api_call.return_value = None 161 | else: 162 | api_call.return_value = None 163 | # Arguments 164 | arg_nodes_in = api_call_node.findall("./in/param") 165 | arg_nodes_out = api_call_node.xpath('./out/param[not(@name="ret_val")]') 166 | if len(arg_nodes_in) and len(arg_nodes_out): 167 | api_call.has_in_out_args = True 168 | api_call.arguments = self._parse_argument_nodes(arg_nodes_in, arg_nodes_out) 169 | return api_call 170 | 171 | @staticmethod 172 | def _node_has_children(node) -> bool: 173 | """ 174 | Indicates whether the XML node has children 175 | :param node: Node to check for children 176 | :return: Bool that indicates whether the XML node has children 177 | """ 178 | return len(node.getchildren()) > 0 179 | 180 | @staticmethod 181 | def _node_has_attribute(node, attribute: str) -> bool: 182 | """ 183 | Indicates whether the XML node has the given attribute 184 | :param node: Node to check for presence of attribute 185 | :param attribute: Name of attribute to check presence for 186 | :return: Bool that indicates whether the XML node has the attribute 187 | """ 188 | return attribute in node.attrib.keys() 189 | 190 | def _parse_argument_nodes(self, in_nodes: List[Any], out_nodes: List[Any]) -> List[Argument]: 191 | """ 192 | Parses the XML argument nodes ( and XML tag children) 193 | :param in_nodes: List of children nodes of the XML tag 194 | :param out_nodes: List of children nodes of the XML tag 195 | :return: List of Arguments objects containing the parsed XML argument nodes 196 | """ 197 | arguments = list() 198 | if len(in_nodes): 199 | # In parameters available 200 | for in_node in in_nodes: 201 | arg = self._parse_argument_node(in_node, is_in=True) 202 | arguments.append(arg) 203 | if len(out_nodes): 204 | # Out parameters available 205 | for out_node in out_nodes: 206 | arg = self._parse_argument_node(out_node, is_out=True) 207 | arguments.append(arg) 208 | return arguments 209 | 210 | def _parse_argument_node(self, node, is_in: bool = False, is_out: bool = False) -> Argument: 211 | """ 212 | Parses the XML argument node (XML tag ) 213 | :param node: Argument node to parse 214 | :param is_in: Indicates whether the node is an in parameter 215 | :param is_out: Indicates whether the node is an in parameter 216 | :return: Argument object containing the parsed parameter information 217 | """ 218 | arg_obj = Argument() 219 | arg_obj.is_in = is_in 220 | arg_obj.is_out = is_out 221 | if self._node_has_attribute(node, "name"): 222 | arg_obj.name = node.attrib["name"] 223 | if self._node_has_children(node): 224 | # Argument is a pointer 225 | arg_obj.value = self._parse_pointer_node(node) 226 | else: 227 | # Argument has simple value 228 | if self._node_has_attribute(node, "value"): 229 | arg_obj.value = self._parse_arg_value(node.attrib["value"]) 230 | return arg_obj 231 | 232 | def _parse_pointer_node(self, node) -> Pointer: 233 | """ 234 | Parses an XML pointer node (XML tags , , ) 235 | :param node: XML pointer node 236 | :return: Pointer object with the parsed information 237 | """ 238 | p = Pointer() 239 | if self._node_has_attribute(node, "value"): 240 | p.address = self._parse_arg_value(node.attrib["value"]) 241 | if node.attrib["type"] == "ptr": 242 | if self._node_has_children(node): 243 | child = node.getchildren()[0] 244 | if child.tag == "deref": 245 | # String value 246 | if child.attrib["type"] == "str": 247 | if self._node_has_attribute(child, "value"): 248 | return self._parse_arg_value(child.attrib["value"]) 249 | else: 250 | return None 251 | # No named arguments, just value 252 | elif self._node_has_attribute(child, "value"): 253 | p.arguments = self._parse_arg_value(child.attrib["value"]) 254 | # Named arguments 255 | elif child.attrib["type"] == "container": 256 | p.arguments = self._parse_container_node(child) 257 | # Array 258 | elif child.attrib["type"] == "array": 259 | p.arguments = self._parse_array_node(child) 260 | elif node.attrib["type"] == "array": 261 | p.arguments = self._parse_array_node(node) 262 | elif node.attrib["type"] == "container": 263 | p.arguments = self._parse_container_node(node) 264 | return p 265 | 266 | def _parse_container_node(self, node) -> List[Argument]: 267 | """ 268 | Parses the arguments of an XML container node () 269 | :param node: XML container node 270 | :return: List of Arguments objects 271 | """ 272 | container_args = list() 273 | for member in node.getchildren(): 274 | container_args.append(self._parse_argument_node(member)) 275 | return container_args 276 | 277 | def _parse_array_node(self, node) -> List[Argument]: 278 | """ 279 | Parses the arguments of an XML array node () 280 | :param node: XML array node 281 | :return: List of Argument objects 282 | """ 283 | args = list() 284 | for ix, item in enumerate(node.getchildren()): 285 | if item.attrib["type"] == "container": 286 | args += self._parse_container_node(item) 287 | else: 288 | i = self._parse_item_node(item) 289 | args.append(i) 290 | return args 291 | 292 | def _parse_item_node(self, node) -> Optional[Pointer | Argument]: 293 | if self._node_has_attribute(node, "type"): 294 | if node.attrib["type"] == "ptr": 295 | return self._parse_pointer_node(node) 296 | else: 297 | if self._node_has_attribute(node, "value"): 298 | return self._parse_arg_value(node.attrib["value"]) 299 | return None 300 | 301 | @staticmethod 302 | def _parse_arg_value(value: str) -> Any: 303 | """ 304 | Parses the argument value 305 | :param value: Value to parse 306 | :return: Parsed value 307 | """ 308 | value = value.lower() 309 | # Hexadecimal int 310 | if value.startswith("0x"): 311 | try: 312 | int_val = int(value, 0) 313 | return int_val 314 | except ValueError: 315 | pass 316 | # Int 317 | try: 318 | int_val = int(value) 319 | return int_val 320 | except ValueError: 321 | pass 322 | # String 323 | value = value.strip(" \"") 324 | value = value.replace("\\\\", "\\") 325 | return value 326 | 327 | @staticmethod 328 | def _parse_timestamp(str_val: str) -> Optional[datetime]: 329 | pattern = ["%d.%m.%Y %H:%M:%S.%f", "%d.%m.%Y %H:%M"] 330 | ts = None 331 | for p in pattern: 332 | try: 333 | ts = datetime.strptime(str_val, p) 334 | if ts: 335 | break 336 | except Exception: 337 | continue 338 | return ts 339 | -------------------------------------------------------------------------------- /dynmx/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /dynmx/helpers/argument_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | import argparse 20 | 21 | 22 | class ArgumentHelper: 23 | """ 24 | Represents a helpers for script argument handling 25 | """ 26 | 27 | def __init__(self): 28 | """ 29 | Constructor 30 | """ 31 | # Define default parameters 32 | self._default_params = { 33 | "format": "overview", 34 | "show-log": False, 35 | "detail": False, 36 | "log-level": "info", 37 | } 38 | 39 | def handle(self) -> argparse.Namespace: 40 | """ 41 | Handles the script parameter 42 | :return: Parsed arguments 43 | """ 44 | parser = self._build_arg_parser() 45 | args = parser.parse_args() 46 | return args 47 | 48 | def _build_arg_parser(self) -> argparse.ArgumentParser: 49 | """ 50 | Builds the needed argument parser 51 | :return: ArgumentParser object 52 | """ 53 | parser = argparse.ArgumentParser(description="Detect dynmx signatures in dynamic program execution information (function logs)") 54 | subparsers = parser.add_subparsers(title="sub-commands", 55 | description="task to perform", 56 | dest="command") 57 | # General script parameters 58 | parser.add_argument("--format", 59 | "-f", 60 | choices=["overview", "detail"], 61 | default=self._default_params["format"], 62 | help="Output format") 63 | parser.add_argument("--show-log", 64 | action='store_true', 65 | default=self._default_params["show-log"], 66 | help="Show all log output on stdout") 67 | parser.add_argument("--log", 68 | "-l", 69 | type=argparse.FileType("a+"), 70 | help="log file") 71 | parser.add_argument("--log-level", 72 | choices=["debug", "info", "error"], 73 | default=self._default_params["log-level"], 74 | help="Log level (default: {})".format(self._default_params["log-level"])) 75 | parser.add_argument("--worker", 76 | "-w", 77 | metavar="N", 78 | type=int, 79 | help="Number of workers to spawn (default: number of processors - 2)") 80 | # Parameters for command 'detect' 81 | parser_detect = subparsers.add_parser("detect", 82 | help="Detects a dynmx signature") 83 | req_detect_args = parser_detect.add_argument_group("required arguments") 84 | req_detect_args.add_argument("--sig", 85 | "-s", 86 | nargs="+", 87 | required=True, 88 | help="dynmx signature(s) to detect") 89 | req_detect_args.add_argument("--input", 90 | "-i", 91 | nargs="+", 92 | required=True, 93 | help="Input files") 94 | parser_detect.add_argument("--recursive", 95 | "-r", 96 | help="Search for input files recursively", 97 | action="store_true", 98 | default=False) 99 | parser_detect.add_argument("--json-result", 100 | help="JSON formatted result file", 101 | type=argparse.FileType("w+")) 102 | parser_detect.add_argument("--runtime-result", 103 | help="Runtime statistics file formatted in CSV", 104 | type=argparse.FileType("w+")) 105 | parser_detect.add_argument("--detect-all", 106 | help="Detect signature in all processes and do not stop after the first detection", 107 | action="store_true", 108 | default=False) 109 | # Parameters for command 'check' 110 | parser_check = subparsers.add_parser("check", 111 | help="Checks the syntax of dynmx signature(s)") 112 | req_check_args = parser_check.add_argument_group("required arguments") 113 | req_check_args.add_argument("--sig", 114 | "-s", 115 | nargs="+", 116 | required=True, 117 | help="dynmx signature(s) to check") 118 | # Parameters for command 'convert' 119 | parser_convert = subparsers.add_parser("convert", 120 | help="Converts function logs to the dynmx generic function log format") 121 | req_convert_args = parser_convert.add_argument_group( 122 | "required arguments") 123 | req_convert_args.add_argument("--input", 124 | "-i", 125 | nargs="+", 126 | required=True, 127 | help="Input files to convert") 128 | parser_convert.add_argument("--output-dir", 129 | "-o", 130 | help="Output directory for the converted files") 131 | parser_convert.add_argument("--nocompress", 132 | help="Do not compress the converted function log", 133 | action="store_true") 134 | parser_convert.add_argument("--recursive", 135 | help="Search for input files recursively", 136 | action="store_true", 137 | default=False) 138 | # Parameters for command 'stats' 139 | parser_stats = subparsers.add_parser("stats", 140 | help="Statistics of function logs") 141 | req_stats_args = parser_stats.add_argument_group( 142 | "required arguments") 143 | req_stats_args.add_argument("--input", 144 | "-i", 145 | nargs="+", 146 | required=True, 147 | help="Input files to calculate statistics from") 148 | parser_stats.add_argument("--recursive", 149 | "-r", 150 | help="Search for input files recursively", 151 | action="store_true", 152 | default=False) 153 | # Parameters for command 'resources' 154 | parser_resources = subparsers.add_parser("resources", 155 | help="Resource activity derived from function log") 156 | req_resources_args = parser_resources.add_argument_group( 157 | "required arguments") 158 | req_resources_args.add_argument("--input", 159 | "-i", 160 | nargs="+", 161 | required=True, 162 | help="Input files to derive resource activity from") 163 | parser_resources.add_argument("--recursive", 164 | "-r", 165 | help="Search for input files recursively", 166 | action="store_true", 167 | default=False) 168 | return parser 169 | -------------------------------------------------------------------------------- /dynmx/helpers/flog_parser_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, List, TYPE_CHECKING 20 | from datetime import datetime 21 | 22 | from dynmx.core.pointer import Pointer 23 | 24 | if TYPE_CHECKING: 25 | from dynmx.core.api_call import Argument 26 | 27 | 28 | class FlogParserHelper: 29 | """ 30 | Helper class for function log parsers 31 | """ 32 | 33 | @staticmethod 34 | def consolidate_args(args_in: List[Argument], args_out: List[Argument]) -> List[Argument]: 35 | """ 36 | Consolidates the in- and outbound arguments that were parsed based on the name and value of the argument 37 | :param args_in: List of Argument objects containing the inbound arguments of the API call 38 | :param args_out: List of Argument objects containing the outbound arguments of the API call 39 | :return: Consolidated list of Argument objects 40 | """ 41 | out_args_to_keep = [] 42 | if args_out and len(args_out) > 0: 43 | for ix, arg_out in enumerate(args_out): 44 | found_ix = FlogParserHelper._find_arg(args_in, arg_out.name) 45 | if found_ix is not None: 46 | arg_in = args_in[found_ix] 47 | if not isinstance(arg_out.value, Pointer): 48 | if arg_in.value is None or arg_in.value == arg_out.value: 49 | arg_in.value = arg_out.value 50 | arg_in.is_out = True 51 | else: 52 | out_args_to_keep.append(ix) 53 | else: 54 | if isinstance(arg_in.value, Pointer): 55 | if arg_in.value.address == arg_out.value.address: 56 | if isinstance(arg_in.value, list): 57 | if len(arg_in.value.arguments) == 0: 58 | arg_in.value.arguments = arg_out.value.arguments 59 | arg_in.is_out = True 60 | else: 61 | if not arg_in.value == arg_out.value: 62 | out_args_to_keep.append(ix) 63 | arg_in.is_out = True 64 | else: 65 | if arg_in.value.arguments is None or \ 66 | arg_in.value.arguments == arg_out.value.arguments: 67 | arg_in.value.arguments = arg_out.value.arguments 68 | arg_in.is_out = True 69 | else: 70 | out_args_to_keep.append(ix) 71 | else: 72 | if arg_in.value == arg_out.value.address: 73 | arg_in.value = arg_out.value 74 | arg_in.is_out = True 75 | else: 76 | out_args_to_keep.append(ix) 77 | for ix in out_args_to_keep: 78 | args_in.append(args_out[ix]) 79 | return args_in 80 | 81 | @staticmethod 82 | def _find_arg(args: List[Argument], name: str) -> Optional[Argument]: 83 | """ 84 | Finds an argument based on the name in a list of arguments 85 | :param args: List of Argument objects 86 | :param name: Name of argument to search for 87 | :return: Index of the found Argument object. If no argument was found None is returned. 88 | """ 89 | for ix, arg in enumerate(args): 90 | if arg.name == name: 91 | return ix 92 | return None 93 | 94 | @staticmethod 95 | def parse_timestamp(str_val: str) -> datetime: 96 | """ 97 | Parses a timestamp 98 | :param str_val: Timestamp string 99 | :return: Parsed timestamp 100 | """ 101 | # 2021-06-03 16:06:54,855 102 | pattern = ["%d.%m.%Y %H:%M:%S.%f", "%d.%m.%Y %H:%M", "%Y-%m-%d %H:%M:%S,%f"] 103 | ts = None 104 | for p in pattern: 105 | try: 106 | ts = datetime.strptime(str_val, p) 107 | if ts: 108 | break 109 | except Exception: 110 | continue 111 | return ts 112 | 113 | @staticmethod 114 | def is_gzip_compressed(file_path: str) -> bool: 115 | """ 116 | Checks whether a file is gzip compressed 117 | :param file_path: Path to the file that should be checked 118 | :return: Bool indicating whether the file is gzip compressed 119 | """ 120 | gzip_magic_bytes = b'\x1f\x8b' 121 | with open(file_path, "rb") as f: 122 | magic_bytes = f.read(2) 123 | return gzip_magic_bytes == magic_bytes 124 | 125 | @staticmethod 126 | def parse_value(value_str: str) -> int | str: 127 | """ 128 | Parses a value 129 | """ 130 | parsed_value = None 131 | # Is the value a hex value? 132 | if isinstance(value_str, str) and value_str.startswith("0x"): 133 | parsed_value = int(value_str, 0) 134 | else: 135 | parsed_value = value_str 136 | # Is the value a float 137 | if isinstance(value_str, str) and "." in value_str: 138 | try: 139 | parsed_value = float(parsed_value) 140 | except ValueError: 141 | pass 142 | # Is the value an integer? 143 | else: 144 | try: 145 | parsed_value = int(parsed_value) 146 | except ValueError: 147 | pass 148 | return parsed_value 149 | -------------------------------------------------------------------------------- /dynmx/helpers/logging_globals.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from multiprocessing import Queue 19 | from typing import Optional 20 | 21 | 22 | # Logging queue used for handling logs in multiprocessing scenarios 23 | logging_queue: Optional[Queue] = None -------------------------------------------------------------------------------- /dynmx/helpers/logging_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, TextIO 20 | import logging 21 | import logging.config 22 | import logging.handlers 23 | from multiprocessing import Manager, Queue, current_process 24 | from threading import Thread 25 | import time 26 | 27 | import dynmx.helpers.logging_globals 28 | 29 | # Idea for multiprocess logging found in 30 | # https://www.youtube.com/watch?v=axjfFB81FrE 31 | 32 | 33 | class LoggingThread: 34 | """ 35 | Represents a thread for centrally handling log messages that were written to a multiprocessing Queue 36 | """ 37 | 38 | def __init__(self): 39 | self._logging_thread = Thread( 40 | target=LoggingThread._logging_thread_run, 41 | args=(dynmx.helpers.logging_globals.logging_queue,) 42 | ) 43 | self._logging_thread.start() 44 | 45 | @staticmethod 46 | def _logging_thread_run(queue: Queue) -> None: 47 | """ 48 | Runs the logging thread 49 | :param queue: Multiprocessing queue for storing log messages 50 | """ 51 | while True: 52 | record = queue.get() 53 | if record is None: 54 | break 55 | logger = logging.getLogger(record.name) 56 | logger.handle(record) 57 | 58 | def terminate_logger(self) -> None: 59 | """ 60 | Terminates the logging thread 61 | """ 62 | dynmx.helpers.logging_globals.logging_queue.put(None) 63 | self._logging_thread.join() 64 | 65 | 66 | class LoggingHelper: 67 | """ 68 | Represents a centralized logger to log events 69 | """ 70 | 71 | @staticmethod 72 | def set_up_logging(log_level: Optional[int] = logging.DEBUG, logfile: Optional[TextIO] = None, 73 | show_console_log: Optional[bool] = False) -> LoggingThread: 74 | """ 75 | Sets up the logging configurations 76 | :param log_level: Log level 77 | :param logfile: Log file 78 | :param show_console_log: Indicates whether to show log messages on the console 79 | :return: Returns the logging thread which handles log messages 80 | """ 81 | root_logger = logging.getLogger() 82 | root_logger.setLevel(log_level) 83 | logging.Formatter.converter = time.gmtime 84 | log_formatter = logging.Formatter( 85 | "%(asctime)s+0000 [%(levelname)s] (%(name)s) [PID: %(process)d] [%(flog_path)s]: %(message)s") 86 | if show_console_log: 87 | console_handler = logging.StreamHandler() 88 | console_handler.setFormatter(log_formatter) 89 | root_logger.addHandler(console_handler) 90 | if logfile: 91 | file_handler = logging.StreamHandler(logfile) 92 | file_handler.setFormatter(log_formatter) 93 | root_logger.addHandler(file_handler) 94 | # Multiprocessing 95 | dynmx.helpers.logging_globals.logging_queue = Manager().Queue() 96 | return LoggingThread() 97 | 98 | @staticmethod 99 | def set_up_process_logging(log_level: Optional[int] = logging.DEBUG, queue: Optional[Queue] = None) -> None: 100 | """ 101 | Sets up logging for processes run via multiprocessing 102 | :param log_level: Log level 103 | :param queue: Multiprocessing queue to store log messages 104 | """ 105 | if current_process().name != 'MainProcess' and not Queue: 106 | pass 107 | elif queue is not None: 108 | queue_handler = logging.handlers.QueueHandler(queue) 109 | queue_handler.set_name(name=current_process().pid.__str__()) 110 | root_logger = logging.getLogger() 111 | root_logger.setLevel(log_level) 112 | if queue_handler.name not in [x.name for x in root_logger.handlers]: 113 | root_logger.addHandler(queue_handler) 114 | 115 | @staticmethod 116 | def get_logger(logger_name: str, flog_path: Optional[str] = "") -> logging.LoggerAdapter: 117 | """ 118 | Returns a new logger 119 | :param logger_name: Name of the logger 120 | :param flog_path: Function log path that should be included in the log message 121 | """ 122 | logger = logging.getLogger(logger_name) 123 | # Make use of logging adapters to inject the function log path in each log message if it is passed to this 124 | # function 125 | extra = {'flog_path': flog_path} 126 | logging_adapter = logging.LoggerAdapter(logger, extra) 127 | return logging_adapter 128 | 129 | @staticmethod 130 | def shutdown_log() -> None: 131 | """ 132 | Shuts the logging down 133 | """ 134 | logging.shutdown() 135 | -------------------------------------------------------------------------------- /dynmx/helpers/multiprocessing_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, Tuple 20 | from multiprocessing import Pool, cpu_count, set_start_method, Queue 21 | 22 | from dynmx.helpers.logging_helper import LoggingHelper 23 | 24 | 25 | class MultiprocessingHelper: 26 | """ 27 | Represents a helper for setting up multiprocessing 28 | """ 29 | 30 | @staticmethod 31 | def set_up() -> None: 32 | # Spawn new processes instead of forking them to be platform-independent 33 | set_start_method("spawn", force=True) 34 | 35 | @staticmethod 36 | def get_pool(num_of_workers: Optional[int] = None, log_level: Optional[int] = None, 37 | queue: Optional[Queue] = None) -> Tuple[int, Pool]: 38 | if not num_of_workers: 39 | num_of_workers = cpu_count() - 2 40 | # We need to initialize the logging in each process since we spawn processes instead of forking them 41 | proc_pool = Pool( 42 | processes=num_of_workers, 43 | maxtasksperchild=1000, 44 | initializer=LoggingHelper.set_up_process_logging, 45 | initargs=(log_level, queue) 46 | ) 47 | return num_of_workers, proc_pool 48 | 49 | @staticmethod 50 | def get_cpu_count() -> int: 51 | """ 52 | Returns the CPU count 53 | :return: CPU count 54 | """ 55 | return cpu_count() 56 | -------------------------------------------------------------------------------- /dynmx/helpers/output_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Optional, TYPE_CHECKING, List, Dict, Set, TextIO 20 | from enum import Flag, auto 21 | import json 22 | 23 | if TYPE_CHECKING: 24 | from dynmx.core.function_log import FunctionLog 25 | from dynmx.detection.detection_result import DetectionResult 26 | from dynmx.core.statistics import Statistics 27 | 28 | 29 | class OutputHelper: 30 | """ 31 | Helper class for output 32 | """ 33 | 34 | def __init__(self, show_log, output_format): 35 | self._show_log = show_log 36 | self._output_format = output_format 37 | 38 | def render_header(self, version: str) -> None: 39 | """ 40 | Prints the header 41 | :param version: Version to show in the header 42 | """ 43 | print(''' 44 | 45 | | 46 | __| _ _ _ _ _ 47 | / | | | / |/ | / |/ |/ | /\/ 48 | \_/|_/ \_/|/ | |_/ | | |_/ /\_/ 49 | /| 50 | \| 51 | ''') 52 | print(" Ver. {}, by 0x534a".format(version)) 53 | print("") 54 | print("") 55 | 56 | if self._show_log: 57 | print("[+] Log output") 58 | 59 | def render_detection_run_info(self, num_of_function_logs: int, num_of_signatures: int, resources_needed: bool, 60 | num_of_workers: int) -> None: 61 | """ 62 | Renders the detection run information 63 | :param num_of_function_logs: Number of function logs 64 | :param num_of_signatures: Number of signatures 65 | :param resources_needed: Indicates whether resources are extracted from the function log 66 | :param num_of_workers: Number of workers 67 | """ 68 | if self._show_log: 69 | return 70 | print("[+] Parsing {} function log(s)".format(num_of_function_logs)) 71 | print("[+] Loaded {} dynmx signature(s)".format(num_of_signatures)) 72 | if resources_needed: 73 | print("[+] Extracting resources from function log(s)") 74 | print("[+] Starting detection process with {} worker(s). This probably takes some time...".format(num_of_workers)) 75 | 76 | def render_flog_info(self, number_of_function_logs: int, command: str) -> None: 77 | """ 78 | Renders function log information 79 | :param number_of_function_logs: Number of function logs 80 | :param command: Command the function logs are processed with 81 | """ 82 | if self._show_log: 83 | return 84 | print("[+] Parsing {} function log(s)".format(number_of_function_logs)) 85 | print("[+] Processing function log(s) with the command '{}'...".format(command)) 86 | 87 | def _render_result_str(self, command_output: str) -> str: 88 | """ 89 | Renders the command output string 90 | :param command_output: Output of the command 91 | :return: Rendered result string 92 | """ 93 | output = "" 94 | if self._show_log: 95 | output += "[+] End of log output\n" 96 | output += "\n[+] Result\n" 97 | output += command_output 98 | return output 99 | 100 | def render_resources_output_str(self, flogs: List[FunctionLog]) -> str: 101 | """ 102 | Renders the resource output string 103 | :param flogs: List of function logs to extract resources information from 104 | :return: Rendered resources output string 105 | """ 106 | output = "" 107 | categories = ["filesystem", "registry", "network"] 108 | for ix, flog in enumerate(flogs): 109 | output += "Function log: {} ({})\n".format(flog.name, flog.file_path) 110 | for p in flog.processes: 111 | output += "\tProcess: {} (PID: {})\n".format(p.name,p.os_id) 112 | for cat in categories: 113 | resources = p.aam.get_resources_by_category(cat) 114 | if resources: 115 | output += "\t\t{}:\n".format(cat.capitalize()) 116 | for resource in resources: 117 | if self._output_format == OutputType.DETAIL: 118 | access_ops = [] 119 | for op in resource.access_operations: 120 | access_ops.append(op.name) 121 | output += "\t\t\t{} ({})\n".format(resource.get_location(), ",".join(access_ops)) 122 | else: 123 | output += "\t\t\t{}\n".format(resource.get_location()) 124 | return self._render_result_str(output) 125 | 126 | def _group_detection_results_by_flog(self, detection_results: List[DetectionResult]) \ 127 | -> Dict[str, List[DetectionResult]]: 128 | """ 129 | Groups detection results by the function log 130 | :param detection_results: List of detection results 131 | :return: Dictionary containing the detection results grouped by the function log 132 | """ 133 | detected_signatures_by_flog = {} 134 | for result in detection_results: 135 | if result and result.detected: 136 | if result.flog_name not in detected_signatures_by_flog.keys(): 137 | detected_signatures_by_flog[result.flog_path] = [] 138 | detected_signatures_by_flog[result.flog_path].append(result) 139 | return detected_signatures_by_flog 140 | 141 | def _group_detected_signatures_by_flog(self, detection_results: List[DetectionResult]) -> Dict[str, Set]: 142 | """ 143 | Groups the detected signatures by the function log 144 | :param detection_results: List of detection results 145 | :return: Dictionary of detected signatures grouped by the function log 146 | """ 147 | detected_signatures = {} 148 | for result in detection_results: 149 | if result and result.detected: 150 | if result.flog_name not in detected_signatures.keys(): 151 | detected_signatures[result.flog_path] = set() 152 | detected_signatures[result.flog_path].add(result.signature_name) 153 | return detected_signatures 154 | 155 | def render_detection_output_str(self, detection_results: List[DetectionResult]) -> str: 156 | """ 157 | Renders the detection output string 158 | :param detection_results: List of detection results 159 | :return: Rendered detection output string 160 | """ 161 | output = "" 162 | if self._output_format == OutputType.DETAIL: 163 | output = self._render_detailed_detection_output_str( 164 | self._group_detection_results_by_flog(detection_results) 165 | ) 166 | elif self._output_format == OutputType.OVERVIEW: 167 | detected_signatures = self._group_detected_signatures_by_flog(detection_results) 168 | for flog_path, detected_signatures in detected_signatures.items(): 169 | for sig in detected_signatures: 170 | output += "{}\t{}\n".format(sig, flog_path) 171 | return self._render_result_str(output) 172 | return self._render_result_str(output) 173 | 174 | @staticmethod 175 | def _render_detailed_detection_output_str(detection_results: Dict[str, List[DetectionResult]]) -> str: 176 | """ 177 | Returns the detection results for 'detail' output format 178 | :param detection_results: Detection results 179 | :return: Detection results for 'detail' output format as string 180 | """ 181 | output = "" 182 | for flog_path, results in detection_results.items(): 183 | output += "Function log: {}\n".format(flog_path) 184 | for result in results: 185 | output += "\tSignature: {}\n".format(result.signature_name) 186 | for p in result.detected_processes: 187 | output += "\t\tProcess: {} (PID: {})\n".format( 188 | p.process_name, 189 | p.process_os_id 190 | ) 191 | output += "\t\tNumber of Findings: {}\n".format(len(p.findings)) 192 | for index, finding in enumerate(p.findings): 193 | output += "\t\t\tFinding {}\n".format(index) 194 | for api_call in finding.api_calls: 195 | if api_call.flog_index: 196 | output += "\t\t\t\t{} : API Call {} (Function log line {}, index {})\n".format( 197 | finding.detection_block_key, 198 | api_call.function_name, 199 | api_call.flog_index, 200 | api_call.index 201 | ) 202 | else: 203 | output += "\t\t\t\t{} : API Call {} (index {})\n".format( 204 | finding.detection_block_key, api_call.function_name, api_call.index) 205 | for resource in finding.resources: 206 | output += "\t\t\t\t{} : Resource {}\n".format(finding.detection_block_key, resource) 207 | output += "\n" 208 | return output 209 | 210 | def render_check_output_str(self, check_results: List[bool]) -> str: 211 | """ 212 | Renders the check output 213 | :param check_results: Signature check results 214 | :return: Check output as string 215 | """ 216 | output = "" 217 | for sig_path, check_result in check_results.items(): 218 | if check_result: 219 | output += "[OK]]\t{}\n".format(sig_path) 220 | else: 221 | output += "[FAIL]\t{}\n".format(sig_path) 222 | return self._render_result_str(output) 223 | 224 | def render_stats_output_str(self, stats_objs: List[Statistics]) -> str: 225 | """ 226 | Generates the statistic output 227 | :param stats_objs: Statistic objects containing statistics for function log 228 | :return: Statistic output as string 229 | """ 230 | output = "" 231 | for ix, stats in enumerate(stats_objs): 232 | output += "Function log: {}\n".format(stats.flog.name) 233 | output += "\tNumber of Processes: {}\n".format(stats.num_of_processes) 234 | output += "\tNumber of API calls: {}\n".format(stats.num_of_api_calls) 235 | output += "\tNumber of unique API calls: {}\n".format(stats.num_of_unique_api_calls) 236 | output += "\tFlop API calls:\n" 237 | for sys_call_name, count in stats.flop_api_calls.items(): 238 | output += "\t\t{}: {}\n".format(sys_call_name, count) 239 | output += "\tTop API calls:\n" 240 | for sys_call_name, count in stats.top_api_calls.items(): 241 | output += "\t\t{}: {}\n".format(sys_call_name, count) 242 | return self._render_result_str(output) 243 | 244 | def render_error(self, message: str): 245 | """ 246 | Renders an error message 247 | :param message: Error message 248 | """ 249 | if not self._show_log: 250 | print("[-] {}".format(message)) 251 | 252 | def render_converted_flog(self, flog_path: str, conversion_time: float, output_path: str) -> None: 253 | """ 254 | Renders information of a converted function log 255 | :param flog_path: Function log path 256 | :param conversion_time: Run time of the conversion process 257 | :param output_path: Output path 258 | """ 259 | if not self._show_log: 260 | print("[+] Converted function log '{}' in {:.4f}s to output directory '{}'".format( 261 | flog_path, conversion_time, output_path)) 262 | 263 | @staticmethod 264 | def write_runtime_result_file(runtime_file: TextIO, result_objs: List[DetectionResult]) -> None: 265 | """ 266 | Writes the CSV-formatted runtime results file 267 | :param runtime_file: Runtime result file 268 | :param result_objs: Detection results 269 | """ 270 | csv_separator = "|" 271 | attributes = ["flog_name", "flog_path", "signature_name", "signature_path", "detected", "runtime_flog_parsing", 272 | "runtime_resource_extraction", "runtime_signature_detection"] 273 | header = csv_separator.join(attributes) 274 | runtime_file.write("{}\n".format(header)) 275 | for result_obj in result_objs: 276 | attribute_values = [] 277 | for attribute in attributes: 278 | if hasattr(result_obj, attribute): 279 | val = getattr(result_obj, attribute) 280 | if not val: 281 | val = "" 282 | if not isinstance(val, str): 283 | val = str(val) 284 | attribute_values.append(val) 285 | csv_line = csv_separator.join(attribute_values) 286 | runtime_file.write("{}\n".format(csv_line)) 287 | 288 | @staticmethod 289 | def write_detection_json_result_file(json_file: TextIO, result_objs: List[DetectionResult]) -> None: 290 | """ 291 | Writes the detection results as JSON file 292 | :param json_file: JSON result file 293 | :param result_objs: Detection results 294 | """ 295 | # Build dicts and write to result file with json 296 | detection_results_dict = [] 297 | for result in result_objs: 298 | if result: 299 | detection_results_dict.append(result.get_as_dict()) 300 | json_file.write(json.dumps(detection_results_dict, indent=2)) 301 | 302 | 303 | class OutputType(Flag): 304 | OVERVIEW = auto() 305 | DETAIL = auto() 306 | 307 | @staticmethod 308 | def get_entry_by_str(str_entry: str) -> Optional[OutputType]: 309 | if str_entry.lower() == "overview": 310 | return OutputType.OVERVIEW 311 | elif str_entry.lower() == "detail": 312 | return OutputType.DETAIL 313 | else: 314 | return None 315 | 316 | -------------------------------------------------------------------------------- /dynmx/helpers/regex_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from typing import Tuple 20 | import re 21 | 22 | 23 | # Precompile regex patterns for better performance 24 | REGEX_VMRAY_TIME = re.compile(r'^\[\d{4}\.\d{3}\]') 25 | REGEX_VMRAY_TIME_END = re.compile(r'^\[\d{4}\.\d{3}\]$') 26 | REGEX_API_CALL_NAME = re.compile(r'^[0-9a-zA-Z-_:?@].+$') 27 | REGEX_VMRAY_API_CALL_RETURN = re.compile(r'^\(.*?\)( returned.*?.+)?$') 28 | # Arguments 29 | REGEX_ARGUMENT_VALUE = re.compile(r'\"(?:\\.|[^\"\\])*\"') 30 | REGEX_ARGUMENT_VALUE_NORMALIZED = re.compile(r'\(normalized: (\"(?:\\.|[^\"\\])*\")\)') 31 | REGEX_CLOSING_PARANTHESES = re.compile(r'\)$') 32 | # Pointers 33 | REGEX_POINTER = re.compile(r'^0x[0-9a-fA-F]*\*') 34 | REGEX_STRUCT_POINTER = re.compile(r'^0x[0-9a-fA-F]*\*\(') 35 | REGEX_VALUE_POINTER = re.compile(r'^0x[0-9a-fA-F]*\*=') 36 | REGEX_NESTED_POINTER = re.compile(r'^0x[0-9a-fA-F]*\*') 37 | REGEX_POINTER_ADDRESS = re.compile(r'^0x[0-9a-fA-F]*') 38 | REGEX_PARANTHESES = re.compile(r'^\(*') 39 | 40 | 41 | class RegexHelper: 42 | @staticmethod 43 | def is_regex_matching(string_to_check: str, regex_pattern: str, ignore_case: bool = True) -> bool: 44 | """ 45 | Indicates whether a regex_pattern is matching a string 46 | :param string_to_check: String to check regex pattern against 47 | :param regex_pattern: Regex pattern 48 | :param ignore_case: Decides whether to matching is case insensitive 49 | :return: Indicates whether regex_pattern is matching string 50 | """ 51 | if ignore_case: 52 | p = re.compile(regex_pattern, re.IGNORECASE) 53 | else: 54 | p = re.compile(regex_pattern) 55 | return p.search(string_to_check) is not None 56 | 57 | @staticmethod 58 | def is_regex_pattern(string_to_check: str) -> bool: 59 | """ 60 | Checks whether a given string is a regex pattern 61 | :param string_to_check: String that should be checked 62 | :return: Indicates whether the string is a regex pattern 63 | """ 64 | regex_chars = "^([{+*.|\\$" 65 | for regex_char in regex_chars: 66 | if regex_char in string_to_check: 67 | return True 68 | return False 69 | 70 | @staticmethod 71 | def is_variable(string_to_check: str) -> bool: 72 | """ 73 | Checks whether the string is a variable (indicated by $()) 74 | :param string_to_check: String that should be checked 75 | :return: Indicates whether the string is a variable 76 | """ 77 | if isinstance(string_to_check, str): 78 | p = re.compile(r'^\$\([ -~]+\)$') 79 | return p.search(string_to_check) is not None 80 | else: 81 | return False 82 | 83 | @staticmethod 84 | def get_variable_name(var_string: str) -> str: 85 | """ 86 | Returns the name of the variable 87 | :param var_string: String that contains variable name 88 | :return: Name of variable 89 | """ 90 | variable_name = re.sub(r'^\$\(', '', var_string) 91 | variable_name = re.sub(r'\)$', '', variable_name) 92 | return variable_name 93 | 94 | @staticmethod 95 | def get_key_value_pair(line: str, separator: str = "=") -> Tuple[str, str]: 96 | """ 97 | Splits a line separated by given separator into key and value 98 | :param line: String containing the line to split 99 | :param separator: Separator string that separates key and value 100 | :return: Tuple consisting out of parsed key and value 101 | """ 102 | r = re.search(r'(^[ -~]+ ?{} )([ -~]+)'.format(separator), line) 103 | return (r.group(1)[:-2].strip(), r.group(2).strip()) 104 | -------------------------------------------------------------------------------- /dynmx/helpers/resource_helper.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | from __future__ import annotations 19 | from urllib.parse import urlparse 20 | import ipaddress 21 | 22 | 23 | class ResourceHelper: 24 | """ 25 | Helper class for resources 26 | """ 27 | 28 | @staticmethod 29 | def is_url(url: str) -> bool: 30 | try: 31 | result = urlparse(url) 32 | return all([result.scheme, result.netloc]) 33 | except: 34 | return False 35 | 36 | @staticmethod 37 | def is_ip_address(value: str) -> bool: 38 | try: 39 | ip = ipaddress.ip_address(value) 40 | return True 41 | except ValueError: 42 | return False 43 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | anytree==2.7.2 2 | lxml==4.9.2 3 | pyparsing==2.4.5 4 | PyYAML==5.2 5 | six==1.13.0 6 | stringcase==1.2.0 7 | --------------------------------------------------------------------------------