├── .gitignore ├── CHANGES ├── LICENSE ├── LibRadar ├── Data │ ├── IntermediateData │ │ ├── api.csv │ │ ├── api_just_invoke.csv │ │ ├── invokeFormat.txt │ │ ├── sort_tag_rules.py │ │ ├── strict_api.csv │ │ └── tag_rules.csv │ ├── RawData │ │ └── README.md │ └── logging.conf ├── __init__.py ├── _settings.py ├── api_dictionary.py ├── api_permission.py ├── brand_lib.py ├── data_cnt.py ├── data_run_radar.py ├── data_size.py ├── data_threshold.py ├── db_dumping.py ├── dex_extracting.py ├── dex_parser.py ├── dex_tree.py ├── job_dispatching.py ├── lib_tagging.py ├── lib_tagging_gui.py ├── libradar.py ├── lite_dataset.py ├── rputil.py └── rputil_test.py ├── MANIFEST.in ├── README.md ├── artwork └── LibRadar.png ├── docs ├── PyPI_Index.rst ├── QuickStart.md └── compare_with_libd.md ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── result │ ├── test_db_case_1.txt │ └── test_db_case_2.txt ├── test.py ├── test_db.py ├── test_logging.py ├── test_multiprocessing.py ├── test_queue.py └── test_tk.py └── tool ├── .redis.conf.swp ├── AXMLPrinter2.jar ├── jad ├── jar_decompiler.sh └── redis.conf /.gitignore: -------------------------------------------------------------------------------- 1 | LibRadar/Data/RawData 2 | log* 3 | LibRadar/Data/Decompiled 4 | LibRadar/Data/IntermediateData 5 | *.DS_Store 6 | a/* 7 | b/* 8 | 9 | # .idea 10 | .idea 11 | # Byte-compiled / optimized / DLL files 12 | __pycache__/ 13 | *.py[cod] 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | env/ 21 | build/ 22 | develop-eggs/ 23 | dist/ 24 | downloads/ 25 | eggs/ 26 | .eggs/ 27 | lib/ 28 | lib64/ 29 | parts/ 30 | sdist/ 31 | var/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *,cover 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | LibRadar Changelog 2 | =============== 3 | 4 | 5 | __version 2.2.1 (current version)__ 6 | 7 | 1. Updates some lib rules 8 | 2. Add Lite version 9 | 10 | __version 2.2.0__ 11 | 12 | 1. Change Md5 to SHA256. 13 | 2. Fix bugs 14 | 15 | __version 2.1.0__ 16 | 17 | 1. better hashing algorithm 18 | 2. full and convincing API list 19 | 3. incremental database 20 | 1. partial match support 21 | 1. using dex other than apktool to analyse apps 22 | 23 | __version 1.3.0__ 24 | 25 | 1. Ajax support. 26 | 27 | __version 1.2.5__ 28 | 29 | 1. Assert 3 parts. 30 | 2. Repair the problem "three ';' in 'tgst5.dat'". 31 | 3. Update repetition counting. 32 | 4. Merge the results of marked libs and unmarked libs. 33 | 5. Fix '/' bug. 34 | 6. Sort the result. 35 | 36 | __version 1.2.3__ 37 | 38 | 1. Add 'LData' Branch for Jingyue's research. 39 | 2. Add var RM_STATUS for smali code deleting control. 40 | 3. Modify path in function *all_over* so that there's no need to input the full path of this python script any more. 41 | 4. Replace chinese description with web address. 42 | 43 | __version 1.2.1__ 44 | 45 | Remove lib code in smali files and zip them into a new file. 46 | 47 | __version 1.2.0__ 48 | 49 | Modularity 50 | 51 | __version 1.1.9__ 52 | 53 | 1. Update Output Format. 54 | 2. Add library type. 55 | 3. Update library fingerprint data for better recognition. 56 | 57 | __version 1.1.7__ 58 | 59 | 1. Update library fingerprint data for better recognition. 60 | 2. Put plenty data into final output. 61 | - "dn": 311 - Repetitions 62 | - "lib": "pollfish" - Library 63 | - "sp": "com/pollfish/f/a" - Simplified Path 64 | - "bh": 32370 - B_Hash 65 | - "btc": 40 - B_Total_Call 66 | - "btn": 12 - B_Total_Number 67 | - "pn": "com/pollfish" - Package Name 68 | 3. Permission detection of Libraries. 69 | 70 | __version 1.1.5__ 71 | 72 | Add Permission Detection of packages. 73 | 74 | __version 1.1.3__ 75 | 76 | 1. Modified Tagged Library Data and Sorted it. 77 | 1. Remove print('*' * 60) and print('Task: '+self.tag+' Starts.') 78 | 1. Remove 'minutes' tag because it is useless. 79 | 1. Add specific time consuming tag. 80 | 1. time_decode = TimeRecord('Target App Decoding') 81 | - time_load = TimeRecord('Lib Data Loading') 82 | - time_extract = TimeRecord('Feature Extracting') 83 | - time_compare = TimeRecord('Library Searching') 84 | 1. New algorithm. 85 | 1. Sort library data. 86 | 1. Use binary search to find the library. 87 | 88 | __version 1.1.2__ 89 | 90 | Update and modify library data set. 91 | 92 | __version 1.1.1__ 93 | 94 | Replace 320,000 apps data with 1,000,000 apps data. 95 | 96 | __version 1.1.0__ 97 | 98 | A new version with optimized code, detailed comments and simplified data. 99 | 100 | __version 1.0.1__ 101 | 102 | Uploading bug fixed. 103 | 104 | __version 1.0.0__ 105 | 106 | First complete Version with complicated code which can be used with Node.js. 107 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | -------------------------------------------------------------------------------- /LibRadar/Data/IntermediateData/sort_tag_rules.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import os 19 | import shutil 20 | 21 | rules = [] 22 | 23 | copy_num = 1 24 | while True: 25 | if os.path.exists("tag_rules.csv.backup.%d" % copy_num): 26 | copy_num += 1 27 | continue 28 | shutil.copyfile("tag_rules.csv", "tag_rules.csv.backup.%d" % copy_num) 29 | break 30 | 31 | with open("tag_rules.csv") as rules_file: 32 | for line in rules_file: 33 | if line.startswith("Package Name,Library Name,Type,Official Website"): 34 | continue 35 | if len(line.split(',')) != 4: 36 | logging.error("Num of Commas Wrong!\n%s" % line) 37 | break 38 | rules.append(line) 39 | 40 | rules.sort() 41 | with open("tag_rules.csv", "w") as rules_file: 42 | rules_file.write("Package Name,Library Name,Type,Official Website\n") 43 | for line in rules: 44 | rules_file.write(line) 45 | -------------------------------------------------------------------------------- /LibRadar/Data/RawData/README.md: -------------------------------------------------------------------------------- 1 | # Raw Data 2 | 3 | This directory is used for store raw data from outside. 4 | Data here could be token as known conditions. 5 | 6 | # Content 7 | 8 | From android7.jar to android25.jar 9 | 10 | # Quick Start 11 | 12 | - Download Android SDK using *Android SDK Manager*. 13 | - Select SDK Platform for each version(API Level). 14 | - Open your path of SDK. (e.g. ~/Android/Sdk/platforms) 15 | - Copy android.jar into libradar/Data/RawData and rename it. 16 | 17 | ```bash 18 | cp ~/Android/Sdk/platform/android-21/android.jar $LibRadar_Folder$/Data/RawData/android21.jar 19 | ``` 20 | -------------------------------------------------------------------------------- /LibRadar/Data/logging.conf: -------------------------------------------------------------------------------- 1 | #logger.conf 2 | ######## 3 | [loggers] 4 | keys=root,radar,info 5 | 6 | [logger_root] 7 | level=DEBUG 8 | handlers=hand01,hand02 9 | 10 | [logger_radar] 11 | level=DEBUG 12 | handlers=hand01,hand02 13 | qualname=libradar 14 | propagate=0 15 | 16 | [logger_info] 17 | level=INFO 18 | handlers=hand02 19 | propagate=0 20 | qualname=libradar_info 21 | 22 | ######## 23 | [handlers] 24 | keys=hand01,hand02 25 | [handler_hand01] 26 | class=StreamHandler 27 | level=INFO 28 | formatter=form02 29 | args=(sys.stderr,) 30 | [handler_hand02] 31 | class=FileHandler 32 | level=DEBUG 33 | formatter=form01 34 | args=('log_libradar.txt', 'a') 35 | 36 | ######## 37 | [formatters] 38 | keys=form01,form02 39 | [formatter_form01] 40 | format=%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s 41 | datefmt=%a, %d %b %Y %H:%M:%S 42 | [formatter_form02] 43 | format=%(asctime)s: %(levelname)-8s %(message)s 44 | datefmt= -------------------------------------------------------------------------------- /LibRadar/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # LibRadar 18 | # ~~~~~ 19 | # 20 | # An automatically detecting tool for Android third-party libraries. 21 | # Ideas about this tool is written as a publisher, you could view the [PDF] 22 | # (http://dl.acm.org/citation.cfm?id=2889178) here. 23 | 24 | from libradar import LibRadar 25 | 26 | __all__ = ( 27 | "__title__", "__summary__", "__uri__", "__version__", "__author__", 28 | "__email__", "__license__", "__copyright__", 29 | ) 30 | 31 | __title__ = "libradar" 32 | __summary__ = "A tool for Android library detection." 33 | __uri__ = "https://github.com/pkumza/libradar" 34 | 35 | __version__ = "2.2.1" 36 | 37 | __author__ = "Zachary Marv (马子昂)" 38 | __email__ = "ma@zablog.me" 39 | 40 | __license__ = "Apache License, Version 2.0" 41 | __copyright__ = "Copyright 2017 " + __author__ 42 | -------------------------------------------------------------------------------- /LibRadar/_settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # LibRadar settings.py 18 | 19 | # Settings about python scripts in LibRadar folder. 20 | # All the scripts in LibRadar need import this file. 21 | 22 | 23 | 24 | import os 25 | import logging.config 26 | 27 | """ 28 | whether clean the workspace after work 29 | 0 : Clean nothing 30 | 1 : Clean res files 31 | 2 : Clean smali files 32 | 3 : Clean everything 33 | """ 34 | CLEAN_WORKSPACE = 0 35 | 36 | """ 37 | Database use db 0 as default 38 | """ 39 | 40 | DB_HOST = 'localhost' 41 | DB_PORT = 6379 42 | DB_ID = 2 43 | DB_ID_REP = 1 44 | # if you don't have Password, delete DB_PSWD 45 | DB_PSWD = '' 46 | 47 | DB_FEATURE_CNT = 'feature_cnt' 48 | DB_FEATURE_WEIGHT = 'feature_weight' 49 | DB_UN_OB_PN = 'un_ob_pn' 50 | DB_UN_OB_CNT = 'un_ob_cnt' 51 | 52 | 53 | """ 54 | running_processes 55 | 56 | Use multi-processing could fully use the cores of cpu. 57 | Once I set QUEUE_TIME_OUT 5. After about two hours, three processes returns. So it should be little longer. 58 | I set it 30 yesterday and in two hours' processing, every process runs well. 59 | """ 60 | # RUNNING_PROCESS_NUMBER = 8 61 | RUNNING_PROCESS_NUMBER = 1 62 | QUEUE_TIME_OUT = 30 63 | 64 | 65 | """ 66 | IGNORE ZERO API FILES 67 | 68 | If there's no API in a class file, just ignore it. 69 | If there's no API in a package, just ignore it. 70 | """ 71 | IGNORE_ZERO_API_FILES = True 72 | 73 | """ 74 | Config Files 75 | """ 76 | 77 | SCRIPT_PATH = os.path.split(os.path.realpath(__file__))[0] 78 | if not os.path.exists(SCRIPT_PATH + '/Data'): 79 | os.mkdir(SCRIPT_PATH + '/Data') 80 | FILE_LOGGING = SCRIPT_PATH + '/Data/logging.conf' 81 | FILE_RULE = SCRIPT_PATH + '/Data/IntermediateData/tag_rules.csv' 82 | LITE_DATASET_10 = SCRIPT_PATH + '/Data/IntermediateData/lite_dataset_10.csv' 83 | 84 | """ 85 | Logs 86 | """ 87 | 88 | logging.config.fileConfig(FILE_LOGGING) 89 | # create logger 90 | logger = logging.getLogger('radar') 91 | -------------------------------------------------------------------------------- /LibRadar/api_dictionary.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # API_Dict.py 18 | # 19 | # WARNING!!! Could be run under Mac OS only. 20 | # 21 | # You don't need to run this again!!! Just use Data/IntermediateData/invokeFormat.txt 22 | # 23 | # This is the first step for data collection. However, jad is out of data and it could only been easily used on Mac. 24 | # What's more, running this script need to download all the versions of Android SDK. 25 | # So, if you do not want to learn about the principle, just ignore this script. 26 | # 27 | # Firstly, get android.jar in Android SDK folder. 28 | # Rename them and place them into Data/RawData folder. 29 | # Glob module will automatically detect them and decompile them. 30 | # In this case, we could extract all the APIs. 31 | # Finally, some non-sense API should be removed. 32 | # 33 | # Non-sense API: 34 | # As an example, a package named Lcom/degoo/android/fbmassage is not a library, which belongs to an app named 35 | # "Princess Full Body Massage APK". This package contains no API other than 15 "Ljava/lang/Object;->()" 36 | # It is very likely that another packages share the same feature. 37 | # 38 | # In my database of 4600 apps, There're already 140 packages shares the same feature. That's definitely wrong! 39 | # (MD5: 5019771e5f8ce4bc333b504a3a6bc4a6) 40 | # 41 | # Implementation: 42 | # ApiDictionaryGenerator.ignore_list 43 | # 44 | # Warning: Working Directory should be LibRadar other than LibRadar/LibRadar 45 | 46 | # Take API Level 16,18,19,21,22,23,24,25 in account, We could found 34299 APIs without overloading 47 | # Ignore 1761 APIs 48 | # 2016/12/30 49 | 50 | # Take API Level 7,8,9,...,25 in account, we could found 34683 APIs without overloading. 51 | # (From Android 2.1 to newest Android 7.1.1) 52 | # Ignore 1775 APIs 53 | # 2017/01/10 54 | 55 | import os.path 56 | import commands 57 | # ## import redis 58 | import glob 59 | from _settings import * 60 | 61 | """ 62 | Information about Database. 63 | Redis is used here and the default setting points to localhost:6379 64 | If your Redis IP and Port is different, you can change the value here. 65 | In my redis.conf, I set 32 for the maximum number of databases. 66 | What's the databases for? 67 | 68 | 0 : Android.jar Classes 69 | 1 : Android.jar APIs with return_type and arguments 70 | 2 : Android.jar APIs with only full class name and method name 71 | """ 72 | 73 | 74 | class Singleton(object): 75 | """ 76 | Implementation for Singleton 77 | """ 78 | def __new__(cls): 79 | if not hasattr(cls, '_inst'): 80 | cls._inst = super(Singleton, cls).__new__(cls) 81 | return cls._inst 82 | 83 | 84 | class ApiDictionaryGenerator(Singleton): 85 | """ 86 | A class used for collecting API data 87 | """ 88 | 89 | def __init__(self): 90 | """ 91 | __init__ 92 | Create connection to redis database. 93 | open a file for api output. 94 | """ 95 | self.jar_list = [] 96 | # ## self.redis_class_name = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_CLASS_NAME) 97 | # ## logger.warning("Clean all the keys in databases") 98 | # ## self.redis_class_name.flushdb() 99 | # ## self.redis_android_api = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ANDROID_API) 100 | # ## self.redis_android_api.flushdb() 101 | # ## self.redis_android_api_simplified = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_API_INVOKE) 102 | # ## self.redis_android_api_simplified.flushdb() 103 | # ## self.api_set = set() 104 | # ## self.txt_output_api = open("./Data/IntermediateData/api.txt", 'w') 105 | self.api_simplified_set = set() 106 | self.txt_invoke_format = open(SCRIPT_PATH + "/Data/IntermediateData/invokeFormat.txt", 'w') 107 | self.ignore_list = [ 108 | "Ljava/lang", 109 | "Ljava/util/logging", 110 | "Landroid/util/Log", 111 | "Landroid/Manifest", 112 | "Landroid/R", 113 | "Landroid/test", 114 | "Ljunit", 115 | "Lorg/apache/commons/logging/Log" 116 | ] 117 | 118 | def __del__(self): 119 | """ 120 | __del__ 121 | StrictRedis doesn't implement close or quit methods. 122 | We don't need to close redis connection here. 123 | """ 124 | self.txt_invoke_format.close() 125 | 126 | @staticmethod 127 | def if_jad_exists(): 128 | """ 129 | Test if jad command exists. 130 | Run `jad` command as a test. If the status is 32512, it means that system cannot 131 | find the `jad` command. If the response is 256, `jad` can be run but `jad` 132 | cannot runs stably without arguments. If the status is neither 32512 nor 256, 133 | I don't know what happened. 134 | """ 135 | # run `jad` command as a test here to exam if jad is installed. 136 | status, out = commands.getstatusoutput('jad') 137 | if status == 32512: 138 | logger.critical("Jad is not runnable. please put `tool/jad` into your $PATH environment.") 139 | raise AssertionError 140 | if status == 256: 141 | pass 142 | else: 143 | logger.warning("Maybe there is something wrong with jad status.") 144 | 145 | """ 146 | Take jar file into account. It is ok to have only one version of android.jar 147 | android.jar could be found in $YOUR_ANDROID_SDK_ROOT/platforms/android-$VERSION 148 | 149 | For Example: 150 | /Library/Android_SDK/platforms/android-11/android.jar 151 | 152 | Use add_jar to add a file. 153 | Use add_jar several times or add_jars to add a list of files. 154 | """ 155 | def add_jar(self, jar_file): 156 | self.jar_list.append(jar_file) 157 | 158 | def add_jars(self, jar_files): 159 | self.jar_list.extend(jar_files) 160 | 161 | """ 162 | Use jad to decompile jar file into android.jar.dir 163 | 164 | Step: 165 | 1 - Unzip .jar file 166 | 2 - use jad to decompile .class file into .java file. 167 | """ 168 | def decompile_jar(self): 169 | self.if_jad_exists() 170 | for jar in self.jar_list: 171 | logger.info("Decompiling %s" % jar) 172 | cmd = "./tool/jar_decompiler.sh " + jar 173 | os.popen(cmd) 174 | 175 | """ 176 | Walk through the directories to find all the class names and methods (SDK API). 177 | 178 | 179 | """ 180 | def walk_dir(self): 181 | for jar in self.jar_list: 182 | dir_to_be_walked = jar + ".dir" 183 | logger.info("Walk %s" % dir_to_be_walked) 184 | for dirName, subdirList, fileList in os.walk(dir_to_be_walked): 185 | for filename in fileList: 186 | if len(filename) > 4 and filename[-5:] == ".java": 187 | full_path_name = '/'.join(dirName.split('/')[4:]) + '/' + filename 188 | class_name = full_path_name[:-5].replace('/', '.') 189 | ''' 190 | Contents in inner class file like 'R$style.java' are already included in 191 | the parent class file. So we need to ignore the java file with symbol 192 | '$' in it. 193 | ''' 194 | if '$' in full_path_name: 195 | continue 196 | self.read_java(full_path_name, class_name, jar) 197 | # ## self.redis_class_name.incr(class_name) 198 | # clean the directory 199 | if CLEAN_WORKSPACE: 200 | logger.info("Cleaning the directory which is already walked.") 201 | os.system('rm -rf %s' % dir_to_be_walked) 202 | # ## logger.info("API Count is %d counting overloading." % len(self.api_set)) 203 | logger.info("API Count is %d without overloading." % len(self.api_simplified_set)) 204 | logger.info("Write the APIs into txt file as a backup.") 205 | # ## for api in self.api_set: 206 | # ## self.txt_output_api.write(api + '\n') 207 | api_simplified_list = list() 208 | # Ignore those APIs that appears in ignore_list. Such as Landroid/util/Log 209 | ignore_cnt = 0 210 | for api_s in self.api_simplified_set: 211 | flag = True 212 | for ign_api in self.ignore_list: 213 | if ign_api in api_s: 214 | flag = False 215 | break 216 | if flag: 217 | api_simplified_list.append(api_s) 218 | else: 219 | ignore_cnt += 1 220 | logger.info("Ignore %d APIs" % ignore_cnt) 221 | for api_s in sorted(api_simplified_list): 222 | self.txt_invoke_format.write(api_s + '\n') 223 | 224 | def read_java(self, full_path_name, class_name, jar): 225 | """ 226 | Read APIs from java file. 227 | """ 228 | open_java_file = open(jar + '.dir/' + full_path_name, 'r') 229 | brackets_count = 0 230 | current_inner_class = "" 231 | for line in open_java_file: 232 | # if the line is a comment. 233 | if len(line.strip()) > 2 and line.strip()[:2] == "//": 234 | continue 235 | if '{' in line: 236 | brackets_count += 1 237 | if '}' in line: 238 | brackets_count -= 1 239 | # outer class 240 | if ('public' in line or 'protected' in line) and 'class' in line and brackets_count == 0: 241 | continue 242 | # inner class 243 | if ('public' in line or 'protected' in line) and 'class' in line and brackets_count == 1: 244 | current_inner_class = line.split('class')[1].strip() 245 | # if there's 'extends' here in this string 246 | if " " in current_inner_class: 247 | current_inner_class = current_inner_class.split(' ')[0] 248 | continue 249 | if ('public' in line or 'protected' in line) and 'interface' in line and brackets_count == 1: 250 | current_inner_class = line.split('interface')[1].strip() 251 | if " " in current_inner_class: 252 | current_inner_class = current_inner_class.split(' ')[0] 253 | continue 254 | # method (API) 255 | if ('public' in line or 'protected' in line) and '(' in line and ')' in line: 256 | left_part = line.split('(')[0] 257 | method_name = left_part.split(' ')[-1] 258 | return_type = left_part.split(' ')[-2] 259 | if "extends" in method_name: 260 | pass 261 | if "extends" in return_type: 262 | pass 263 | ''' 264 | if the value is public, that means the function is a constructive method. 265 | I use '#' here to tag that the method does not have a return type. 266 | 267 | If the value is not '#', the value is put into database 0 for a count. 268 | ''' 269 | # ## if return_type == 'public' or return_type == 'protected': 270 | # ## return_type = '#' 271 | # ## else: 272 | # ## self.redis_class_name.incr(return_type) 273 | right_part = line.split('(')[1] 274 | # parameters of the method 275 | paras = [] 276 | paras_number = len(right_part.split(' ')) 277 | for i in range(paras_number): 278 | if i % 2 == 0: 279 | para_type = right_part.split(' ')[i] 280 | # ## self.redis_class_name.incr(para_type) 281 | paras.append(para_type) 282 | # reconstruct the method 283 | full_class_name = class_name 284 | if brackets_count == 2: 285 | full_class_name += '$' 286 | full_class_name += current_inner_class 287 | parameters_string = "" 288 | for i in range(paras_number / 2): 289 | if i != 0: 290 | parameters_string += ',' 291 | parameters_string += paras[i] 292 | # ## method_declare = "%s %s->%s(%s)" % (return_type, full_class_name, method_name, parameters_string) 293 | point_to_slash = full_class_name.replace('.', '/') 294 | if method_name == class_name.split('.')[-1]: 295 | method_name = "" 296 | method_invoke = "L%s;->%s" % (point_to_slash, method_name) 297 | 298 | # ## self.redis_android_api_simplified.incr(method_invoke) 299 | # ## self.api_set.add(method_declare) 300 | # ## self.redis_android_api.incr(method_declare) 301 | self.api_simplified_set.add(method_invoke) 302 | 303 | open_java_file.close() 304 | 305 | 306 | class ApiDictionaryGeneratorWrapper: 307 | """ 308 | ADG Wrapper 309 | If you are not interested in the details, just use this class. 310 | Input a list of file names which you have already placed into Data/RawData folder. 311 | There're no more thing you need to do here. 312 | information about classes and APIs are automatically stalled into txt file and database. 313 | """ 314 | def __init__(self, jar_list): 315 | # create an instance. 316 | logger.info("Creating an instance of ApiDictionaryGenerator") 317 | adg = ApiDictionaryGenerator() 318 | # add the jar into list. 319 | logger.info("Adding jar path into jar list") 320 | adg.add_jars(jar_list) 321 | # decompiling the jar file. decompiling is only needed once. 322 | logger.info("Decompiling jar") 323 | adg.decompile_jar() 324 | # walk through the directory to find APIs.clean 325 | adg.walk_dir() 326 | 327 | 328 | if __name__ == "__main__": 329 | """ 330 | In my test case, I use android.jar of version21 and android.jar of version24 here. 331 | 332 | How to use this line of code? 333 | 1. firstly, change android.jar's name and put them into ./Data/RawData 334 | 2. write the names here as a list 335 | 3. run this python file. 336 | What happened when this file is running? 337 | 1. files are extracted into ./Data/RawData/$directory_name$.dir 338 | 2. the program will automatically read the class file with jad and convert them into java 339 | 3. this script read the java files and search for the classes and APIs 340 | 4. classes and native types are stalled into database 0 in redis 341 | 5. APIs are written into api.txt 342 | 6. APIs are also placed into database 1 in redis 343 | 344 | PS: 345 | There are commonly 30 thousands of APIs in one version of android.jar 346 | """ 347 | jar_file_list = glob.glob(SCRIPT_PATH + "/Data/RawData/*.jar") 348 | api_dict_generator_wrapper = ApiDictionaryGeneratorWrapper(jar_file_list) 349 | -------------------------------------------------------------------------------- /LibRadar/api_permission.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # API Permission 18 | 19 | # Get permission from PScout 20 | 21 | # Input LibRadar/Data/RawData/mapping_x.x.x.csv 22 | # Input invokeFormat.txt 23 | 24 | # Output api_permission.csv 25 | 26 | # Refer: https://developer.android.com/reference/android/Manifest.permission.html 27 | 28 | import glob 29 | 30 | STRICT_API = True 31 | 32 | def generate_permission(): 33 | # invokes 34 | invokes = dict() 35 | with open("Data/IntermediateData/invokeFormat.txt") as invoke_format: 36 | for line in invoke_format: 37 | caller_method = line.split(";->")[1] 38 | # Some wrong invoke do not have any characters. 39 | flag = False 40 | for character in caller_method: 41 | if character >= 'a' and character <= 'z' or character >= 'A' and character <= 'Z': 42 | flag = True 43 | break 44 | if flag: 45 | invokes[line.strip()] = set() 46 | # There's no change when I put old permission into account!! 2017-02-20 47 | pscout_old_files = glob.glob("Data/RawData/*_allmappings.txt") 48 | for old_file in pscout_old_files: 49 | current_p = "" 50 | with open(old_file, 'r') as pscout: 51 | for line in pscout: 52 | if line[0] <= '9' and line[0] >= '0': 53 | continue 54 | if line[0] == 'P': 55 | permission = line[11:].strip() 56 | if line[0] == '<': 57 | caller_class = line[1:line.index(':')] 58 | caller_method = line[line.index(" ") : line.index("(")] 59 | caller_method = caller_method[caller_method.rindex(" ") + 1:] 60 | invoke = "L" + caller_class + ";->" + caller_method 61 | if invoke in invokes: 62 | invokes[invoke].add(permission) 63 | else: 64 | if not STRICT_API: 65 | invokes[invoke] = set() 66 | invokes[invoke].add(permission) 67 | # Now we got 4.1.1 4.2.2 4.4.4 5.0.2 5.1.1 as sample 68 | pscout_files = glob.glob("Data/RawData/mapping_*.csv") 69 | for pscout_file in pscout_files: 70 | with open(pscout_file, 'r') as pscout: 71 | for line in pscout: 72 | line_split = line.split(',') 73 | caller_class = line_split[0] 74 | caller_method = line_split[1] 75 | caller_method_desc = line_split[2] 76 | permission = line_split[3] 77 | version = line_split[4] 78 | if caller_class == "CallerClass": 79 | continue 80 | if permission == "Unknown" or permission == "Parent": 81 | continue 82 | invoke = "L" + caller_class + ";->" + caller_method 83 | if invoke in invokes: 84 | invokes[invoke].add(permission) 85 | else: 86 | if not STRICT_API: 87 | invokes[invoke] = set() 88 | invokes[invoke].add(permission) 89 | invoke_list = list() 90 | for invoke in invokes: 91 | invoke_list.append([invoke, sorted(list(invokes[invoke]))]) 92 | invoke_list.sort() 93 | if STRICT_API: 94 | api_csv_filename = "Data/IntermediateData/strict_api.csv" 95 | else: 96 | api_csv_filename = "Data/IntermediateData/api.csv" 97 | with open(api_csv_filename, "w") as api_file: 98 | for invoke in invoke_list: 99 | api_file.write(invoke[0] + ",") 100 | for permi in invoke[1]: 101 | api_file.write(permi) 102 | api_file.write(":") 103 | api_file.write("\n") 104 | 105 | 106 | 107 | if __name__ == '__main__': 108 | generate_permission() -------------------------------------------------------------------------------- /LibRadar/brand_lib.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # LibRadar brand_lib.py 19 | # 20 | # When you find an APK file has a lib, but you could not find it in my database, you could just use this script to 21 | # insert it into database with force. 22 | 23 | import libradar 24 | import sys 25 | 26 | 27 | class BrandLib(libradar.LibRadar): 28 | """ 29 | BrandLib 30 | """ 31 | def __init__(self, apk_path, lib_package, standard_package): 32 | """ 33 | Init LibRadar instance with apk_path as a basestring. 34 | Create a Tree for every LibRadar instance. The tree describe the architecture of the apk. Every package is a 35 | node. 36 | :param apk_path: basestring 37 | """ 38 | libradar.LibRadar.__init__(self, apk_path) 39 | self.lib_package = lib_package 40 | self.standard_package = standard_package 41 | 42 | def brand(self): 43 | # use analyse in libradar. 44 | # Do not place the same logic in two place! 45 | self.analyse() 46 | # brand lib 47 | return self.tree.brand(self.lib_package, self.standard_package) 48 | 49 | 50 | if __name__ == '__main__': 51 | if len(sys.argv) < 3 or len(sys.argv) > 4: 52 | print("== Brand_lib takes 2 or 3 parameters! ==") 53 | print(" Usage 1: not obfuscated package name.") 54 | print(" python brand_lib.py sample.apk Lcom/google/zxing") 55 | print(" Usage 2: obfuscated package name.") 56 | print(" python brand_lib.py sample.apk Lcom/a/a Lcom/tencent/tauth") 57 | exit(1) 58 | apk_path = sys.argv[1] 59 | lib_package = sys.argv[2] 60 | standard_package = lib_package 61 | if (len(sys.argv) == 4): 62 | standard_package = sys.argv[3] 63 | brand_lib = BrandLib(apk_path, lib_package, standard_package) 64 | res = brand_lib.brand() 65 | print(res) 66 | -------------------------------------------------------------------------------- /LibRadar/data_cnt.py: -------------------------------------------------------------------------------- 1 | from _settings import * 2 | import redis 3 | import math 4 | 5 | db = redis.StrictRedis(host=DB_HOST,port=DB_PORT, db=2, password=DB_PSWD) 6 | 7 | counter = dict() 8 | for i in range(100): 9 | counter[i] = 0 10 | 11 | po_list = [] 12 | 13 | cursor = 0 14 | for i in range(7255): 15 | if i % 10 == 0: 16 | print ("Progress:%d" % i) 17 | ct = 1000 18 | if i == 7254: 19 | ct = 946 20 | res = db.hscan(name="feature_cnt", cursor=cursor, count=ct) 21 | cursor = res[0] 22 | for k in res[1]: 23 | counter[int(math.log(float(res[1][k]),2))] += 1 24 | if int(res[1][k]) > 2000 : 25 | anwser = db.hget(name="feature_weight", key=k) 26 | po_list.append((int(anwser),db.hget(name="un_ob_pn", key=k))) 27 | 28 | print counter 29 | po_list.sort(key=lambda x:(-x[0])) 30 | for item in po_list: 31 | print item -------------------------------------------------------------------------------- /LibRadar/data_run_radar.py: -------------------------------------------------------------------------------- 1 | import libradar 2 | import json 3 | import glob 4 | 5 | apks = glob.glob("/Volumes/banana/apks/*") 6 | 7 | i = -1 8 | total = 10 9 | 10 | while i < total: 11 | try: 12 | i += 1 13 | print "Progress %d" % i 14 | apk_path = apks[i] 15 | lrd = libradar.LibRadar(apk_path) 16 | res = lrd.compare() 17 | print(json.dumps(res, indent=4, sort_keys=True)) 18 | #for item in res: 19 | # if item["Library"] != "Unknown" and "Standard Package" in item and "Package" in item and item["Standard Package"] != item["Package"]: 20 | # print "NOT_MATCH_PACKAGE_NAME" 21 | # print item["Standard Package"] 22 | # print item["Package"] 23 | except Exception, e: 24 | total += 1 25 | print Exception,":", e -------------------------------------------------------------------------------- /LibRadar/data_size.py: -------------------------------------------------------------------------------- 1 | from _settings import * 2 | import redis 3 | import math 4 | 5 | db = redis.StrictRedis(host=DB_HOST,port=DB_PORT, db=2, password=DB_PSWD) 6 | 7 | counter = dict() 8 | for i in range(100): 9 | counter[i] = 0 10 | 11 | 12 | 13 | cursor = 0 14 | for i in range(7255): 15 | if i % 10 == 0: 16 | print ("Progress:%d" % i) 17 | res = db.hscan(name="feature_weight", cursor=cursor, count=1000) 18 | cursor = res[0] 19 | for k in res[1]: 20 | counter[int(math.log(float(res[1][k]),2))] += 1 21 | 22 | print counter -------------------------------------------------------------------------------- /LibRadar/data_threshold.py: -------------------------------------------------------------------------------- 1 | from _settings import * 2 | import redis 3 | import math 4 | import csv 5 | 6 | db = redis.StrictRedis(db=2) #, password=DB_PSWD) 7 | 8 | result = list() 9 | 10 | THRESHOLD = 50 11 | 12 | dict_tag_rules = dict() 13 | new_prefix_list = list() 14 | if os.path.exists(FILE_RULE): 15 | file_rules = open(FILE_RULE, 'r') 16 | csv_rules_reader = csv.reader(file_rules, delimiter=',', quotechar='|') 17 | for row in csv_rules_reader: 18 | dict_tag_rules[row[0]] = row[1] 19 | file_rules.close() 20 | 21 | 22 | cursor = 0 23 | for i in range(7255):#255): 24 | if i % 10 == 0: 25 | print ("Progress:%d" % i) 26 | res = db.hscan(name="feature_cnt", cursor=cursor, count=1000) 27 | cursor = res[0] 28 | for i in res[1]: 29 | if int(res[1][i]) > THRESHOLD: 30 | un_ob_pn = db.hget(name="un_ob_pn", key=i) 31 | flag = True 32 | for lib in dict_tag_rules: 33 | if lib == un_ob_pn: 34 | if dict_tag_rules[lib] == "no": 35 | flag = False 36 | break 37 | else: 38 | result.append(un_ob_pn) 39 | break 40 | elif len(lib) > len(un_ob_pn) : 41 | continue 42 | elif lib[:len(un_ob_pn)] == un_ob_pn and dict_tag_rules[lib] != "no": 43 | flag = False 44 | break 45 | if flag == False: 46 | continue 47 | if 1 < len(un_ob_pn.split('/')) <= 3: 48 | result.append(un_ob_pn) 49 | 50 | 51 | for r in result: 52 | print r 53 | print len(result) 54 | -------------------------------------------------------------------------------- /LibRadar/db_dumping.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Dumping features in database into file for LibRadar Lite (Instant Detection.) 18 | # 19 | # created at 2017/01/09 20 | 21 | import binascii 22 | import redis 23 | from _settings import * 24 | 25 | 26 | # deprecated 2017-04-09 27 | def dump_database(): 28 | pass 29 | """ 30 | db_feature_count = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_FEATURE_COUNT) 31 | db_feature_weight = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_FEATURE_WEIGHT) 32 | db_un_ob_pn = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_UN_OB_PN) 33 | dump_file = open(SCRIPT_PATH + "/Data/IntermediateData/db_dump.csv", "w") 34 | # Run once, so do not care the complexity. 35 | for key in db_feature_count.keys(): 36 | count = db_feature_count.get(key) 37 | if int(count) <= 4: 38 | continue 39 | weight = db_feature_weight.get(key) 40 | package_name = db_un_ob_pn.get(key) 41 | dump_file.write("%s;%s;%s;%s\n" % (binascii.hexlify(key), count, weight, package_name)) 42 | """ 43 | 44 | if __name__ == "__main__": 45 | dump_database() 46 | -------------------------------------------------------------------------------- /LibRadar/dex_extracting.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # DEX Extractor 19 | # 20 | # This script is used to extract features and other information from DEX files. 21 | # :copyright: (c) 2016 by Zachary Ma 22 | # : Project: LibRadar 23 | # 24 | # Target: 25 | # Get a dex file 26 | # Generate features from the dex file. 27 | # Get the features of classes and packages. 28 | # 29 | # Implementation: 30 | # Firstly, get class defines from dex file. 31 | # For each class, we could generate it's APIs and so as the sha256 feature. 32 | # As the class definition contains the path, so we could construct a tree for the classes. 33 | # 34 | # e.g. 35 | # 36 | # root 37 | # / \ 38 | # android com 39 | # / / \ 40 | # support google facebook 41 | # / / \ \ 42 | # v4 gson ads c.class 43 | # | | \ 44 | # media e.class purchase 45 | # / \ | 46 | # a.class b.class d.class 47 | # 48 | # For every package, we generate features based on it's child. 49 | # for example, the feature of /android/support/v4/media is based on a.class and b.class. 50 | # If we need to generate features of all packages, we could construct a tree for them. 51 | # To minimize the complexity, and avoid constructing the tree, I sorted the class defines first. 52 | # 53 | # android/support/v4/media/a 54 | # android/support/v4/media/b 55 | # com/facebook/c 56 | # com/google/ads/purchase/d 57 | # com/google/gson/e 58 | # 59 | # It's like an in-order traversal for this tree: 60 | # 61 | # Create a stack for this. (Type: PackageNode) 62 | # Scan android/support/v4/media/a and put 'android', 'support', 'v4', 'media' into the stack. 63 | # Get the feature of a.class 64 | # Update media's feature 65 | # Get the feature of b.class 66 | # Update media's feature 67 | # Get the feature of c.class 68 | # Pop media and put the feature into db 69 | # Update and pop v4 and put the feature into db 70 | # Update and pop support and put the feature into db 71 | # Update and pop android and put the feature into db 72 | # Push com 73 | # Push facebook 74 | # Update facebook with c.class 75 | # Pop facebook 76 | # Push google 77 | # Push ads 78 | # Push purchase 79 | # Update Purchase's feature with d 80 | # Pop Purchase 81 | # Pop ads 82 | # Push gson 83 | # Update gson with e 84 | # Pop gson 85 | # Update and pop google 86 | # Update and pop com 87 | 88 | 89 | import os.path 90 | import hashlib 91 | import redis 92 | import dex_parser 93 | import time 94 | from _settings import * 95 | import sys 96 | 97 | 98 | class PackageNode: 99 | """ 100 | PackageNode 101 | Every Node of PackageNodeList is an instance of PackageNode. 102 | 103 | contains: 104 | sha256_list: children's sha256 105 | the sha256 feature of current node is based on sha256 from children. 106 | sort children's sha256 and then generate its own sha256 feature. 107 | 108 | path: the path is current packages' folder's name. 109 | e.g. Lcom/google/ads 's path is "ads" 110 | 111 | full_path: It is easy to imaging that full path of Lcom/google/ads is Lcom/google/ads 112 | 113 | weight: How many APIs are used in this package. 114 | """ 115 | def __init__(self, path, full_path): 116 | """ 117 | Init PackageNode with path 118 | Weight and sha256 list are initiated as empty. 119 | :param path: basestring 120 | :param full_path: basestring 121 | """ 122 | self.sha256_list = list() 123 | self.path = path 124 | self.full_path = full_path 125 | self.weight = 0 126 | 127 | def generate_sha256(self): 128 | """ 129 | Generate sha256 of current node based on children's sha256. 130 | :return: current Node's raw_sha256 and weight. 131 | """ 132 | curr_sha256 = hashlib.sha256() 133 | self.sha256_list.sort() 134 | for sha256_item in self.sha256_list: 135 | curr_sha256.update(sha256_item) 136 | # if not IGNORE_ZERO_API_FILES or len(self.sha256_list) != 0: 137 | # logger.debug("sha256: %s Weight: %-6d PackageName: %s" % 138 | # (curr_sha256.hexdigest(), self.weight, '/'.join(self.full_path))) 139 | return curr_sha256.hexdigest(), self.weight 140 | 141 | 142 | class PackageNodeList: 143 | """ 144 | A list (could be token as a stack) of PackageNodes. 145 | Used to implement the algorithm in introduction. 146 | 147 | One instance for one DexExtractor 148 | """ 149 | def __init__(self): 150 | self.pn_list = list() 151 | self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD) 152 | 153 | def flush_db(self): 154 | """ 155 | Flush databases 156 | :return: None 157 | """ 158 | certain_flush = raw_input("You really want to flush database?(Y/N)") 159 | if certain_flush != "Y": 160 | logger.info("Do not flush.") 161 | return 162 | logger.warning("Flush Database") 163 | self.db.flushdb() 164 | 165 | def catch_a_class_def(self, package_name, class_sha256, class_weight): 166 | """ 167 | catch a class definition 168 | 169 | Class definitions are sorted. 170 | Every time this function catch a new class definition. 171 | It got the class's package name, sha256 and class weight(API count) 172 | 173 | :param package_name: basestring 174 | :param class_sha256: basestring 175 | :param class_weight: int 176 | :return: None 177 | """ 178 | package_path_parts_list = package_name.split('/') 179 | common_depth = 0 180 | """ 181 | len(self.pn_list) The Length of the Stack 182 | len(package_path_parts_list) The Length of current Class's package 183 | common_depth The Common parts' Length 184 | 185 | Operation 1: 186 | pop accomplished packages in pn_list 187 | 188 | Operation 2: 189 | append new parts from package_p... into pn_list 190 | 191 | e.g. 192 | self.pn_list Lair/br/com/bitlabs/SWFPlayer/Player len: 6 193 | package_p... Lair/br/com/bitlabs/Software len: 5 194 | -------- 195 | common_depth: 4 196 | 197 | Operation 1: 198 | Generate feature of Lair/br/com/bitlabs/SWFPlayer/Player 199 | Pop Player 200 | Update feature of Lair/br/com/bitlabs/SWFPlayer 201 | Generate feature of Lair/br/com/bitlabs/SWFPlayer 202 | Pop SWFPlayer 203 | Update feature of Lair/br/com/bitlabs 204 | 205 | Operation 2: 206 | Create new Node Software 207 | """ 208 | # Get Common Depth ---- Test how many stages are the same. 209 | while common_depth < len(self.pn_list) and common_depth < len(package_path_parts_list): 210 | if package_path_parts_list[common_depth] != self.pn_list[common_depth].path: 211 | break 212 | common_depth += 1 213 | # Operation 1 214 | pn_list_size = len(self.pn_list) 215 | for d in range(pn_list_size - 1, common_depth - 1, -1): 216 | stage_to_be_pop = self.pn_list[-1] 217 | child_sha256, curr_weight = stage_to_be_pop.generate_sha256() 218 | if len(self.pn_list) > 1: 219 | stage_to_be_update = self.pn_list[-2] 220 | stage_to_be_update.sha256_list.append(child_sha256) 221 | stage_to_be_update.weight += curr_weight 222 | package_exist = self.db.hget(DB_FEATURE_CNT, child_sha256) 223 | """ 224 | If there's no instance in database. 225 | incr count 226 | set weight 227 | set un ob pn 228 | un ob pn count incr 229 | Else 230 | incr count 231 | If un ob pn == current pn 232 | un ob pn count incr 233 | Else 234 | un ob pn count decr 235 | if un ob pn count <= 0 236 | un ob pn = current pn 237 | """ 238 | while True: 239 | pipe = self.db.pipeline(transaction=False) 240 | if package_exist is None: 241 | pipe.hincrby(name=DB_FEATURE_CNT, key=child_sha256, amount=1) 242 | pipe.hset(name=DB_FEATURE_WEIGHT, key=child_sha256, value=curr_weight) 243 | pipe.hset(name=DB_UN_OB_PN, key=child_sha256, value='/'.join(stage_to_be_pop.full_path)) 244 | pipe.hset(name=DB_UN_OB_CNT, key=child_sha256, value=1) 245 | else: 246 | pipe.hincrby(name=DB_FEATURE_CNT, key=child_sha256, amount=1) 247 | db_un_ob_pn = self.db.hget(name=DB_UN_OB_PN, key=child_sha256) 248 | if db_un_ob_pn is None: 249 | logger.error("db_un_ob_pn should not be None here.") 250 | if '/'.join(stage_to_be_pop.full_path) == db_un_ob_pn: 251 | pipe.hincrby(name=DB_UN_OB_CNT, key=child_sha256, amount=1) 252 | else: 253 | pipe.hincrby(name=DB_UN_OB_CNT, key=child_sha256, amount=-1) 254 | db_un_ob_pn_count = self.db.hget(name=DB_UN_OB_CNT, key=child_sha256) 255 | if db_un_ob_pn_count is None: 256 | logger.error("db_un_ob_pn_count should not be None here.") 257 | if int(db_un_ob_pn_count) <= 0: 258 | pipe.hset(name=DB_UN_OB_PN, key=child_sha256, value='/'.join(stage_to_be_pop.full_path)) 259 | # forget to reset the count, which caused some count appears to be negative number. 260 | pipe.hset(name=DB_UN_OB_CNT, key=child_sha256, value=0) 261 | pipe.execute() 262 | break 263 | self.pn_list.pop() 264 | # Operation 2 265 | for d in range(common_depth, len(package_path_parts_list)): 266 | self.pn_list.append(PackageNode(package_path_parts_list[d], package_path_parts_list[:d + 1])) 267 | 268 | # add sha256 269 | if len(self.pn_list) != 0: 270 | self.pn_list[-1].sha256_list.append(class_sha256) 271 | self.pn_list[-1].weight += class_weight 272 | 273 | 274 | class DexExtractor: 275 | """ 276 | DEX Extractor 277 | 278 | """ 279 | def __init__(self, dex_name): 280 | """ 281 | Init the Feature Extractor 282 | :param dex_name: basestring 283 | """ 284 | self.dex_name = dex_name 285 | # self.dex is an instance of dex_parser.DexFile() 286 | # clear it here 287 | self.dex = None 288 | # use as a stack 289 | self.package_node_list = list() 290 | """ 291 | Use redis database to exam whether a call is an Android API consumes 27% running time. 292 | I think it should be replaced by a hash table as the API list could not be modified during the progress. 293 | """ 294 | invoke_file = open(SCRIPT_PATH + "/Data/IntermediateData/invokeFormat.txt", 'r') 295 | self.invokes = set() 296 | for line in invoke_file: 297 | self.invokes.add(line[:-1]) 298 | 299 | def get_api_list(self, dex_method, api_list): 300 | if dex_method.dexCode is None: 301 | return 302 | offset = 0 303 | insns_size = dex_method.dexCode.insnsSize * 4 304 | while offset < insns_size: 305 | op_code = int(dex_method.dexCode.insns[offset:offset + 2], 16) 306 | decoded_instruction = dex_parser.dexDecodeInstruction(self.dex, dex_method.dexCode, offset) 307 | smali_code = decoded_instruction.smaliCode 308 | if smali_code is None: 309 | logger.warning("smali code is None.") 310 | if decoded_instruction == 0: 311 | break 312 | else: 313 | offset += decoded_instruction.length 314 | continue 315 | # Next Instruction. 316 | offset += decoded_instruction.length 317 | if smali_code == 'nop': 318 | break 319 | # 4 invokes from 0x6e to 0x72 320 | if 0x6e <= op_code <= 0x72: 321 | if decoded_instruction.getApi in self.invokes: 322 | api_list.append(decoded_instruction.getApi) 323 | return 324 | 325 | def extract_class(self, dex_class_def_obj): 326 | class_sha256 = hashlib.sha256() 327 | # API List 328 | # a list for basestring 329 | api_list = list() 330 | # direct methods 331 | last_method_index = 0 332 | for k in range(len(dex_class_def_obj.directMethods)): 333 | current_method_index = last_method_index + dex_class_def_obj.directMethods[k].methodIdx 334 | last_method_index = current_method_index 335 | self.get_api_list(dex_class_def_obj.directMethods[k], api_list=api_list) 336 | # virtual methods 337 | last_method_index = 0 338 | for k in range(len(dex_class_def_obj.virtualMethods)): 339 | current_method_index = last_method_index + dex_class_def_obj.virtualMethods[k].methodIdx 340 | last_method_index = current_method_index 341 | self.get_api_list(dex_class_def_obj.virtualMethods[k], api_list=api_list) 342 | # Use sort to pass the tree construction stage. 343 | # In this case, we could only use a stack to create the package features. 344 | api_list.sort() 345 | for api in api_list: 346 | class_sha256.update(api) 347 | if not IGNORE_ZERO_API_FILES or len(api_list) != 0: 348 | pass 349 | return len(api_list), class_sha256.hexdigest(), class_sha256.hexdigest() 350 | 351 | def extract_dex(self): 352 | # Log Start 353 | logger.debug("Extracting %s" % self.dex_name) 354 | # Validate existing 355 | if not os.path.isfile(self.dex_name): 356 | logger.error("%s not file" % self.dex_name) 357 | return -1 358 | # Create a Dex object 359 | self.dex = dex_parser.DexFile(self.dex_name) 360 | pnl = PackageNodeList() 361 | # Generate sha256 from Dex 362 | 363 | class_info_list = list() 364 | for dex_class_def_obj in self.dex.dexClassDefList: 365 | weight, raw_sha256, hex_sha256 = self.extract_class(dex_class_def_obj=dex_class_def_obj) 366 | class_name = self.dex.getDexTypeId(dex_class_def_obj.classIdx) 367 | """ 368 | I got many \x01 here before the class name. 369 | such as '\x01Lcom/vungle/publisher/inject' 370 | don't know exactly but could use code below to deal with it. 371 | """ 372 | if class_name[0] is not 'L': 373 | l_index = class_name.find('L') 374 | if l_index == '-1': 375 | continue 376 | class_name = class_name[l_index:] 377 | if IGNORE_ZERO_API_FILES and weight == 0: 378 | continue 379 | class_info_list.append((class_name, weight, raw_sha256)) 380 | """ 381 | Sort the info list with the package name. 382 | """ 383 | class_info_list.sort(cmp=lambda x, y: cmp(x[0], y[0])) 384 | for class_info in class_info_list: 385 | # logger.debug("class_name %s" % class_name) 386 | class_name = class_info[0] 387 | raw_sha256 = class_info[2] 388 | weight = class_info[1] 389 | last_slash = class_name.rfind('/') 390 | # If a class belongs to root, just ignore it because it hardly be a library. 391 | if last_slash == -1: 392 | continue 393 | # get the package name 394 | # for class name Lcom/company/air/R; It's package name is Lcom/company/air 395 | package_name = class_name[:last_slash] 396 | pnl.catch_a_class_def(package_name, raw_sha256, weight) 397 | # Let PackageNodeList pop all the nodes. 398 | pnl.catch_a_class_def("", "", 0) 399 | return 0 400 | 401 | 402 | if __name__ == "__main__": 403 | # A test for dex extractor here. 404 | assert len(sys.argv) == 2 405 | logger.critical(" ------------------------- START ------------------------- ") 406 | de = DexExtractor(sys.argv[1]) 407 | if de.extract_dex() < 0: 408 | logger.error("Wrong!") 409 | logger.critical(" -------------------------- END -------------------------- ") 410 | # os.system("eject cdrom") 411 | -------------------------------------------------------------------------------- /LibRadar/dex_tree.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # DEX Tree 19 | # 20 | # This script is used to implement the tree node and tree structure. 21 | 22 | 23 | from _settings import * 24 | from collections import Counter 25 | import hashlib 26 | import csv 27 | import redis 28 | import zlib 29 | import rputil 30 | 31 | 32 | # tag_rules 33 | labeled_libs = list() 34 | no_lib = list() 35 | 36 | with open(FILE_RULE, 'r') as file_rules: 37 | csv_rules_reader = csv.reader(file_rules, delimiter=',', quotechar='|') 38 | for row in csv_rules_reader: 39 | if row[1] == "no": 40 | no_lib.append(row) 41 | else: 42 | labeled_libs.append(row) 43 | 44 | 45 | class TreeNode(object): 46 | """ 47 | Tree Node Structure 48 | { 49 | sha256 : 02b018f5b94c5fbc773ab425a15b8bbb // In fact sha256 is the non-hex one 50 | weight : 1023 // How many APIs in this Node 51 | pn : Lcom/facebook/internal // Current package name 52 | parent : // Parent node 53 | children: dict("pn": ) // Children nodes 54 | match : list( tuple(package_name, match_weight) ) // match lib list 55 | } 56 | """ 57 | def __init__(self, n_weight=-1, n_pn="", n_parent=None): 58 | self.sha256 = "" 59 | self.weight = n_weight 60 | self.pn = n_pn 61 | self.parent = n_parent 62 | self.children = dict() 63 | self.match = list() 64 | self.permissions = set() 65 | self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD) 66 | self.api_id_list = [] 67 | 68 | def insert(self, package_name, weight, sha256, permission_list, api_id_list): 69 | # no matter how deep the package is, add permissions here. 70 | for permission in permission_list: 71 | self.permissions.add(permission) 72 | # no matter how deep the package is, add api_id_list 73 | # self.api_id_list = self.api_id_list + api_id_list 74 | current_depth = 0 if self.pn == "" else self.pn.count('/') + 1 75 | target_depth = package_name.count('/') + 1 76 | if current_depth == target_depth: 77 | self.sha256 = sha256 78 | self.api_id_list = api_id_list 79 | return "F: %s" % package_name 80 | target_package_name = '/'.join(package_name.split('/')[:current_depth + 1]) 81 | if target_package_name in self.children: 82 | self.children[target_package_name].weight += weight 83 | return self.children[target_package_name].insert(package_name, weight, sha256, permission_list, api_id_list) 84 | else: 85 | self.children[target_package_name] = TreeNode(n_weight=weight, n_pn=target_package_name, n_parent=self) 86 | return self.children[target_package_name].insert(package_name, weight, sha256, permission_list, api_id_list) 87 | 88 | def brand(self, package_name, standard_package): 89 | current_depth = 0 if self.pn == "" else self.pn.count('/') + 1 90 | target_depth = package_name.count('/') + 1 91 | if current_depth == target_depth: 92 | yes_or_no = raw_input("Warning: Brand %s as a new library? (Y/n)" % self.pn) 93 | if yes_or_no == 'Y' or yes_or_no == 'y': 94 | try: 95 | self.db.hincrby(name=DB_FEATURE_CNT, key=self.sha256, amount=10000000) 96 | self.db.hset(name=DB_FEATURE_WEIGHT, key=self.sha256, value=self.weight) 97 | self.db.hset(name=DB_UN_OB_PN, key=self.sha256, value=standard_package) 98 | self.db.hset(name=DB_FEATURE_CNT, key=self.sha256, value=100000000) 99 | except: 100 | return "Error in database." 101 | return "Success." 102 | else: 103 | return "Did nothing. Bye~" 104 | else: 105 | target_package_name = '/'.join(package_name.split('/')[:current_depth + 1]) 106 | if target_package_name in self.children: 107 | return self.children[target_package_name].brand(package_name, standard_package) 108 | else: 109 | return "Package Not found in this APK." 110 | 111 | 112 | class Tree(object): 113 | """ 114 | Tree 115 | """ 116 | def __init__(self): 117 | self.root = TreeNode() 118 | self.db = None 119 | self.feature = None 120 | self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD) 121 | self.db_rep = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID_REP, password=DB_PSWD) 122 | 123 | def insert(self, package_name, weight, sha256, permission_list, api_id_list): 124 | self.root.insert(package_name, weight, sha256, permission_list, api_id_list) 125 | 126 | def brand(self, package_name, standard_package): 127 | return self.root.brand(package_name, standard_package) 128 | 129 | def pre_order_res(self, visit, res): 130 | self._pre_order_res(node=self.root, visit=visit, res=res) 131 | 132 | def _pre_order_res(self, node, visit, res): 133 | ret = visit(node, res) 134 | if ret < 0: 135 | return 136 | else: 137 | for child_pn in node.children: 138 | self._pre_order_res(node.children[child_pn], visit, res) 139 | 140 | def pre_order_res_ret(self, visit, res, ret): 141 | self._pre_order_res_ret(node=self.root, visit=visit, res=res, ret=ret) 142 | 143 | def _pre_order_res_ret(self, node, visit, res, ret): 144 | retu = visit(node, res, ret) 145 | if retu < 0: 146 | return 147 | else: 148 | for child_pn in node.children: 149 | self._pre_order_res_ret(node.children[child_pn], visit, res, ret) 150 | 151 | def pre_order(self, visit): 152 | self._pre_order(self.root, visit) 153 | 154 | def _pre_order(self, node, visit): 155 | ret = visit(node) 156 | if ret < 0: 157 | return 158 | else: 159 | for child_pn in node.children: 160 | self._pre_order(node.children[child_pn], visit) 161 | 162 | def post_order(self, visit): 163 | self._post_order(self.root, visit) 164 | 165 | def _post_order(self, node, visit): 166 | for child_pn in node.children: 167 | self._post_order(node.children[child_pn], visit) 168 | visit(node) 169 | 170 | @staticmethod 171 | def _cal_sha256(node): 172 | # Ignore Leaf Node 173 | if len(node.children) == 0 and node.sha256 != "": 174 | return 175 | # Everything seems Okay. 176 | cur_sha256 = hashlib.sha256() 177 | sha256_list = list() 178 | for child in node.children: 179 | sha256_list.append(node.children[child].sha256) 180 | sha256_list.sort() 181 | for sha256_item in sha256_list: 182 | cur_sha256.update(sha256_item) 183 | node.sha256 = cur_sha256.hexdigest() 184 | # you could see node.pn here. e.g. Lcom/tencent/mm/sdk/modelpay 185 | 186 | def cal_sha256(self): 187 | """ 188 | Calculate sha256 for every package 189 | :return: 190 | """ 191 | self.post_order(visit=self._cal_sha256) 192 | 193 | def _match(self, node): 194 | a, c, u = None, None, None 195 | pipe = self.db.pipeline() 196 | pipe.hget(name=DB_UN_OB_PN, key=node.sha256) 197 | pipe.hget(name=DB_FEATURE_CNT, key=node.sha256) 198 | pipe.hget(name=DB_UN_OB_CNT, key=node.sha256) 199 | pipe_res = pipe.execute() 200 | a, c, u = pipe_res 201 | 202 | # if could not find this package in database, search its children. 203 | if a is None: 204 | return 1 205 | # Potential Name is not convincing enough. 206 | if u < 8 or float(u) / float(c) < 0.3: 207 | return 2 208 | flag_not_deeper = False 209 | for lib in labeled_libs: 210 | # if the potential package name is the same as full lib path 211 | # do not search its children 212 | if lib[0] == a: 213 | node.match.append([lib, node.weight, int(c)]) 214 | continue 215 | # If they have the same length but not equal to each other, just continue 216 | if len(lib[0]) == len(a): 217 | continue 218 | # if the potential package name is part of full lib path, search its children 219 | # e.g. a is Lcom/google, we could find it as a part of Lcom/google/android/gms, so search its children for 220 | # more details 221 | if len(a) < len(lib[0]) and a == lib[0][:len(a)] and lib[0][len(a)] == '/': 222 | continue 223 | # If the lib path is part of potential package name, add some count into parent's match list. 224 | if len(a) > len(lib[0]) and lib[0] == a[:len(lib[0])] and a[len(lib[0])] == '/': 225 | depth_diff = a.count('/') - lib[0].count('/') 226 | cursor = node 227 | for i in range(depth_diff): 228 | # cursor should not be the root, so cursor's parent should not be None. 229 | if cursor.parent.parent is not None: 230 | cursor = cursor.parent 231 | else: 232 | # root's parent is None 233 | # This situation exists 234 | # For Example: If it takes Lcom/a/b as Lcom/google/android/gms/ads/mediation/customevent, 235 | # It will find its ancestor until root or None. 236 | return 4 237 | flag = False 238 | for matc in cursor.match: 239 | # if matc[0][0] == lib[0]: 240 | if matc[0] == lib: 241 | flag = True 242 | if matc[1] != cursor.weight: 243 | matc[1] += node.weight 244 | if not flag: 245 | cursor.match.append([lib, node.weight, c]) 246 | flag_not_deeper = True 247 | continue 248 | """ 249 | One degree deeper! 250 | 深入探测一层 251 | 252 | There's a situation that a package is a library and the child of a package is also a library. 253 | 库是存在相互嵌套的。 254 | 255 | As we all know that Lcom/unity3d is definitely a Game Engine library. There could be some sub-package 256 | like Lcom/unity3d/player, Lcom/unity3d/plugin, Lcom/unity3d/sdk, etc. So we take Lcom/unity3d as the 257 | root package of this library. 258 | 比如，Lcom/unity3d 显然是Unity3D这个游戏引擎，在游戏引擎下可能会有player, plugin, sdk等次级包（文件夹），所以我们很 259 | 显然地把Lcom/unity3d作为游戏引擎的根包。 260 | 261 | However, Lcom/unity3d/ads is an Advertisement library. 262 | 但是，Lcom/unity3d/ads是Unity3D公司推出的广告库 263 | 264 | If we do not search one degree deeper, we could only find the game engine other than the ads library. 265 | Likewise, we could not find Landroid/support/v4 anymore if we take Landroid/support as a library. 266 | 如果我们不继续搜索的话，那么对于一个应用，我们只能检测到Unity3D这个引擎，无法检测到Unity3D Ads这个广告库。 267 | 268 | Implementation: 269 | 实现： 270 | if lib[0] == a, we continue search his children. 271 | if lib[0] == a 这个后面从return变成了continue，我们会继续搜索它的子节点 272 | 273 | if we already found his child, we will not search deeper. 274 | 在后面的代码中，如果已经知道的就是子节点，那么就不会继续深层的搜了。 275 | 276 | In my original code, I found a bug that the match degree is larger than the total amount of weight. 277 | This is impossible. After debugging, I found that if I add the match value multiple times, the match 278 | weight could overflow. 279 | 在我原来有bug的代码中，我发现匹配的similarity有大于1的情况，即com/facebook这个库的similarity大于了1。这是因为match 280 | 被我加总了数次 281 | 282 | For example: 283 | There's a library Lcom/google/android/gson, weight is 189 284 | we found Lcom/google/android/gson, so add the weight 189 285 | we found Lcom/google/android/gson/internal, so add the weight 24 286 | we found Lcom/google/android/gson/stream, so add the weight 43 287 | In this case, the weight of package gson overflows. 288 | 举例来看： 289 | 对于Lcom/google/android/gson这个包来说，它的API数量是189 290 | 搜索中找到 Lcom/google/android/gson， weight加上189 291 | 搜索中找到 Lcom/google/android/gson/internal， weight加上24 292 | 搜索中找到 Lcom/google/android/gson/stream, weight加上 43 293 | 这样显然就溢出了。 294 | 295 | Because we only search 1 degree deeper, the match situation of Lcom/google/android/gson is only true or 296 | false. In this case, we just need to check if the weight has overflowed before add weight. as the code: 297 | if matc[1] != cursor.weight: 298 | matc[1] += node.weight 299 | 因为我们可以多搜一层，所以判断是否溢出很简单。因为对于上层的库来说，也就只有两种情况，那就是匹配到和没匹配到。所以只需要 300 | 检测一下是否已经超出就行了。 301 | """ 302 | if flag_not_deeper: 303 | return -1 304 | # Never find a good match, search its children. 305 | return 5 306 | 307 | def match(self): 308 | self.pre_order(visit=self._match) 309 | 310 | def _find_untagged(self, node, res): 311 | # If there's already some matches here, do not search its children. non-sense. 312 | a, c, u = None, None, None 313 | if len(node.match) != 0: 314 | return -1 315 | pipe = self.db.pipeline() 316 | pipe.hget(name=DB_UN_OB_PN, key=node.sha256) 317 | pipe.hget(name=DB_FEATURE_CNT, key=node.sha256) 318 | pipe.hget(name=DB_UN_OB_CNT, key=node.sha256) 319 | pipe_res = pipe.execute() 320 | a, c, u = pipe_res 321 | 322 | 323 | if a is None: 324 | return 1 325 | # If the package name is already in no_lib list, ignore it and search its children. 326 | for non_lib in no_lib: 327 | if non_lib[0] == a: 328 | return 1 329 | # Potential Name is not convincing enough. search its children 330 | if float(u) / float(c) < 0.5 or node.weight < 50 or int(c) < 20: 331 | return 2 332 | 333 | # JSON support 334 | utg_lib_obj = dict() # untagged library object 335 | utg_lib_obj["Package"] = node.pn 336 | utg_lib_obj["Standard Package"] = a 337 | utg_lib_obj["Library"] = "Unknown" 338 | utg_lib_obj["Popularity"] = int(c) 339 | utg_lib_obj["Weight"] = node.weight 340 | 341 | res.append(utg_lib_obj) 342 | 343 | # OLD Print 344 | # print("----") 345 | # print("Package: %s" % node.pn) 346 | # print("Match Package: %s" % u) 347 | # print("Library: Unknown.") 348 | # print("Popularity: %s" % c) 349 | # print("API count: %s" % node.weight) 350 | 351 | def find_untagged(self, res): 352 | self.pre_order_res(visit=self._find_untagged, res=res) 353 | 354 | @staticmethod 355 | def _get_lib(node, res): 356 | for matc in node.match: 357 | if float(matc[1]) / float(node.weight) < 0.1 and matc[0][0] != node.pn: 358 | continue 359 | # JSON 360 | lib_obj = dict() 361 | lib_obj["Package"] = node.pn # cpn 362 | lib_obj["Library"] = matc[0][1] # lib 363 | lib_obj["Standard Package"] = matc[0][0] # pn 364 | lib_obj["Type"] = matc[0][2] # tp 365 | lib_obj["Website"] = matc[0][3] # ch 366 | lib_obj["Match Ratio"] = "%d/%d" % (matc[1], node.weight) # no similarity in V1 367 | lib_obj["Popularity"] = matc[2] # dn 368 | lib_obj["Permission"] = sorted(list(node.permissions)) 369 | res.append(lib_obj) 370 | # Old Print 371 | # print("----") 372 | # print("Package: %s" % node.pn) 373 | # print("Library: %s" % matc[0][1]) 374 | # print("Standard Package: %s" % matc[0][0]) 375 | # print("Type: %s" % matc[0][2]) 376 | # print("Website: %s" % matc[0][3]) 377 | # print("Similarity: %d/%d" % (matc[1], node.weight)) 378 | # print("Popularity: %d" % matc[2]) 379 | # permission_out = "" 380 | # for permission in sorted(list(node.permissions)): 381 | # permission_out += (permission + ",") 382 | # if len(permission_out) > 0: 383 | # permission_out = permission_out[:-1] 384 | # print("Permissions:" + permission_out) 385 | return 0 386 | 387 | def get_lib(self, res): 388 | self.pre_order_res(visit=self._get_lib, res=res) 389 | 390 | @staticmethod 391 | def _get_repackage_main(node, res, ret): 392 | if node.pn in res: 393 | return -1 394 | if len(node.children) == 0: 395 | ret.extend(node.api_id_list) 396 | ret += node.api_id_list 397 | return 0 398 | 399 | def get_repackage_main(self, res, hex_sha256): 400 | # res is a list of libraries. Result. 401 | pn_list = list() 402 | for item in res: 403 | pn_list.append(item["Package"]) 404 | ret = list() 405 | self.pre_order_res_ret(visit=self._get_repackage_main, res=pn_list, ret=ret) 406 | ret_length = len(ret) 407 | kvd = dict(Counter(ret)) 408 | str = rputil.Util.dict2str(kvd) 409 | zstr = zlib.compress(str,1) 410 | self.db_rep.hset(name="apk_feature", key=hex_sha256, value=zstr) 411 | self.db_rep.zadd("apk_weight", ret_length, hex_sha256 ) 412 | -------------------------------------------------------------------------------- /LibRadar/job_dispatching.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # dispatcher 19 | # Android Package Extractor Dispatcher 20 | # This script is a dispatcher for scheduling. 21 | 22 | 23 | # Step 1 jar 反编译，得到api dict [tech: jad] 24 | # Step 2 hash feature Tree [tech: sha256] 25 | # Step 3 [tech: Redis] 26 | # Step 3.1 [tech: 众数机制] 27 | # Step 3.2 [tech: 分布式、并行] 28 | # Step 3.3 Redis 导出 29 | # Step 4 Detector 30 | # Step 5 online [tech: node.js] 31 | 32 | from multiprocessing import Pool 33 | import multiprocessing 34 | import glob 35 | from _settings import * 36 | import hashlib 37 | import zipfile 38 | import dex_extracting 39 | import redis 40 | 41 | 42 | class DexExtractorWrapper: 43 | """ 44 | Dex Extractor Wrapper 45 | 46 | Dex Extractor is a class for extracting features from dex file. 47 | DexExtractorWrapper is a container for the extractor. 48 | DexExtractorWrapper occupies a process and got a queue from arguments. 49 | In the main loop, it get an item from the queue and then create a new instance of DexExtractor and run it. 50 | """ 51 | def __init__(self, p_name, queue): 52 | """ 53 | Init the instance with process name and queue(full of app path) 54 | :param p_name: basestring 55 | :param queue: multiprocessing.Manager().Queue 56 | """ 57 | self.p_name = p_name 58 | self.queue = queue 59 | # MD5 for current app, for the current instance of Dex Extractor 60 | self.sha256 = "" 61 | # Path for current app. 62 | self.app_path = "" 63 | 64 | def execute(self): 65 | logger.info("Process %s is running" % self.p_name) 66 | while True: 67 | try: 68 | self.app_path = self.queue.get(block=True, timeout=QUEUE_TIME_OUT) 69 | except: # multiprocessing.Manager().Queue.Empty: 70 | break 71 | logger.debug("Process %s is extracting %s" % (self.p_name, self.app_path)) 72 | try: 73 | self.get_sha256() 74 | except: 75 | # Yeah, Too broad exception it is. But I don't care. 76 | # There could be many types of exception but what I want to do is ignore the wrong apk and focus on 77 | # the next app. 78 | logger.error("Process %s get Md5 error!" % self.p_name) 79 | continue 80 | # logger.info("Process %s got sha256 %s" % (self.p_name, self.sha256)) 81 | # If the apk file is broken, it can not be unzipped. 82 | try: 83 | zf = zipfile.ZipFile(self.app_path, mode="r") 84 | dex_file_extracted = zf.extract("classes.dex", SCRIPT_PATH + "/Data/Decompiled/%s" % self.sha256) 85 | except: 86 | logger.error("Process %s, not a valid Zip file." % self.p_name) 87 | continue 88 | try: 89 | de = dex_extracting.DexExtractor(dex_file_extracted) 90 | de.extract_dex() 91 | except: 92 | logger.critical("Process %s, extracting error!!" % self.p_name) 93 | continue 94 | try: 95 | cmd = 'rm -rf ' + SCRIPT_PATH + "/Data/Decompiled/%s" % self.sha256 96 | os.system(cmd) 97 | except: 98 | logger.error("Process %s, rm error" % self.p_name) 99 | logger.info("Process %s returns" % self.p_name) 100 | 101 | def get_sha256(self): 102 | if not os.path.isfile(self.app_path): 103 | logger.critical("file path %s is not a file" % self.app_path) 104 | raise AssertionError 105 | file_sha256 = hashlib.sha256() 106 | f = file(self.app_path, 'rb') 107 | while True: 108 | block = f.read(4096) 109 | if not block: 110 | break 111 | file_sha256.update(block) 112 | f.close() 113 | file_sha256_value = file_sha256.hexdigest() 114 | logger.debug("APK %s's MD5 is %s" % (self.app_path, file_sha256_value)) 115 | self.sha256 = file_sha256_value 116 | return file_sha256_value 117 | 118 | 119 | def run_dex_extractor_wrapper(process_name, q): 120 | dew = DexExtractorWrapper(process_name, q) 121 | dew.execute() 122 | 123 | 124 | class DexExtractorDispatcher: 125 | """ 126 | Dex Extractor Dispatcher 127 | """ 128 | def __init__(self, folder_full_of_apps): 129 | self.folder = folder_full_of_apps 130 | 131 | @staticmethod 132 | def clear_decompiled(): 133 | cmd = 'rm -rf ' + SCRIPT_PATH + '/Data/Decompiled' 134 | os.system(cmd) 135 | 136 | def execute(self): 137 | q = multiprocessing.Manager().Queue() 138 | p = Pool() 139 | logger.info("Pool created") 140 | app_list = glob.glob("%s/*" % self.folder) 141 | for apk in app_list: 142 | if len(apk) < 4 or apk[-4:] != ".apk": 143 | continue 144 | q.put(apk) 145 | for i in range(RUNNING_PROCESS_NUMBER): 146 | process_name = str(i).zfill(2) 147 | p.apply_async(run_dex_extractor_wrapper, args=(process_name, q)) 148 | logger.info("Waiting for all sub-processes done.") 149 | p.close() 150 | p.join() 151 | logger.critical("All sub-processes done.") 152 | 153 | if __name__ == "__main__": 154 | ded = DexExtractorDispatcher("/home/zachary/Projects/apks") 155 | ded.clear_decompiled() 156 | ded.execute() 157 | # q:-) 158 | # "eject" is just a trick as a reminder. Do not work on Mac obviously. Remove it when necessary. 159 | # d:-) 160 | os.system("eject cdrom") 161 | -------------------------------------------------------------------------------- /LibRadar/lib_tagging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Library tagger 18 | # 19 | # This script should be an interact-able script. 20 | 21 | 22 | 23 | import redis 24 | import csv 25 | import os.path 26 | from _settings import * 27 | 28 | 29 | class Tagger: 30 | """ 31 | Database tagger. 32 | 33 | My algorithm could only tell you that which package is library. 34 | I could tell you the package name but I don't have specific information like it's real library name, official 35 | website 36 | 37 | Rule 38 | A rule is established for tagging libraries. 39 | As there's many libraries, it could be many Lcom/google/android/gms, it is very hard to tag them one by one. 40 | So we need a rule file and tag all the libraries with package name started with 'Lcom/google/android/gms' as 41 | the library GMS. 42 | 43 | Rule information could be taken as a hash table. 44 | Key: 45 | package name 46 | Value: 47 | Library Name , Library Type, Official Website, 48 | """ 49 | def __init__(self, base_count=20, base_weight=100): 50 | self.db = redis.StrictRedis(host=DB_HOST, port=DB_PORT, db=DB_ID, password=DB_PSWD) 51 | """ 52 | Rules: 53 | There would be hundreds of lines of rules. Quite small the file should be. 54 | Thus I use a **file** rather that put them into database because that will be visible for developers. 55 | Every time we should import the file into main memory and use a dict for quick access, but it won't take a 56 | large portion of main memory. 57 | tag_rules.csv should be less than 100KB. 58 | """ 59 | self.dict_tag_rules = dict() 60 | self.new_prefix_list = list() 61 | if os.path.exists(FILE_RULE): 62 | file_rules = open(FILE_RULE, 'r') 63 | csv_rules_reader = csv.reader(file_rules, delimiter=',', quotechar='|') 64 | for row in csv_rules_reader: 65 | self.dict_tag_rules[row[0]] = (row[1], row[2], row[3]) 66 | file_rules.close() 67 | file_rules_w = open(FILE_RULE, 'a') 68 | self.csv_rule_writer = csv.writer(file_rules_w, delimiter=',', quotechar='|') 69 | self.base_count = base_count 70 | self.base_weight = base_weight 71 | logger.debug("LibRadar Tagger Initiated.") 72 | self.features = list() 73 | 74 | def get_potential_list(self): 75 | logger.debug("Searching in database. Need a few seconds.") 76 | # Yeah, use 'keys' function may block a while, but lib_tagging.py is designed for professional use only. 77 | # Up to now, do not like to support multi-threading. 78 | 79 | f_cnt = self.db.hgetall(name=DB_FEATURE_CNT) 80 | f_weight = self.db.hgetall(name=DB_FEATURE_WEIGHT) 81 | f_un_ob_pn = self.db.hgetall(name=DB_UN_OB_PN) 82 | for key in f_cnt: 83 | curr_count = int(f_cnt[key]) 84 | if curr_count < self.base_count: 85 | continue 86 | curr_weight = int(f_weight[key]) 87 | if curr_weight < self.base_weight: 88 | continue 89 | # sha256, count, weight, pn 90 | self.features.append((key, curr_count, curr_weight, f_un_ob_pn[key])) 91 | # sort with count and weight 92 | # count is more important so I gave it 3 times weight. 93 | self.features.sort(cmp=lambda x, y: cmp(y[1]*3 + y[2], x[1] * 3 + x[2])) 94 | return self.features 95 | 96 | def set_rule(self, ipt_pn, ipt_real_name, ipt_type, ipt_website): 97 | self.csv_rule_writer.writerow([ipt_pn, ipt_real_name, ipt_type, ipt_website]) 98 | # use for ignore libs that already tagged. 99 | self.dict_tag_rules[ipt_pn] = (ipt_real_name, ipt_type, ipt_website) 100 | 101 | def exist(self, full_package_name): 102 | flag = False 103 | for key_pn in self.dict_tag_rules: 104 | if key_pn == full_package_name[:len(key_pn)]: 105 | flag = True 106 | break 107 | return flag 108 | 109 | 110 | class TaggerCli: 111 | def __init__(self): 112 | self._print_title() 113 | self._set_base() 114 | self._tag() 115 | 116 | @staticmethod 117 | def _print_title(): 118 | print("|---- LibRadar Tagging System -----|") 119 | print("| Version: 2.0.1.dev1 |") 120 | print("| Author: Zachary Ma |") 121 | print("|----------------------------------|") 122 | 123 | def _set_base(self): 124 | """ 125 | Option settings. 126 | """ 127 | """ 128 | ignore 'only tag new' option 129 | 130 | only_tag_new = True 131 | while True: 132 | try: 133 | only_tag_new_option = raw_input("Only tag libraries that have not been tagged (Y/N): ") 134 | if only_tag_new_option == "N" or only_tag_new_option == "n": 135 | only_tag_new = False 136 | break 137 | if only_tag_new_option == "Y" or only_tag_new_option == "y": 138 | only_tag_new = True 139 | break 140 | print("Sorry, I'm not sure what you mean.") 141 | except: 142 | continue 143 | """ 144 | base_count = -1 145 | base_weight = -1 146 | try: 147 | base_count = input("Set the minimum number of repetitions (Default 20): ") 148 | except NameError: 149 | base_count = 20 150 | except SyntaxError: 151 | base_count = 20 152 | finally: 153 | print("The minimum number of repetitions is set as %d." % base_count) 154 | try: 155 | base_weight = input("Set the minimum API count of libraries (Default 100): ") 156 | except NameError: 157 | base_weight = 100 158 | except SyntaxError: 159 | base_weight = 100 160 | finally: 161 | print("The minimum API count of library is set as %d" % base_weight) 162 | self.tagger = Tagger(base_count, base_weight) 163 | 164 | def _tag(self): 165 | self.features = self.tagger.get_potential_list() 166 | # Print length of features 167 | feature_num = len(self.features) 168 | feature_iterator = 0 169 | print("There're %d features need to be tagged" % feature_num) 170 | while True: 171 | if feature_iterator >= feature_num: 172 | break 173 | # If already tagged: 174 | if self.tagger.exist(self.features[feature_iterator][3]): 175 | print(" Ignore %d %s" % (feature_iterator + 1, self.features[feature_iterator][3])) 176 | feature_iterator += 1 177 | continue 178 | # If not tagged. 179 | print("-" * 60) 180 | print("# Tagging %d/%d." % (feature_iterator + 1, feature_num)) 181 | print(" Potential Package Name: %s" % self.features[feature_iterator][3]) 182 | print(" Number of repetitions: %s" % self.features[feature_iterator][1]) 183 | print(" API contains: %s" % self.features[feature_iterator][2]) 184 | print("-" * 60) 185 | ipt_know = raw_input("Do you know this library? (Y:Yes/ N:No/ W: It's not a library!)") 186 | if ipt_know == 'N' or ipt_know == 'n': 187 | print("OK, next one.") 188 | feature_iterator += 1 189 | continue 190 | if ipt_know == 'W' or ipt_know == 'w': 191 | print("OK, next one.") 192 | feature_iterator += 1 193 | if ipt_know != 'Y' and ipt_know != 'y': 194 | print("Sorry? I don't know what you mean.") 195 | continue 196 | ipt_pn = raw_input("Please input its true package name: ") 197 | # input longer warning! 198 | if len(ipt_pn) > len(self.features[feature_iterator][3]): 199 | logger.warning("Input longer that the original potential package name???") 200 | ipt_re = raw_input("Insist(Y/N).") 201 | if ipt_re == 'N' or ipt_re == 'n': 202 | continue 203 | # prefix not the same warning! 204 | if self.features[feature_iterator][3][:len(ipt_pn)] != ipt_pn: 205 | logger.warning("Prefix not the same??? %s,%s" % (self.features[feature_iterator][3], ipt_pn)) 206 | ipt_re = raw_input("Insist(Y/N).") 207 | if ipt_re == 'N' or ipt_re == 'n': 208 | continue 209 | # Real Name 210 | ipt_real_name = raw_input("Input Library's real name: ") 211 | # type 212 | ipt_lib_type = raw_input("Input Library Type: ") 213 | # Official website 214 | ipt_website = raw_input("Input Official Website: ") 215 | self.tagger.set_rule(ipt_pn, ipt_real_name, ipt_lib_type, ipt_website) 216 | 217 | # Next 218 | feature_iterator += 1 219 | -------------------------------------------------------------------------------- /LibRadar/lib_tagging_gui.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # Library tagger GUI 19 | # 20 | # When tagging libraries using this script, you could refer to 21 | # https://raw.githubusercontent.com/pkumza/LibRadar/v1.4.0/data/tgst5.dat and 22 | # https://github.com/pkumza/LibRadar/blob/dev/extract/step2.py 23 | # 24 | # Introduction: 25 | # This script is not used for tagging libraries actually, It gives you some untagged libraries and you could set a 26 | # rule for tagging. 27 | # As there would be so many packages and so many libraries, it not possible to tag them one by one. 28 | # There could be many versions of libraries, so there's some repeat work. 29 | # 30 | # We presume that more than half package names of a library is un-obfuscated, in this case, we could know the true 31 | # package name but we could not get the real library name, the website, type, or the other information. 32 | # 33 | # tag_rules.csv is a projection from un-obfuscated package name to its detailed information. 34 | # 35 | # Usage: 36 | # Run this script or just modify Data/IntermediateData/tag_rules.csv 37 | 38 | 39 | import lib_tagging 40 | import csv 41 | import os 42 | from _settings import FILE_RULE 43 | 44 | from Tkinter import * # 导入 Tkinter 库 45 | import ttk 46 | 47 | 48 | class TaggerGui(Frame): 49 | def __init__(self, master=None): 50 | Frame.__init__(self, master) 51 | self.pack() 52 | # Frame 1 Welcome 53 | self.create_f1_welcome() 54 | # read csv 55 | # contains items like android/support/v4, org/apache/http, 56 | self.labeled_prefix = list() 57 | # contains items like com/android, com/google 58 | self.no_lib_prefix = list() 59 | if os.path.exists(FILE_RULE): 60 | file_rules = open(FILE_RULE, 'r') 61 | csv_rules_reader = csv.reader(file_rules, delimiter=',', quotechar='|') 62 | for row in csv_rules_reader: 63 | if row[1] == "no": 64 | self.no_lib_prefix.append(row[0]) 65 | else: 66 | self.labeled_prefix.append(row[0]) 67 | file_rules.close() 68 | self.file_rules_w = open(FILE_RULE, 'a') 69 | self.csv_rule_writer = csv.writer(self.file_rules_w, delimiter=',', quotechar='|') 70 | 71 | def create_f1_welcome(self): 72 | self.hello_label = Label(self, 73 | text='\nHello!\nThis is LibRadar Tagging System.\n' 74 | 'Author: Zachary Marv') 75 | self.hello_label.pack(side=TOP) 76 | self.next_button = Button(self, text="Next", command=self.destroy_f1) 77 | self.next_button.pack() 78 | self.quit_button = Button(self, text='Quit', command=self.quit) 79 | self.quit_button.pack() 80 | 81 | def destroy_f1(self): 82 | self.hello_label.destroy() 83 | self.next_button.destroy() 84 | self.quit_button.destroy() 85 | self.create_f2_init() 86 | 87 | def create_f2_init(self): 88 | self.master.geometry("300x210") 89 | self.hello_label = Label(self, 90 | text="\nPlease input some arguments for this system.\nThis would take a few moments," 91 | " please wait.") 92 | self.hello_label.pack() 93 | self.count_label = Label(self, text="Base count (default 20):") 94 | self.count_label.pack() 95 | self.count_text = StringVar() 96 | self.count_entry = Entry(self, textvariable=self.count_text) 97 | self.count_entry.pack() 98 | self.weight_text = StringVar() 99 | self.weight_label = Label(self, text="Weight count (default 20):") 100 | self.weight_label.pack() 101 | self.weight_entry = Entry(self, textvariable=self.weight_text) 102 | self.weight_entry.pack() 103 | self.d2c3_button = Button(self, text="Next", command=self.destroy_f2) 104 | self.d2c3_button.pack() 105 | 106 | def destroy_f2(self): 107 | self.base_count = -1 108 | self.base_weight = -1 109 | good_flag = True 110 | try: 111 | if self.weight_text.get() == "": 112 | self.base_weight = 20 113 | else: 114 | self.base_weight = int(self.weight_text.get()) 115 | if self.count_text.get() == "": 116 | self.base_count = 20 117 | else: 118 | self.base_count = int(self.count_text.get()) 119 | 120 | except: 121 | good_flag = False 122 | if self.base_count < 0: 123 | good_flag = False 124 | if self.base_weight < 0: 125 | good_flag = False 126 | self.hello_label.destroy() 127 | self.count_label.destroy() 128 | self.count_entry.destroy() 129 | self.weight_label.destroy() 130 | self.weight_entry.destroy() 131 | self.d2c3_button.destroy() 132 | if good_flag: 133 | self.init_data_for_f3() 134 | else: 135 | self.create_f2_init() 136 | 137 | def init_data_for_f3(self): 138 | """frame for tagging""" 139 | # self.master.geometry("300x120") 140 | self.tagger = lib_tagging.Tagger(self.base_count, self.base_weight) 141 | self.features = self.tagger.get_potential_list() 142 | self.create_f3_init() 143 | 144 | def create_f3_init(self): 145 | """ 146 | 147 | :return: 148 | """ 149 | """ 150 | INFO 151 | """ 152 | self.master.geometry("300x520") 153 | self.label_index = -1 154 | self.labeled_cnt = 0 155 | 156 | """ 157 | GUI 158 | """ 159 | self.label_message = Label(self, text="") 160 | self.label_message.pack() 161 | self.label_progress = Label(self, text="\nProgress:") 162 | self.label_progress.pack() 163 | 164 | self.label_count = Label(self, text="\nPopularity:") 165 | self.label_count.pack() 166 | self.n_label_count = Label(self, text="") 167 | self.n_label_count.pack() 168 | self.label_weight = Label(self, text="API number:") 169 | self.label_weight.pack() 170 | self.n_label_weight = Label(self, text="") 171 | self.n_label_weight.pack() 172 | self.label_package_name = Label(self, text="Package Name:") 173 | self.label_package_name.pack() 174 | self.n_label_package_name = Label(self, text="") 175 | self.n_label_package_name.pack() 176 | self.label_tagging = Label(self, text="Library prefix:") 177 | self.label_tagging.pack() 178 | self.t_lib_prefix = StringVar() 179 | self.entry_prefix = Entry(self, textvariable=self.t_lib_prefix) 180 | self.entry_prefix.pack() 181 | self.label_lib_name = Label(self, text="Library Name:") 182 | self.label_lib_name.pack() 183 | self.t_lib_name = StringVar() 184 | self.entry_name = Entry(self, textvariable=self.t_lib_name) 185 | self.entry_name.pack() 186 | self.label_type = Label(self, text="Library Type:") 187 | self.label_type.pack() 188 | self.t_lib_type = StringVar() 189 | self.type_chosen = ttk.Combobox(self, textvariable=self.t_lib_type) 190 | self.type_chosen['values'] = ( 191 | "Development Aid", 192 | "Social Network", 193 | "Advertisement", 194 | "App Market", 195 | "Mobile Analytics", 196 | "Payment", 197 | "Game Engine", 198 | "Map/LBS", 199 | "GUI Component", 200 | "Development Framework", 201 | "Not Sure" 202 | ) 203 | self.type_chosen.current(0) 204 | self.type_chosen.pack() 205 | self.label_website = Label(self, text="Official SDK Website:") 206 | self.label_website.pack() 207 | self.t_website = StringVar() 208 | self.entry_website = Entry(self, textvariable=self.t_website) 209 | self.entry_website.pack() 210 | self.button_tag = Button(self, text="Tag!", command=self.insert_csv) 211 | self.button_tag.pack() 212 | self.button_ignore = Button(self, text="I don't know", command=self.ignore_f3) 213 | self.button_ignore.pack() 214 | self.button_not = Button(self, text="This package is definitely not a library!", command=self.insert_not_csv) 215 | self.button_not.pack() 216 | 217 | self.create_f3() 218 | 219 | def tagged(self, package_name): 220 | """ 221 | Determine if package_name needed to be tagged. 222 | :param package_name: the package name to be determined. 223 | :return: boolean 224 | """ 225 | # never found it is tagged. 226 | flag = False 227 | for prefix in self.labeled_prefix: 228 | if prefix in package_name: 229 | flag = True 230 | break 231 | if flag: 232 | return flag 233 | for prefix in self.no_lib_prefix: 234 | if prefix == package_name: 235 | flag = True 236 | break 237 | return flag 238 | 239 | def create_f3(self): 240 | """ 241 | loop and loop 242 | :return: 243 | """ 244 | 245 | # Clear Text 246 | self.t_website.set("") 247 | self.t_lib_name.set("") 248 | 249 | # Index++ 250 | self.label_index += 1 251 | # Ignore those tagged libs. 252 | while self.label_index < len(self.features) and self.tagged(self.features[self.label_index][3]): 253 | self.label_index += 1 254 | if self.label_index >= len(self.features): 255 | self.destroy_f3() 256 | return 257 | 258 | self.t_lib_prefix.set(self.features[self.label_index][3]) 259 | # Show information 260 | self.label_progress["text"] = "\nProgress: %d/%d" % (self.label_index + 1, len(self.features)) 261 | self.n_label_count["text"] = str(self.features[self.label_index][1]) 262 | self.n_label_weight["text"] = str(self.features[self.label_index][2]) 263 | self.n_label_package_name["text"] = str(self.features[self.label_index][3]) 264 | 265 | def ignore_f3(self): 266 | self.label_message["text"] = "" 267 | self.create_f3() 268 | 269 | def insert_csv(self): 270 | """ 271 | 272 | :return: 273 | """ 274 | self.label_message["text"] = "" 275 | good_flag = True 276 | lib_name = "" 277 | lib_type = "" 278 | website = "" 279 | prefix = "" 280 | try: 281 | prefix = self.t_lib_prefix.get() 282 | if self.features[self.label_index][3][:len(prefix)] != prefix: 283 | self.label_message["text"] = "Prefix not the same! try again!" 284 | self.label_index -= 1 285 | self.create_f3() 286 | return 287 | lib_name = self.t_lib_name.get() 288 | website = self.t_website.get() 289 | lib_type = self.t_lib_type.get() 290 | except: 291 | good_flag = False 292 | if good_flag: 293 | # insert into csv 294 | self.csv_rule_writer.writerow([prefix, lib_name, lib_type, website]) 295 | # insert the prefix into list 296 | self.labeled_prefix.append(prefix) 297 | else: 298 | # ignore 299 | pass 300 | self.create_f3() 301 | 302 | def insert_not_csv(self): 303 | """ 304 | 305 | :return: 306 | """ 307 | self.label_message["text"] = "" 308 | good_flag = True 309 | lib_name = "" 310 | website = "" 311 | prefix = "" 312 | try: 313 | prefix = str(self.features[self.label_index][3]) 314 | lib_name = "no" 315 | website = "no" 316 | except: 317 | good_flag = False 318 | if good_flag: 319 | # insert into csv 320 | self.csv_rule_writer.writerow([prefix, lib_name, "no", website]) 321 | # insert the prefix into list 322 | self.no_lib_prefix.append(prefix) 323 | else: 324 | # ignore 325 | pass 326 | self.create_f3() 327 | 328 | 329 | def destroy_f3(self): 330 | self.label_message.destroy() 331 | self.label_progress.destroy() 332 | self.label_count.destroy() 333 | self.n_label_count.destroy() 334 | self.label_weight.destroy() 335 | self.n_label_weight.destroy() 336 | self.label_package_name.destroy() 337 | self.n_label_package_name.destroy() 338 | self.label_tagging.destroy() 339 | self.entry_prefix.destroy() 340 | self.label_lib_name.destroy() 341 | self.entry_name.destroy() 342 | self.label_website.destroy() 343 | self.entry_website.destroy() 344 | self.button_tag.destroy() 345 | self.button_ignore.destroy() 346 | self.button_not.destroy() 347 | 348 | # Create Exit 349 | self.create_f4() 350 | 351 | def create_f4(self): 352 | """ 353 | You have tagged XXXX libs. 354 | :param self: 355 | :return: 356 | """ 357 | self.hello_label = Label(self, text='The End\nplease exit.') 358 | self.hello_label.pack(side=TOP) 359 | pass 360 | 361 | 362 | if __name__ == '__main__': 363 | app = TaggerGui() 364 | # Set title 365 | app.master.title('LibRadar Tagging System') 366 | # Size 367 | app.master.geometry("300x160") 368 | # Main loop 369 | app.mainloop() 370 | -------------------------------------------------------------------------------- /LibRadar/libradar.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # LibRadar 19 | # 20 | # main function for instant detection. 21 | 22 | 23 | import sys 24 | from _settings import * 25 | import dex_tree 26 | import dex_parser 27 | import hashlib 28 | import zipfile 29 | import json 30 | from collections import Counter 31 | 32 | class LibRadar(object): 33 | """ 34 | LibRadar 35 | """ 36 | def __init__(self, apk_path): 37 | """ 38 | Init LibRadar instance with apk_path as a basestring. 39 | Create a Tree for every LibRadar instance. The tree describe the architecture of the apk. Every package is a 40 | node. 41 | :param apk_path: basestring 42 | """ 43 | self.apk_path = apk_path 44 | self.tree = dex_tree.Tree() 45 | self.dex_name = "" 46 | # Instance of Dex Object in dex_parser. 47 | self.dex = None 48 | """ 49 | Use redis database to exam whether a call is an Android API consumes 27% running time. 50 | I think it should be replaced by a hash table as the API list could not be modified during the progress. 51 | """ 52 | self.k_api_v_permission = dict() 53 | with open(SCRIPT_PATH + "/Data/IntermediateData/strict_api.csv", 'r') as api_and_permission: 54 | api_id = 0 55 | for line in api_and_permission: 56 | api, permission_with_colon = line.split(",") 57 | permissions = permission_with_colon[:-2].split(":") 58 | # delete the last empty one 59 | permission_list = list() 60 | for permission in permissions: 61 | if permission != "": 62 | permission_list.append(permission) 63 | self.k_api_v_permission[api] = (permission_list, api_id) 64 | api_id += 1 65 | """ 66 | invoke_file = open(SCRIPT_PATH +"/Data/IntermediateData/invokeFormat.txt", 'r') 67 | self.invokes = set() 68 | for line in invoke_file: 69 | self.invokes.add(line[:-1]) 70 | """ 71 | 72 | def __del__(self): 73 | # Delete dex file 74 | if CLEAN_WORKSPACE >= 3: 75 | os.remove(self.dex_name) 76 | os.removedirs(self.dex_name[:-12]) 77 | 78 | def unzip(self): 79 | # If it is a valid file 80 | if not os.path.isfile(self.apk_path): 81 | logger.error("%s is not a valid file." % self.apk_path) 82 | raise AssertionError 83 | # If it is a apk file 84 | if len(self.apk_path) <= 4 or self.apk_path[-4:] != ".apk": 85 | logger.error("%s is not a apk file.") 86 | raise AssertionError 87 | # Get SHA256 88 | self.hex_sha256 = self.get_sha256() 89 | # Unzip 90 | zf = zipfile.ZipFile(self.apk_path, mode='r') 91 | # Transfer the unzipped dex file name to self.dex_name 92 | self.dex_name = zf.extract("classes.dex", SCRIPT_PATH + "/Data/Decompiled/%s" % self.hex_sha256) 93 | return self.dex_name 94 | 95 | def get_sha256(self): 96 | if not os.path.isfile(self.apk_path): 97 | logger.critical("file path %s is not a file" % self.apk_path) 98 | raise AssertionError 99 | file_sha256 = hashlib.sha256() 100 | f = file(self.apk_path, 'rb') 101 | while True: 102 | block = f.read(4096) 103 | if not block: 104 | break 105 | file_sha256.update(block) 106 | f.close() 107 | file_sha256_value = file_sha256.hexdigest() 108 | logger.debug("APK %s's MD5 is %s" % (self.apk_path, file_sha256_value)) 109 | return file_sha256_value 110 | 111 | def get_api_list(self, dex_method, api_list, permission_list): 112 | if dex_method.dexCode is None: 113 | return 114 | offset = 0 115 | insns_size = dex_method.dexCode.insnsSize * 4 116 | while offset < insns_size: 117 | op_code = int(dex_method.dexCode.insns[offset:offset + 2], 16) 118 | decoded_instruction = dex_parser.dexDecodeInstruction(self.dex, dex_method.dexCode, offset) 119 | smali_code = decoded_instruction.smaliCode 120 | if smali_code is None: 121 | logger.warning("smali code is None.") 122 | continue 123 | # Next Instruction. 124 | offset += decoded_instruction.length 125 | if smali_code == 'nop': 126 | break 127 | # 4 invokes from 0x6e to 0x72 128 | if 0x6e <= op_code <= 0x72: 129 | if decoded_instruction.getApi in self.k_api_v_permission: 130 | api_list.append(decoded_instruction.getApi) 131 | for permission in self.k_api_v_permission[decoded_instruction.getApi][0]: 132 | permission_list.add(permission) 133 | return 134 | 135 | def extract_class(self, dex_class_def_obj): 136 | class_sha256 = hashlib.sha256() 137 | # API List 138 | # a list for basestring 139 | api_list = list() 140 | permission_list = set() 141 | # direct methods 142 | last_method_index = 0 143 | for k in range(len(dex_class_def_obj.directMethods)): 144 | current_method_index = last_method_index + dex_class_def_obj.directMethods[k].methodIdx 145 | last_method_index = current_method_index 146 | self.get_api_list(dex_class_def_obj.directMethods[k], api_list=api_list, permission_list=permission_list) 147 | # virtual methods 148 | last_method_index = 0 149 | for k in range(len(dex_class_def_obj.virtualMethods)): 150 | current_method_index = last_method_index + dex_class_def_obj.virtualMethods[k].methodIdx 151 | last_method_index = current_method_index 152 | self.get_api_list(dex_class_def_obj.virtualMethods[k], api_list=api_list, permission_list=permission_list) 153 | # Use sort to pass the tree construction stage. 154 | # In this case, we could only use a stack to create the package features. 155 | api_list.sort() 156 | for api in api_list: 157 | class_sha256.update(api) 158 | if not IGNORE_ZERO_API_FILES or len(api_list) != 0: 159 | pass 160 | # api_id_list 161 | api_id_list = [] 162 | for api in api_list: 163 | api_id_list.append(self.k_api_v_permission[api][1]) 164 | return len(api_list), class_sha256.hexdigest(), class_sha256.hexdigest(), sorted(list(permission_list)),\ 165 | api_id_list 166 | 167 | def extract_dex(self): 168 | # Log Start 169 | logger.debug("Extracting %s" % self.dex_name) 170 | # Validate existing 171 | if not os.path.isfile(self.dex_name): 172 | logger.error("%s is not a file" % self.dex_name) 173 | return -1 174 | # Create a Dex object 175 | self.dex = dex_parser.DexFile(self.dex_name) 176 | for dex_class_def_obj in self.dex.dexClassDefList: 177 | weight, raw_sha256, hex_sha256, permission_list, api_id_list = \ 178 | self.extract_class(dex_class_def_obj=dex_class_def_obj) 179 | class_name = self.dex.getDexTypeId(dex_class_def_obj.classIdx) 180 | """ 181 | I got many \x01 here before the class name. 182 | such as '\x01Lcom/vungle/publisher/inject' 183 | don't know exactly but could use code below to deal with it. 184 | """ 185 | if class_name[0] is not 'L': 186 | l_index = class_name.find('L') 187 | if l_index == '-1': 188 | continue 189 | class_name = class_name[l_index:] 190 | if IGNORE_ZERO_API_FILES and weight == 0: 191 | continue 192 | self.tree.insert(package_name=class_name, weight=weight, sha256=raw_sha256, 193 | permission_list=permission_list, api_id_list=api_id_list) 194 | return 0 195 | 196 | def analyse(self): 197 | """ 198 | Main function for LibRadar Object. 199 | :return: None 200 | """ 201 | # Step 1: Unzip APK file, only extract the dex file. 202 | self.unzip() 203 | # Step 2: Extract Dex and insert package-level info into Tree 204 | self.extract_dex() 205 | # Step 3: post-order traverse the tree, calculate every package's sha256 value. 206 | self.tree.cal_sha256() 207 | 208 | 209 | def compare(self): 210 | self.analyse() 211 | # Step 4: pre-order traverse the tree, calculate every node's match degree (similarity). 212 | self.tree.match() 213 | # Init res for step 5 & 6 214 | res = list() 215 | # Step 5: traverse the tree, find out all the libraries. 216 | self.tree.get_lib(res) 217 | # Step 6: traverse the tree, find potential libraries that has not been tagged. 218 | self.tree.find_untagged(res) 219 | # Step 7: repackage feature store. 220 | self.tree.get_repackage_main(res, self.hex_sha256) 221 | return res 222 | 223 | 224 | if __name__ == '__main__': 225 | if len(sys.argv) != 2: 226 | print("LibRadar only takes 1 arguments.") 227 | print("Usage:") 228 | print(" $ python libradar.py example.apk") 229 | exit(1) 230 | apk_path = sys.argv[1] 231 | lrd = LibRadar(apk_path) 232 | res = lrd.compare() 233 | print(json.dumps(res, indent=4, sort_keys=True)) -------------------------------------------------------------------------------- /LibRadar/lite_dataset.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from _settings import * 18 | import redis 19 | 20 | THRESHOLD = 10 21 | 22 | db = redis.StrictRedis(host=DB_HOST,port=DB_PORT, db=2, password=DB_PSWD) 23 | 24 | no_lib_packages = set() 25 | 26 | with open("Data/IntermediateData/tag_rules.csv", "r") as rules: 27 | for line in rules: 28 | if line.startswith("Package"): 29 | continue 30 | pn, lib = line.split(',')[0],line.split(',')[1] 31 | if lib == "no": 32 | no_lib_packages.add(pn) 33 | 34 | cursor = 0 35 | dataset = open("Data/IntermediateData/lite_dataset_%d.csv" % THRESHOLD, "w") 36 | for i in range(11123): 37 | if i % 10 == 0: 38 | print ("Progress:%d" % i) 39 | res = db.hscan(name="feature_cnt", cursor=cursor, count=1000) 40 | cursor = res[0] 41 | for k in res[1]: 42 | if int(res[1][k]) > THRESHOLD: 43 | weight = db.hget(name="feature_weight", key=k) 44 | un_ob_cnt = db.hget(name="un_ob_cnt", key=k) 45 | if float(un_ob_cnt) / float(res[1][k]) < 0.2: 46 | continue 47 | un_ob_pn = db.hget(name="un_ob_pn", key=k) 48 | if un_ob_pn in no_lib_packages: 49 | continue 50 | dataset.write("%s,%s,%s,%s,%s\n" %(k, res[1][k], weight, un_ob_cnt, un_ob_pn)) 51 | dataset.close() -------------------------------------------------------------------------------- /LibRadar/rputil.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Copyright 2017 Zachary Marv (马子昂) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Utility for repackage detection use 18 | 19 | from _settings import * 20 | 21 | class Util(object): 22 | """ 23 | Format of String Storage: 24 | 25 | {1:2} 26 | |--------|--------|--------|---------| 27 | |00000000|00000001|00000000|000000010| 28 | |--------|--------|--------|---------| 29 | 30 | 31 | {2:3,4:128} 32 | |--------|--------|--------|---------|--------|--------|--------|---------| 33 | |00000000|00000010|00000000|000000011|00000000|00000100|00000000|100000000| 34 | |--------|--------|--------|---------|--------|--------|--------|---------| 35 | 36 | """ 37 | @staticmethod 38 | def dict2str(kvd): 39 | """ 40 | Convert Key-Value Dict to String 41 | :param kvd: Key-Value Dict 42 | :return: feature_str String 43 | """ 44 | feature_str = "" 45 | assert type(kvd) == dict, 'kvd is a dictionary' 46 | items = kvd.items() 47 | items.sort() 48 | for item in items: 49 | if type(item[0]) is not int: 50 | logger.error("Key not int") 51 | if item[0] >= 256 * 256: 52 | logger.error("Key too Large") 53 | s1 = chr(item[0] / 256) 54 | s2 = chr(item[0] % 256) 55 | if type(item[1]) is not int: 56 | logger.error("Value not int.") 57 | if item[1] >= 256 * 256: 58 | logger.warning("Value too Large.") 59 | s3 = chr(255) 60 | s4 = chr(255) 61 | else: 62 | s3 = chr(item[1] / 256) 63 | s4 = chr(item[1] % 256) 64 | feature_str += s1 + s2 + s3 + s4 65 | return feature_str 66 | 67 | @staticmethod 68 | def str2dict(feature_str): 69 | """ 70 | Convert String to Key-Value Dict 71 | :param feature_str: 72 | :return: kvd 73 | """ 74 | kvd = dict() 75 | feature_mod = len(feature_str) % 4 76 | if feature_mod != 0: 77 | logger.error("feature_str length is not 4X") 78 | feature_length = len(feature_str) 79 | for i in range(0, feature_length, 4): 80 | i1 = ord(feature_str[i]) 81 | i2 = ord(feature_str[i + 1]) 82 | i3 = ord(feature_str[i + 2]) 83 | i4 = ord(feature_str[i + 3]) 84 | key_int = i1 * 256 + i2 85 | value_int = i3 * 256 + i4 86 | kvd[key_int] = value_int 87 | return kvd 88 | 89 | @staticmethod 90 | def get_key(feature_str, offset): 91 | """ 92 | 93 | :param feature_str: 94 | :param offset: 95 | :return: 96 | """ 97 | return ord(feature_str[offset]) * 256 + ord(feature_str[offset + 1]) 98 | 99 | @staticmethod 100 | def get_value(feature_str, offset): 101 | return ord(feature_str[offset + 2]) * 256 + ord(feature_str[offset + 3]) 102 | 103 | @staticmethod 104 | def comp_str(str1, str2): 105 | assert len(str1) % 4 == 0 and len(str2) % 4 == 0, "Feature_str length is not 4X" 106 | feature_length1, feature_length2 = len(str1), len(str2) 107 | cur1, cur2 = 0, 0 108 | diff = 0 109 | sum1, sum2 = 0, 0 110 | while cur1 < feature_length1 or cur2 < feature_length2: 111 | if cur1 == feature_length1 or Util.get_key(str2, cur2) < Util.get_key(str1, cur1): 112 | v2 = Util.get_value(str2, cur2) 113 | sum2 += v2 114 | diff += v2 115 | cur2 += 4 116 | continue 117 | if cur2 == feature_length2 or Util.get_key(str2, cur2) > Util.get_key(str1, cur1): 118 | v1 = Util.get_value(str1, cur1) 119 | sum1 += v1 120 | diff += v1 121 | cur1 += 4 122 | continue 123 | if Util.get_key(str2, cur2) == Util.get_key(str1, cur1): 124 | v1 = Util.get_value(str1, cur1) 125 | v2 = Util.get_value(str2, cur2) 126 | diff += abs(v1 - v2) 127 | sum1 += v1 128 | sum2 += v2 129 | cur1 += 4 130 | cur2 += 4 131 | continue 132 | assert False, "Not reachable." 133 | return float(diff) / (sum1 + sum2) 134 | -------------------------------------------------------------------------------- /LibRadar/rputil_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from rputil import Util 4 | 5 | class TestUtil(unittest.TestCase): 6 | 7 | def test_dict2str(self): 8 | self.d = dict() 9 | self.d[14] = 21 10 | self.d[35] = 6 11 | self.d[2] = 42 12 | self.feature = Util.dict2str(self.d) 13 | self.assertIsInstance(self.feature, str) 14 | 15 | def test_str2dict(self): 16 | d = dict() 17 | d[14] = 21257 18 | d[35] = 68 19 | d[2] = 42 20 | feature = Util.dict2str(d) 21 | kvd = Util.str2dict(feature) 22 | self.assertIsInstance(kvd, dict) 23 | self.assertEqual(d, kvd) 24 | 25 | def test_comp1(self): 26 | d1 = dict() 27 | d1[14] = 212 28 | d1[35] = 68 29 | d1[2] = 42 30 | d2 = dict() 31 | d2[5] = 24 32 | d2[93] = 31 33 | d2[14] = 222 34 | str1 = Util.dict2str(d1) 35 | str2 = Util.dict2str(d2) 36 | print("Test comp : %f" % Util.comp_str(str1, str2)) 37 | 38 | def test_comp_same(self): 39 | d1 = dict() 40 | d1[14] = 212 41 | d1[93] = 31 42 | d1[5] = 24 43 | d2 = dict() 44 | d2[5] = 24 45 | d2[93] = 31 46 | d2[14] = 212 47 | str1 = Util.dict2str(d1) 48 | str2 = Util.dict2str(d2) 49 | print("Just the same : %f" % Util.comp_str(str1, str2)) 50 | 51 | def test_comp_totoal_diff(self): 52 | d1 = dict() 53 | d1[41] = 212 54 | d1[39] = 31 55 | d1[15] = 24 56 | d2 = dict() 57 | d2[5] = 24 58 | d2[93] = 31 59 | d2[14] = 212 60 | str1 = Util.dict2str(d1) 61 | str2 = Util.dict2str(d2) 62 | print("Total Different : %f" % Util.comp_str(str1, str2)) 63 | 64 | if __name__ == '__main__': 65 | unittest.main() -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include MANIFEST.in 2 | include README.md 3 | include LibRadar/Data/IntermediateData/invokeFormat.txt 4 | include LibRadar/Data/IntermediateData/tag_rules.csv 5 | include docs/QuickStart.md 6 | include docs/PyPI_Index.rst 7 | include tool/redis.conf 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LibRadar 2 | LibRadar is an automatic tool for Android library detection. 3 | 4 | Upload your apk file and LibRadar can detect third-party libraries in Android apps accurately and instantly. 5 | 6 | ## Features 7 | 8 | #### Fast 9 | LibRadar takes just several seconds to scan an app and report the list of third-party libraries used in it. 10 | #### Accurate 11 | LibRadar is trained with more than 1 million apps from Google Play, so it can identify virtually all popular libraries in any given Android app. 12 | #### Anti-Obfuscation 13 | Many Android apps are obfuscated with tools such as ProGuard, which makes it difficult to recognize a library by its package names or class names. 14 | LibRadar is obfuscation-resilient since we use features that cannot be obfuscated, such as statistics on Android APIs. 15 | 16 | ## Usage 17 | 18 | ```bash 19 | $ python LibRadar/libradar.py someapp.apk 20 | ``` 21 | View [docs/QuickStart.md](https://github.com/pkumza/LibRadar/blob/master/docs/QuickStart.md) for more information. 22 | 23 | If you want an online trial, just click http://radar.pkuos.org/. (Updated to version 2) 24 | 25 | ## Description for output 26 | |V2|V1|Stands for|Description| 27 | |---|---|---|---| 28 | |Library|lib|Library|Library Name| 29 | |Package|cpn|Current Package Name|The package name from **your given APK** that seems match this library. 'Current' means what you just uploaded.| 30 | |Permissions|p|Permission|The permissions that the library used. It is specified by the API it used.| 31 | |Popularity|dn|Repetitions|The number of the library (of just the **same version**!)| 32 | |Match Ratio|*Not Implemented*|Similarity|Matched Parts/Total Parts| 33 | |Standard Package|pn|Package Name|The package name from the database that seems matched.| 34 | |Type|tp|Type|The type that the library belongs to.| 35 | |Website|ch|Link| Link for the official SDK developer guide website. I forget why I used 'ch' at the very beginning.| 36 | |*Not Implemented*|bh|B\_Hash|The hash value of the package.| 37 | |*Not Implemented*|btc|B\_Total\_Count|The total count of API.| 38 | |*Not Implemented*|btn|B\_Total\_Number|The total types of API.| 39 | |*Not Implemented*|csp|Current Specified Package Name|The sub-package (a part of the whole package) from your given APK that finally, exactly matched with what in the database.| 40 | |*Not Implemented*|sp|Specified Package Name|The sub-package(a part of the whole package) that exactly matched with that in your APK.| 41 | 42 | ## Dev Environment 43 | 44 | * PyPy Version(Optional): 45 | ``` 46 | Python 2.7.12 (aff251e54385, Nov 09 2016, 17:25:49) 47 | [PyPy 5.6.0 with GCC 4.2.1 Compatible Apple LLVM 5.1 (clang-503.0.40)] on darwin 48 | Type "help", "copyright", "credits" or "license" for more information. 49 | ``` 50 | 51 | * Python Version: 52 | ``` 53 | Python 2.7.12 (v2.7.12:d33e0cf91556, Jun 26 2016, 12:10:39) 54 | [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin 55 | Type "help", "copyright", "credits" or "license" for more information. 56 | ``` 57 | 58 | * redis Version : 59 | ``` 60 | Redis server v=3.2.5 sha=00000000:0 malloc=libc bits=64 build=d73d39f287fb87a1 61 | ``` 62 | 63 | * IDE : 64 | ``` 65 | PyCharm 2016.3.2 66 | Build #PY-163.10154.50, built on December 29, 2016 67 | Licensed to Ziang Ma 68 | Subscription is active until September 15, 2017 69 | For educational use only. 70 | JRE: 1.8.0_112-release-408-b6 x86_64 71 | JVM: OpenJDK 64-Bit Server VM by JetBrains s.r.o 72 | ``` 73 | -------------------------------------------------------------------------------- /artwork/LibRadar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pkumza/LibRadar/2fa3891123e5fd97d631fbe14bf029714c328ca3/artwork/LibRadar.png -------------------------------------------------------------------------------- /docs/PyPI_Index.rst: -------------------------------------------------------------------------------- 1 | LibRadar 2 | =================== 3 | 4 | LibRadar is an automatic tool for third-party library (SDK) detection of Android apps. 5 | 6 | 7 | Fast 8 | ------------------- 9 | 10 | LibRadar takes just several seconds to scan an app and report the list of third-party libraries used in it. 11 | 12 | Accurate 13 | ------------------- 14 | 15 | LibRadar is trained with more than 1 million apps from Google Play, so it can identify virtually all popular libraries in any given Android app. 16 | 17 | Anti-Obfuscation 18 | ------------------- 19 | 20 | Many Android apps are obfuscated with tools such as ProGuard, which makes it difficult to recognize a library by its package names or class names. 21 | LibRadar is obfuscation-resilient since we use features that cannot be obfuscated, such as statistics on Android APIs. 22 | -------------------------------------------------------------------------------- /docs/QuickStart.md: -------------------------------------------------------------------------------- 1 | # QuickStart 2 | 3 | ## How to use LibRadar 4 | 5 | ### Lite Version Online! (The lite version can meet most needs) 6 | 7 | 1. Download Code from lite version repo. 8 | - Github 9 | ```bash 10 | $ git clone https://github.com/pkumza/LiteRadar 11 | ``` 12 | 13 | 2. Download Feature File 14 | 15 | Download lite_datasetfile, then move it into LiteRadar/LiteRadar/Data directory. 16 | 17 | [Mirror 1 Github](https://github.com/pkumza/Data_for_LibRadar/blob/master/lite_dataset_10.csv) 18 | 19 | 20 | 3. Use LibRadar to detect libraries. 21 | 22 | ```bash 23 | $ python literadar.py someapp.apk 24 | ``` 25 | 26 | ### Ordinary Version 27 | 28 | 1. Install dependencies. build-essential, python2, pip, git, redis 29 | 30 | Recommend to use pypy, install pypy and pypy-pip to boost LibRadar 31 | 32 | - Install redis>=3.2 33 | ```bash 34 | $ wget http://download.redis.io/releases/redis-3.2.7.tar.gz 35 | $ tar xzf redis-3.2.7.tar.gz 36 | $ cd redis-3.2.7 37 | $ make 38 | $ apt-get install tcl 39 | $ make test 40 | $ make install 41 | ``` 42 | 43 | 2. Install modules 44 | 45 | ``` 46 | $ pip install requirements.txt 47 | ``` 48 | 49 | 3. Download code 50 | 51 | * Github Release (Recommanded) 52 |
53 | Click [https://github.com/pkumza/LibRadar/releases](https://github.com/pkumza/LibRadar/releases) 54 |
55 | Download zip or tar.gz, then extract code from it. 56 | 57 | - Github 58 | ```bash 59 | $ git clone https://github.com/pkumza/LibRadar 60 | ``` 61 | 62 | - Pypi 63 | 64 | Download tar.gz code from https://pypi.python.org/pypi/LibRadar/ and extract the file into your work directory. 65 | Do not use pip install because I didn't include data file into the code, as libradar will get error that it could not 66 | find data file. 67 | 68 | 4. Download LibRadarData.rdb and run redis-server 69 | 70 | - [~Dropbox Link old~](https://www.dropbox.com/s/w31gig6msdo3cdy/dump-20170515-shrink.rdb.tar.gz?dl=0) 71 | - [Dropbox Link](https://www.dropbox.com/s/ljtzw74twt8xgy6/d.tar.gz?dl=0) 72 | 73 | ```bash 74 | $ vi LibRadar/tool/redis.conf. 75 | ( 76 | find "dir /Users/marchon/Projects/Databases" and change it into your rdb path. 77 | find "dbfilename" and change it into your rdb filename. 78 | ) 79 | 80 | $ redis-server tool/redis.conf 81 | ``` 82 | 83 | 5. Use LibRadar to detect libraries. 84 | 85 | ```bash 86 | $ python libradar.py someapp.apk 87 | ``` 88 | 89 | # Example (Version 2.1.1) 90 | 91 | ```bash 92 | $ python LibRadar/LibRadar/libradar.py happy.apk 93 | [ 94 | { 95 | "Library": "Android Support v4", 96 | "Match Ratio": "6777/6777", 97 | "Package": "Landroid/support/v4", 98 | "Permission": [ 99 | "android.permission.BACKUP", 100 | "android.permission.BLUETOOTH_ADMIN", 101 | "android.permission.DUMP", 102 | "android.permission.INTERACT_ACROSS_USERS", 103 | "android.permission.INTERACT_ACROSS_USERS_FULL", 104 | "android.permission.INTERNET", 105 | "android.permission.WAKE_LOCK", 106 | "android.permission.WRITE_SECURE_SETTINGS" 107 | ], 108 | "Popularity": 1610, 109 | "Standard Package": "Landroid/support/v4", 110 | "Type": "Development Aid", 111 | "Website": "http://developer.android.com/reference/android/support/v4/app/package-summary.html" 112 | }, 113 | { 114 | "Library": "Android Support v7", 115 | "Match Ratio": "3111/3111", 116 | "Package": "Landroid/support/v7", 117 | "Permission": [ 118 | "android.permission.DUMP", 119 | "android.permission.INTERACT_ACROSS_USERS", 120 | "android.permission.INTERACT_ACROSS_USERS_FULL", 121 | "android.permission.INTERNET", 122 | "android.permission.MANAGE_APP_TOKENS" 123 | ], 124 | "Popularity": 2, 125 | "Standard Package": "Landroid/support/v7", 126 | "Type": "Development Aid", 127 | "Website": "https://developer.android.com/reference/android/support/v7/app/package-summary.html" 128 | }, 129 | { 130 | "Library": "Bolts Base Library", 131 | "Match Ratio": "325/325", 132 | "Package": "Lbolts", 133 | "Permission": [ 134 | "android.permission.DUMP", 135 | "android.permission.INTERACT_ACROSS_USERS", 136 | "android.permission.INTERACT_ACROSS_USERS_FULL", 137 | "android.permission.INTERNET" 138 | ], 139 | "Popularity": 3378, 140 | "Standard Package": "Lbolts", 141 | "Type": "Development Aid", 142 | "Website": "https://github.com/BoltsFramework/Bolts-Android" 143 | }, 144 | { 145 | "Library": "Amazon AWS", 146 | "Match Ratio": "3673/3673", 147 | "Package": "Lcom/amazonaws", 148 | "Permission": [ 149 | "android.permission.DUMP" 150 | ], 151 | "Popularity": 3, 152 | "Standard Package": "Lcom/amazonaws", 153 | "Type": "Development Aid", 154 | "Website": "http://mvnrepository.com/artifact/com.amazonaws" 155 | }, 156 | { 157 | "Library": "Google Mobile Services", 158 | "Match Ratio": "26342/26344", 159 | "Package": "Lcom/google/android/gms", 160 | "Permission": [ 161 | "android.permission.BACKUP", 162 | "android.permission.BLUETOOTH_ADMIN", 163 | "android.permission.DUMP", 164 | "android.permission.INTERACT_ACROSS_USERS", 165 | "android.permission.INTERACT_ACROSS_USERS_FULL", 166 | "android.permission.INTERNET", 167 | "android.permission.WAKE_LOCK", 168 | "android.permission.WRITE_SECURE_SETTINGS" 169 | ], 170 | "Popularity": "8619", 171 | "Standard Package": "Lcom/google/android/gms", 172 | "Type": "Development Aid", 173 | "Website": "https://developers.google.com/android/reference/com/google/android/gms/package-summary" 174 | }, 175 | { 176 | "Library": "Google Gson", 177 | "Match Ratio": "390/390", 178 | "Package": "Lcom/google/gson", 179 | "Permission": [], 180 | "Popularity": 7366, 181 | "Standard Package": "Lcom/google/gson", 182 | "Type": "Development Aid", 183 | "Website": "https://github.com/google/gson" 184 | }, 185 | { 186 | "Library": "Google Ads", 187 | "Match Ratio": "47/47", 188 | "Package": "Lcom/google/ads", 189 | "Permission": [ 190 | "android.permission.DUMP" 191 | ], 192 | "Popularity": 23586, 193 | "Standard Package": "Lcom/google/ads", 194 | "Type": "Advertisement", 195 | "Website": "https://www.google.com/ads/" 196 | }, 197 | { 198 | "Library": "Firebase", 199 | "Match Ratio": "554/554", 200 | "Package": "Lcom/google/firebase", 201 | "Permission": [ 202 | "android.permission.DUMP", 203 | "android.permission.INTERACT_ACROSS_USERS", 204 | "android.permission.INTERACT_ACROSS_USERS_FULL", 205 | "android.permission.WRITE_SECURE_SETTINGS" 206 | ], 207 | "Popularity": 14, 208 | "Standard Package": "Lcom/google/firebase", 209 | "Type": "Development Aid", 210 | "Website": "http://firebase.com" 211 | }, 212 | { 213 | "Library": "Facebook", 214 | "Match Ratio": "3758/3758", 215 | "Package": "Lcom/facebook", 216 | "Permission": [ 217 | "android.permission.DUMP", 218 | "android.permission.INTERACT_ACROSS_USERS", 219 | "android.permission.INTERACT_ACROSS_USERS_FULL", 220 | "android.permission.INTERNET" 221 | ], 222 | "Popularity": 54, 223 | "Standard Package": "Lcom/facebook", 224 | "Type": "Social Network", 225 | "Website": "https://developers.facebook.com" 226 | }, 227 | { 228 | "Library": "Google Play", 229 | "Match Ratio": "119/119", 230 | "Package": "Lcom/android/vending", 231 | "Permission": [ 232 | "android.permission.DUMP" 233 | ], 234 | "Popularity": 44625, 235 | "Standard Package": "Lcom/android/vending", 236 | "Type": "App Market", 237 | "Website": "https://play.google.com" 238 | }, 239 | { 240 | "Library": "Unknown", 241 | "Package": "Landroid/support/graphics", 242 | "Popularity": 1525, 243 | "Standard Package": "Landroid/support/graphics", 244 | "Weight": 299 245 | }, 246 | { 247 | "Library": "Unknown", 248 | "Package": "Landroid/support/customtabs", 249 | "Popularity": 1805, 250 | "Standard Package": "Landroid/support/customtabs", 251 | "Weight": 221 252 | }, 253 | { 254 | "Library": "Unknown", 255 | "Package": "Lcom/adcolony", 256 | "Popularity": 221, 257 | "Standard Package": "Lcom/adcolony", 258 | "Weight": 3602 259 | } 260 | ] 261 | 262 | ``` 263 | 264 | ## How to develop LibRadar 265 | 266 | #### For Professional use 267 | 268 | 1. Install LibRadar from [pypi](https://pypi.python.org/pypi/LibRadar) or download the code from [Github](http://github.com/pkumza/LibRadar). 269 | 270 | 2. Get some dependency. 271 | - Install Java Runtime Environment. If you already installed jre, ignore this. 272 | - Install JAD. 273 | - Install Redis. 274 | 275 | 3. Get android.jar from Android SDK and place them into $Project_HOME$/Data/RawData, Run APIDict.py 276 | 277 | 4. Change the apk folder name and run job_dispatching.py 278 | 279 | PS: Change variables in _settings.py if you need. 280 | -------------------------------------------------------------------------------- /docs/compare_with_libd.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Env, Dataset and Paras 4 | ====================== 5 | LibRadar: Default settings. 2.6 GB dataset of library features generated from 1,060,000 apks. 6 | 7 | LibRadar's dataset is generated in 2017. 8 | 9 | LibD: Latest code from Github. Taken 30 as threshold. 10 | 11 | LibD's dataset is generated in 2016. 12 | 13 | We randomly select 50 apks downloaded in 2016 as test set. 14 | 15 | There's no intersection between LibRadar's training dataset and test dataset. 16 | 17 | 18 | First Example 19 | ============= 20 | 21 | APK: zozo.android.lostword.apk 22 | 23 | Result of LibRadar 24 | ------------------ 25 | ``` 26 | com/tapjoy 27 | com/nineoldandroids 28 | com/chartboost/sdk 29 | com/fasterxml 30 | com/sponsorpay 31 | com/google/android/gms 32 | com/google/ads 33 | com/squareup/picasso 34 | com/squareup/okhttp 35 | com/unity3d 36 | com/unity3d/ads 37 | com/immersion/hapticmediasdk 38 | com/jirbo/adcolony 39 | com/android/vending 40 | com/threatmetrix 41 | com/facebook 42 | okio 43 | android/support 44 | android/support/v4 45 | bolts 46 | com/noqoush 47 | ``` 48 | 49 | Result of LibD 50 | -------------- 51 | ``` 52 | com/nineoldandroids/view 53 | com/immersion/content 54 | com/applifier/impact 55 | com/fasterxml/jackson 56 | com/threatmetrix/TrustDefenderMobile 57 | com/immersion/hapticmediasdk 58 | com/auer/lps 59 | com/android/vending 60 | ``` 61 | 62 | Key points 63 | ========== 64 | 65 | Library containing 66 | ------------------ 67 | 68 | LibD can hardly tell lib contains. In many test cases, it never found a library contains another. 69 | 70 | In fact, this situation exists. e.g. com/unity3d contains com/unity3d/ads, Unity3d is a game engine while unity3d ads should be taken as an advertisement library. 71 | 72 | Package name anti-obfuscation 73 | ----------------------------- 74 | LibD can solve name obfuscation problem in code but it could not highly solve the obfuscation problem in package name. 75 | In its library list(such as [Threshold50](https://github.com/IIE-LibD/libd/blob/master/liblist/liblist_threshold_50.csv)), there's many obfuscated package name like "/$$OOOOOOOOOOOO/$OOOOOOOOOOOOOOOO/$OOOOOOOOOOOOOOOO". 76 | Not to mention we could not determine if it is a false positive, we can not determine what the code is in the end. 77 | 78 | Library's boundary 79 | ------------------ 80 | It's hard to determine the boundary. In LibD's library list, /com/nineoldandroids/animation is a library and /com/nineoldandroids/view is another. With tagging assistant, LibRadar can solve this problem. 81 | 82 | LibD didn't find com/google/android/gms and com/google/ads because in the first stage, LibD take com/google as a code bundle(library candidates). However, There's no com/google in library list, so LibD missed `Google GMS` and `Google Ads` at the same time. 83 | 84 | Results of 50 apks 85 | ================== 86 | 87 | ``` 88 | APP 1:/Volumes/banana/apks/com.sohomob.android.aeroplane_chess_battle_ludo_2.apk 89 | LibRadar: 90 | Landroid/support 91 | Landroid/support/v4 92 | Lorg/anddev/andengine 93 | Lokio 94 | Lokhttp3 95 | Lcom/google/android/gms 96 | Lcom/google/android/gcm 97 | Lcom/google/ads 98 | Lcom/millennialmedia 99 | Lcom/inmobi 100 | Lcom/flurry/android 101 | Lcom/flurry/sdk 102 | Lcom/android/vending 103 | Lorg/anddev 104 | -------- 105 | LibD: 106 | com/millennialmedia/android 107 | com/inmobi/analytics 108 | com/millennialmedia/google 109 | com/android/vending 110 | ======== 111 | APP 2:/Volumes/banana/apks/com.luckyxmobile.babycare.apk 112 | LibRadar: 113 | Landroid/support 114 | Landroid/support/v4 115 | Landroid/support/v7 116 | Lorg/achartengine 117 | Lcom/viewpagerindicator 118 | Lcom/google/android/exoplayer 119 | Lcom/google/android/gms 120 | Lcom/google/ads 121 | Lcom/google/firebase 122 | Lcom/millennialmedia 123 | Lcom/amazon/device/ads 124 | Lcom/nineoldandroids 125 | Lcom/umeng/analytics 126 | Lcom/mobeta 127 | Lcom/mopub 128 | Lcom/facebook 129 | Lu/aly 130 | Lau/com/bytecode/opencsv 131 | Lcom/dd 132 | Lcom/placed 133 | Lcom/umeng 134 | Lcom/robotium 135 | Lau 136 | -------- 137 | LibD: 138 | com/nineoldandroids/view 139 | com/viewpagerindicator 140 | ======== 141 | APP 3:/Volumes/banana/apks/com.til.timesnews.apk 142 | LibRadar: 143 | Lretrofit 144 | Le/a 145 | Lcom/comscore 146 | Lcom/google/android/gms 147 | Lcom/google/ads 148 | Lcom/google/f 149 | Lcom/a 150 | Lcom/urbanairship 151 | Lcom/facebook 152 | La/a/a 153 | Landroid/support 154 | Landroid/support/v4 155 | Landroid/support/design 156 | Landroid/support/v7 157 | Lorg/jsoup 158 | Lcom/google/c 159 | Lcom/google/a/a 160 | Lcom/lotame 161 | Lcom/digits/sdk/a 162 | Lcom/twitter/sdk/android/core/internal/oauth 163 | Lcom/twitter/sdk/android/core/identity 164 | Lcom/twitter/sdk/android/tweetcomposer 165 | -------- 166 | LibD: 167 | retrofit/http 168 | retrofit/android 169 | ======== 170 | APP 4:/Volumes/banana/apks/zozo.android.lostword.apk 171 | LibRadar: 172 | Lcom/tapjoy 173 | Lcom/nineoldandroids 174 | Lcom/chartboost/sdk 175 | Lcom/fasterxml 176 | Lcom/sponsorpay 177 | Lcom/google/android/gms 178 | Lcom/google/ads 179 | Lcom/squareup/picasso 180 | Lcom/squareup/okhttp 181 | Lcom/unity3d 182 | Lcom/unity3d/ads 183 | Lcom/immersion/hapticmediasdk 184 | Lcom/jirbo/adcolony 185 | Lcom/android/vending 186 | Lcom/threatmetrix 187 | Lcom/facebook 188 | Lokio 189 | Landroid/support 190 | Landroid/support/v4 191 | Lbolts 192 | Lcom/noqoush 193 | -------- 194 | LibD: 195 | com/nineoldandroids/view 196 | com/immersion/content 197 | com/applifier/impact 198 | com/fasterxml/jackson 199 | com/threatmetrix/TrustDefenderMobile 200 | com/immersion/hapticmediasdk 201 | com/auer/lps 202 | com/android/vending 203 | ======== 204 | APP 5:/Volumes/banana/apks/com.pro.app.compass.apk 205 | LibRadar: 206 | Landroid/support 207 | Landroid/support/v4 208 | Lcom/actionbarsherlock 209 | Lcom/google/android/gms 210 | Lcom/startapp 211 | Lcom/a/c 212 | -------- 213 | LibD: 214 | ======== 215 | APP 6:/Volumes/banana/apks/com.car.parking.traffic.car.driving.apk 216 | LibRadar: 217 | Landroid/support 218 | Landroid/support/v4 219 | Lorg/fmod 220 | Lcom/chartboost/sdk 221 | Lcom/google/android/gms 222 | Lcom/google/android/vending 223 | Lcom/google/ads 224 | Lcom/unity3d 225 | Lcom/unity3d/ads 226 | Lcom/androidnative 227 | -------- 228 | LibD: 229 | com/mnp/utils 230 | ======== 231 | APP 7:/Volumes/banana/apks/com.nema.batterycalibration.apk 232 | LibRadar: 233 | -------- 234 | LibD: 235 | ======== 236 | APP 8:/Volumes/banana/apks/com.cerminara.yazzy.apk 237 | LibRadar: 238 | Landroid/support 239 | Landroid/support/design 240 | Landroid/support/v7 241 | Landroid/support/v4 242 | Lorg/nexage 243 | Lcom/mopub 244 | Lcom/appodeal/ads 245 | Lcom/chartboost/sdk 246 | Lcom/applovin 247 | Lcom/github 248 | Lcom/flurry/android 249 | Lcom/flurry/sdk 250 | Lcom/unity3d 251 | Lcom/unity3d/ads2 252 | Lcom/unity3d/ads 253 | Lcom/google/android/gms 254 | Lcom/google/ads 255 | Lcom/google/firebase 256 | Lcom/amazon/device/ads 257 | Lcom/android/vending 258 | Lcom/my 259 | Lcom/cmcm 260 | Lcom/yandex/metrica 261 | -------- 262 | LibD: 263 | com/android/vending 264 | ======== 265 | APP 9:/Volumes/banana/apks/com.rommanapps.children_police.apk 266 | LibRadar: 267 | Landroid/support 268 | Landroid/support/v4 269 | Lbolts 270 | Lcom/google/android/gms 271 | Lcom/google/ads 272 | Lcom/google/gson 273 | Lcom/millennialmedia 274 | Lcom/startapp 275 | Lcom/facebook 276 | Lcom/parse 277 | Lcom/inmobi 278 | -------- 279 | LibD: 280 | com/google/beintoogson 281 | ======== 282 | APP 10:/Volumes/banana/apks/com.mys.runner.iq.apk 283 | LibRadar: 284 | Lnet/minidev/json 285 | Lcom/prime31 286 | Lcom/jayway/jsonpath 287 | Lcom/tapjoy 288 | Lcom/google/android/gms 289 | Lcom/google/android/gcm 290 | Lcom/google/ads 291 | Lcom/google/tagmanager 292 | Lcom/google/analytics 293 | Lcom/purplebrain/adbuddiz 294 | Lcom/unity3d 295 | Lcom/immersion/hapticmediasdk 296 | Lcom/jirbo/adcolony 297 | Lcom/android/vending 298 | Lcom/threatmetrix 299 | Lcom/facebook 300 | Landroid/support 301 | Landroid/support/v4 302 | Lbolts 303 | Lorg/apache/commons 304 | Lorg/fmod 305 | Lorg/scribe 306 | Lorg/springframework 307 | Lnet 308 | -------- 309 | LibD: 310 | com/android/vending 311 | com/unity3d/player 312 | org/fmod 313 | com/immersion/content 314 | com/threatmetrix/TrustDefenderMobile 315 | org/springframework/beans 316 | net/minidev/json 317 | com/immersion/hapticmediasdk 318 | org/apache/commons 319 | com/ami/fundapter 320 | com/jayway/jsonpath 321 | ======== 322 | APP 11:/Volumes/banana/apks/com.saavn.android.apk 323 | LibRadar: 324 | Landroid/support 325 | Landroid/support/multidex 326 | Landroid/support/design 327 | Landroid/support/v4 328 | Landroid/support/v7 329 | Lcom/comscore 330 | Lcom/viewpagerindicator 331 | Lcom/inmobi 332 | Lcom/google/android/exoplayer 333 | Lcom/google/android/gms 334 | Lcom/google/ads 335 | Lcom/a 336 | Lcom/nostra13 337 | Lcom/mopub 338 | Lcom/facebook 339 | -------- 340 | LibD: 341 | ======== 342 | APP 12:/Volumes/banana/apks/com.androidwasabi.livewallpaper.s4drop.apk 343 | LibRadar: 344 | Lcom/appbrain 345 | Lcom/google/android/gms 346 | Lcom/google/ads 347 | Lcom/google/gson 348 | Lcom/badlogic 349 | Lcmn 350 | -------- 351 | LibD: 352 | com/google/gson 353 | ======== 354 | APP 13:/Volumes/banana/apks/com.sanchezapps.football2015.apk 355 | LibRadar: 356 | Lorg/fmod 357 | Lcom/google/android/gms 358 | Lcom/google/ads 359 | Lcom/unity3d 360 | -------- 361 | LibD: 362 | com/unity3d/player 363 | org/fmod 364 | ======== 365 | APP 14:/Volumes/banana/apks/com.skyboard.google.littlefarm.apk 366 | LibRadar: 367 | Lcom/tapjoy 368 | Lcom/chartboost/sdk 369 | Lcom/threatmetrix 370 | Lcom/google/android/gms 371 | Lcom/google/android/gcm 372 | Lcom/google/ads 373 | Lcom/google/gdata 374 | Lcom/google/gson 375 | Lcom/jirbo/adcolony 376 | Lcom/supersonicads/sdk 377 | Lcom/android/vending 378 | Lcom/nostra13 379 | Lcom/flurry/android 380 | Lcom/flurry/sdk 381 | Lcom/facebook 382 | Landroid/support 383 | Landroid/support/v4 384 | Lbolts 385 | Lorg/cocos2dx 386 | Lorg/cocos2dx 387 | Lcom/amplitude 388 | Lcom/mobileapptracker 389 | Lmuneris/android/iap 390 | Lmuneris/android/core 391 | Lmuneris/android/virtualstore 392 | Lmuneris/android/util 393 | Lmuneris/android/crashreport 394 | Lmuneris/android/extensions 395 | Loauth 396 | -------- 397 | LibD: 398 | com/chartboost/sdk 399 | lombok/extern/log4j 400 | oauth/signpost 401 | com/google/gdata 402 | com/jirbo/adcolony 403 | com/threatmetrix/TrustDefenderMobile 404 | lombok/extern/slf4j 405 | com/tapjoy 406 | com/google/gson 407 | com/android/vending 408 | ======== 409 | APP 15:/Volumes/banana/apks/com.daamitt.walnut.app.apk 410 | LibRadar: 411 | Lcom/google/android/gms 412 | Lcom/google/gson 413 | Lcom/github 414 | Lcom/crashlytics 415 | Lcom/facebook 416 | Landroid/support 417 | Landroid/support/design 418 | Landroid/support/v7 419 | Landroid/support/v4 420 | Lse 421 | -------- 422 | LibD: 423 | ======== 424 | APP 16:/Volumes/banana/apks/com.playtika.wildluckcasino.apk 425 | LibRadar: 426 | Lorg/apache/commons 427 | Lmono/com 428 | Lmono/android 429 | Lcom/google/android/gms 430 | Lcom/google/android/gcm 431 | Lcom/google/ads 432 | Lcom/google/tagmanager 433 | Lcom/google/analytics 434 | Lcom/j256 435 | Lcom/chartboost/sdk 436 | Lcom/adobe/fre 437 | Lcom/appsflyer 438 | Lcom/facebook 439 | Landroid/support 440 | Landroid/support/v4 441 | Lbolts 442 | -------- 443 | LibD: 444 | mono/java/util 445 | mono/javax/xml 446 | opentk/platform/android 447 | opentk/platform/android 448 | com/adobe/fre 449 | mono/java/lang 450 | ======== 451 | APP 17:/Volumes/banana/apks/com.Fisherman.Greekwpa.apk 452 | LibRadar: 453 | Landroid/support 454 | Landroid/support/v4 455 | Landroid/support/design 456 | Lcom/google/android/gms 457 | Lcom/google/ads 458 | Lcom/google/firebase 459 | Lcom/facebook 460 | Lcom/pnikosis 461 | -------- 462 | LibD: 463 | ======== 464 | APP 18:/Volumes/banana/apks/com.mixcloud.player.apk 465 | LibRadar: 466 | Landroid/support 467 | Landroid/support/v4 468 | Landroid/support/v7 469 | Landroid/support/multidex 470 | Lbolts 471 | Lio/fabric/sdk/android 472 | Lokio 473 | Lcom/comscore 474 | Lcom/jakewharton/disklrucache 475 | Lcom/google/android/gms 476 | Lcom/google/thirdparty 477 | Lcom/google/ads 478 | Lcom/google/common 479 | Lcom/google/gson 480 | Lcom/millennialmedia 481 | Lcom/squareup/okhttp 482 | Lcom/crashlytics 483 | Lcom/facebook 484 | Lcom/jakewharton 485 | -------- 486 | LibD: 487 | com/avos/avosstatistics 488 | com/jakewharton/disklrucache 489 | com/millennialmedia/android 490 | com/google/thirdparty 491 | ======== 492 | APP 19:/Volumes/banana/apks/com.slickdroid.vaultypro.apk 493 | LibRadar: 494 | Lcom/google/android/gms 495 | Lcom/google/ads 496 | Lcom/facebook 497 | Lbutterknife 498 | Landroid/support 499 | Landroid/support/design 500 | Landroid/support/v7 501 | Landroid/support/v4 502 | Lcom/readystatesoftware 503 | Lpl 504 | Luk 505 | -------- 506 | LibD: 507 | com/readystatesoftware/systembartint 508 | ======== 509 | APP 20:/Volumes/banana/apks/com.danhilment.real.drumset.apk 510 | LibRadar: 511 | Landroid/support 512 | Landroid/support/v4 513 | Lcom/parse 514 | -------- 515 | LibD: 516 | com/fbkimnx/dpkdgjn91481 517 | ======== 518 | APP 21:/Volumes/banana/apks/com.frozennetworkinc.android_beachfoodpartyfrozen.apk 519 | LibRadar: 520 | Landroid/support 521 | Landroid/support/v4 522 | Landroid/support/v7 523 | Landroid/support/multidex 524 | Ldagger 525 | Lcom/sponsorpay 526 | Lcom/chartboost/sdk 527 | Lcom/applovin 528 | Lcom/millennialmedia 529 | Lcom/vungle 530 | Lcom/inmobi 531 | Lcom/amazon/inapp 532 | Lcom/amazon/device/ads 533 | Lcom/google/android/gms 534 | Lcom/google/android/exoplayer 535 | Lcom/google/android/gcm 536 | Lcom/google/ads 537 | Lcom/unity3d 538 | Lcom/unity3d/ads 539 | Lcom/immersion/hapticmediasdk 540 | Lcom/jirbo/adcolony 541 | Lcom/flurry/android 542 | Lcom/flurry/sdk 543 | Lcom/facebook 544 | Lcom/mopub 545 | Lcom/fyber 546 | Lcom/android/vending 547 | Lcom/common/android/analytics 548 | Lcom/common/android/iap_googleplay 549 | Lcom/common/android/facebook 550 | Lcom/common/android/ads 551 | Lcom/common/android/iap_amazon 552 | Lcom/common/android/utils 553 | Lcom/wiyun/engine/utils 554 | Lcom/wiyun/engine/sound 555 | Lcom/wiyun/engine/network 556 | Lcom/wiyun/engine/nodes 557 | -------- 558 | LibD: 559 | com/vungle/log 560 | com/android/vending 561 | dagger/internal 562 | com/applifier/impact 563 | lombok/extern/slf4j 564 | ======== 565 | APP 22:/Volumes/banana/apks/com.libiitech.libiihome.apk 566 | LibRadar: 567 | Landroid/support 568 | Landroid/support/v4 569 | Lorg/cocos2dx 570 | Lorg/cocos2dx 571 | Lcom/umeng/analytics 572 | Lcom/umeng/common 573 | Lcom/chartboost/sdk 574 | Lcom/applovin 575 | Lcom/google/android/gms 576 | Lcom/google/android/gcm 577 | Lcom/google/ads 578 | Lcom/immersion/hapticmediasdk 579 | Lcom/jirbo/adcolony 580 | Lcom/flurry/android 581 | Lcom/flurry/sdk 582 | Lcom/android/vending 583 | Lcom/umeng 584 | -------- 585 | LibD: 586 | com/immersion/hapticmediasdk 587 | com/immersion/content 588 | com/chartboost/sdk 589 | com/umeng/analytics 590 | com/android/vending 591 | lombok/extern/slf4j 592 | org/OpenUDID 593 | com/umeng/common 594 | ======== 595 | APP 23:/Volumes/banana/apks/pl.netigen.bestelectricguitarfree.apk 596 | LibRadar: 597 | Landroid/support 598 | Landroid/support/v4 599 | Landroid/support/v13 600 | Lcom/google/android/gms 601 | Lcom/google/ads 602 | Lcom/google/firebase 603 | Lcom/squareup/picasso 604 | -------- 605 | LibD: 606 | ======== 607 | APP 24:/Volumes/banana/apks/com.magmamobile.game.slice.apk 608 | LibRadar: 609 | Lcom/adwhirl 610 | Lcom/google/ads 611 | Lcom/millennialmedia 612 | Lcom/adfonic 613 | Lcom/inmobi 614 | Lcom/furnace/utils 615 | Lcom/google/android/apps 616 | Lcom/mdotm 617 | Lcom/magmamobile/mmusia/parser 618 | Lcom/magmamobile/mmusia/activities 619 | Lcom/magmamobile/mmusia/image 620 | Lcom/magmamobile/mmusia/views 621 | Lcom/magmamobile/game/gamelib 622 | Lcom/magmamobile/game/cardsLib 623 | Lcom/zestadz 624 | -------- 625 | LibD: 626 | com/millennialmedia/android 627 | com/mdotm/android 628 | com/inmobi/commons 629 | com/zestadz/android 630 | lombok/extern/slf4j 631 | com/millennialmedia/google 632 | ======== 633 | APP 25:/Volumes/banana/apks/com.monotype.android.font.free.pencil.apk 634 | LibRadar: 635 | Landroid/support 636 | Landroid/support/design 637 | Landroid/support/v7 638 | Landroid/support/v4 639 | Lcom/google/android/gms 640 | Lcom/google/ads 641 | Lcom/google/firebase 642 | Lcom/facebook 643 | Lcom/android/volley 644 | La/a/a 645 | -------- 646 | LibD: 647 | ======== 648 | APP 26:/Volumes/banana/apks/com.eboundservices.pakistantv.apk 649 | LibRadar: 650 | Lcom/google/android/gms 651 | Lcom/google/ads 652 | Lcom/squareup/picasso 653 | Lokhttp3 654 | Lokio 655 | Landroid/support 656 | Landroid/support/design 657 | Landroid/support/v7 658 | Landroid/support/v4 659 | Landroid/support/multidex 660 | Lorg/jsoup 661 | -------- 662 | LibD: 663 | ======== 664 | APP 27:/Volumes/banana/apks/com.lufthansa.android.lufthansa.apk 665 | LibRadar: 666 | Lde/greenrobot/dao 667 | Lde/greenrobot/event 668 | Lcom/google/android/gms 669 | Lcom/google/gson 670 | Lcom/squareup/okhttp 671 | Lcom/crashlytics 672 | Lcom/nostra13 673 | Lio/fabric/sdk/android 674 | Lokio 675 | Landroid/support 676 | Landroid/support/design 677 | Landroid/support/v7 678 | Landroid/support/v4 679 | Lorg/simpleframework 680 | Lorg/jsoup 681 | Lio/card 682 | -------- 683 | LibD: 684 | ======== 685 | APP 28:/Volumes/banana/apks/com.blogspot.euapps.skrillexdubstepmusicpad.apk 686 | LibRadar: 687 | Landroid/support 688 | Landroid/support/v4 689 | Landroid/support/v7 690 | Lcom/google/android/gms 691 | Lcom/google/ads 692 | Lcom/android/vending 693 | Lio/huq/sourcekit/network 694 | -------- 695 | LibD: 696 | com/android/vending 697 | ======== 698 | APP 29:/Volumes/banana/apks/com.igg.android.lordsmobile.apk 699 | LibRadar: 700 | Landroid/support 701 | Landroid/support/v4 702 | Lorg/apache/http 703 | Lorg/fmod 704 | Lcom/google/android/gms 705 | Lcom/google/android/gcm 706 | Lcom/google/ads 707 | Lcom/unity3d 708 | Lcom/appsflyer 709 | Lcom/android/vending 710 | Lcom/facebook 711 | Lcom/google/games 712 | -------- 713 | LibD: 714 | org/apache/http 715 | ======== 716 | APP 30:/Volumes/banana/apks/com.ezmusicplayer.demo.apk 717 | LibRadar: 718 | Landroid/support 719 | Landroid/support/v4 720 | Landroid/support/v7 721 | Lcom/google/android/gms 722 | Lcom/mobeta 723 | -------- 724 | LibD: 725 | com/a 726 | ======== 727 | APP 31:/Volumes/banana/apks/com.wiziapp.app104340.apk 728 | LibRadar: 729 | Landroid/support 730 | Landroid/support/v4 731 | Lorg/apache/commons 732 | Lorg/apache/http 733 | Lorg/acra 734 | Lcom/google/android/gms 735 | Lcom/google/android/gcm 736 | Lcom/google/ads 737 | Lcom/google/gson 738 | Lcom/google/analytics 739 | Lcom/loopj 740 | -------- 741 | LibD: 742 | org/apache/commons 743 | com/google/gson 744 | com/google/ads 745 | org/apache/http 746 | lombok/extern/slf4j 747 | org/acra 748 | ======== 749 | APP 32:/Volumes/banana/apks/com.mrdt.racegame.apk 750 | LibRadar: 751 | Lcom/appbrain 752 | Lcom/esotericsoftware 753 | Lcom/google/android/gms 754 | Lcom/google/ads 755 | Lcom/badlogic 756 | Lcmn 757 | -------- 758 | LibD: 759 | com/esotericsoftware/tablelayout 760 | lombok/extern/slf4j 761 | ======== 762 | APP 33:/Volumes/banana/apks/com.g6677.android.petspasalon.apk 763 | LibRadar: 764 | Lorg/cocos2dx 765 | Lorg/cocos2dx 766 | Lcom/tapjoy 767 | Lcom/chartboost/sdk 768 | Lcom/millennialmedia 769 | Lcom/amazon/inapp 770 | Lcom/github 771 | Lcom/google/ads 772 | Lcom/google/common 773 | Lcom/flurry/android 774 | Lcom/inmobi 775 | Lcom/g6677 776 | Lcom/cocos2d 777 | -------- 778 | LibD: 779 | com/inmobi/commons 780 | com/millennialmedia/android 781 | com/chartboost/sdk 782 | com/flurry/android 783 | com/inmobi/androidsdk 784 | ======== 785 | APP 34:/Volumes/banana/apks/com.contactive.apk 786 | LibRadar: 787 | Ljunit 788 | Lretrofit 789 | Lcom/mixpanel/android/java_websocket 790 | Lcom/google/android/gms 791 | Lcom/google/common 792 | Lcom/google/gson 793 | Lcom/nineoldandroids 794 | Lcom/nostra13 795 | Lcom/facebook 796 | Lorg/hamcrest 797 | Lorg/slf4j 798 | Lorg/joda/time 799 | Lorg/junit 800 | Lorg/apache/commons 801 | Lnet/hockeyapp/android 802 | Landroid/support 803 | Landroid/support/v4 804 | Landroid/support/v7 805 | Lbolts 806 | Lch/qos/logback 807 | La_vcard 808 | Lcom/mixpanel 809 | Lcom/google/i18n/phonenumbers/prefixmapper 810 | Lorg/joda 811 | Lorg/jraf 812 | Lnet 813 | Lme/leolin 814 | Lch 815 | -------- 816 | LibD: 817 | com/mindprod/ledatastream 818 | brut/common 819 | lombok/extern/log4j 820 | a_vcard/com/android 821 | com/nineoldandroids/view 822 | retrofit/android 823 | retrofit/http 824 | com/gamelion/Dradfgedhyew 825 | ======== 826 | APP 35:/Volumes/banana/apks/br.com.radios.radiosmobile.radiosnet.apk 827 | LibRadar: 828 | Landroid/support 829 | Landroid/support/v4 830 | Landroid/support/v7 831 | Lcom/google/android/gms 832 | Lcom/google/a 833 | Lcom/bumptech 834 | -------- 835 | LibD: 836 | ======== 837 | APP 36:/Volumes/banana/apks/air.com.zyncstudio.PoolBabyCare.apk 838 | LibRadar: 839 | Landroid/support 840 | Landroid/support/v4 841 | Landroid/support/v13 842 | Landroid/support/v7 843 | Lcom/google/android/gms 844 | Lcom/google/ads 845 | Lcom/adobe/flashplayer 846 | Lcom/adobe/flashruntime 847 | Lcom/adobe/air 848 | Lcom/adobe/fre 849 | Lcom/milkmangames 850 | -------- 851 | LibD: 852 | com/adobe/fre 853 | com/adobe/flashplayer 854 | com/amazonaws/services/ 855 | ======== 856 | APP 37:/Volumes/banana/apks/com.g5e.specialenquiry.apk 857 | LibRadar: 858 | Landroid/support 859 | Landroid/support/v4 860 | Lorg/fmod 861 | Lcom/google/android/vending 862 | Lcom/google/android/gcm 863 | Lcom/b/a 864 | Lcom/facebook 865 | Lcom/d 866 | Lcom/arellomobile 867 | Lcom/g5e/xpromo 868 | Lcom/b 869 | Lcom/swrve/sdk/e 870 | Lcom/swrve/sdk/g 871 | -------- 872 | LibD: 873 | ======== 874 | APP 38:/Volumes/banana/apks/com.appspot.swisscodemonkeys.detector.apk 875 | LibRadar: 876 | Lcom/appbrain 877 | Lcom/google/android/gms 878 | Lcom/viewpagerindicator 879 | Landroid/support 880 | Landroid/support/v4 881 | Landroid/support/v7 882 | -------- 883 | LibD: 884 | ======== 885 | APP 39:/Volumes/banana/apks/com.schnapps.projectorface.apk 886 | LibRadar: 887 | Landroid/support 888 | Landroid/support/v4 889 | Landroid/support/v7 890 | Lcom/nineoldandroids 891 | Lcom/github 892 | Lcom/google/android/gms 893 | Lcom/google/ads 894 | Lcom/daimajia/androidanimations 895 | Lcom/vk/sdk/api/model 896 | Lcom/vk/sdk/api/httpClient 897 | Lcom/vk/sdk/dialogs 898 | Lcom/vk/sdk/util 899 | Lcom/daimajia 900 | -------- 901 | LibD: 902 | com/nineoldandroids/view 903 | com/daimajia/androidanimations 904 | ======== 905 | APP 40:/Volumes/banana/apks/com.kii.safe.apk 906 | LibRadar: 907 | Lbutterknife 908 | Lio/fabric/sdk/android 909 | Landroid/support 910 | Landroid/support/v4 911 | Landroid/support/v7 912 | Landroid/support/design 913 | Lretrofit2 914 | Lcom/google/android/exoplayer 915 | Lcom/google/android/gms 916 | Lcom/google/ads 917 | Lcom/facebook 918 | Lcom/bumptech 919 | Lcom/github 920 | Lcom/immersion/hapticmediasdk 921 | Lcom/crashlytics 922 | Lcom/mopub 923 | Lcom/android/vending 924 | Lpl 925 | Lcom/samsung 926 | -------- 927 | LibD: 928 | com/ami/fundapter 929 | com/atminn/urbanroundup 930 | com/ami/fundapter 931 | com/ami/fundapter 932 | ======== 933 | APP 41:/Volumes/banana/apks/com.epicforce.iFighter2.apk 934 | LibRadar: 935 | Landroid/support 936 | Landroid/support/v4 937 | Lorg/fmod 938 | Lcom/google/android/gms 939 | Lcom/google/a 940 | Lcom/facebook 941 | -------- 942 | LibD: 943 | com/a 944 | ======== 945 | APP 42:/Volumes/banana/apks/com.aitype.android.lang.ar.apk 946 | LibRadar: 947 | Landroid/support 948 | Landroid/support/v4 949 | Landroid/support/v7 950 | Lcom/google/android/gms 951 | Lcom/flurry/android 952 | Lcom/flurry/sdk 953 | Lcom/upalytics/sdk/gson 954 | Lcom/android/volley 955 | Lcom/upalytics 956 | -------- 957 | LibD: 958 | ======== 959 | APP 43:/Volumes/banana/apks/com.g5e.cursedship.android.apk 960 | LibRadar: 961 | Landroid/support 962 | Landroid/support/v4 963 | Lcom/chartboost/sdk 964 | Lcom/flurry/android 965 | Lcom/flurry/sdk 966 | Lcom/google/android/gms 967 | Lcom/google/android/vending 968 | Lcom/google/android/vending 969 | Lcom/google/android/gcm 970 | Lcom/google/ads 971 | Lcom/android/vending 972 | Lcom/android/vending 973 | Lcom/facebook 974 | Lcom/pushwoosh 975 | Lcom/arellomobile 976 | Lcom/fiksu 977 | Lcom/kochava 978 | Lcom/playnomics 979 | Lcom/mobileapptracker 980 | -------- 981 | LibD: 982 | com/flurry/android 983 | com/chartboost/sdk 984 | lombok/extern/slf4j 985 | ======== 986 | APP 44:/Volumes/banana/apks/uk.tapmedia.qrreader.apk 987 | LibRadar: 988 | Lcom/google/zxing 989 | Lcom/google/ads 990 | -------- 991 | LibD: 992 | com/google/ads 993 | ======== 994 | APP 45:/Volumes/banana/apks/eu.softwareworkshop.advancedghostradar.apk 995 | LibRadar: 996 | Lcom/esotericsoftware 997 | Lcom/google/android/gms 998 | Lcom/google/ads 999 | Lcom/google/tagmanager 1000 | Lcom/google/analytics 1001 | Lcom/badlogic 1002 | -------- 1003 | LibD: 1004 | com/esotericsoftware/tablelayout 1005 | lombok/extern/slf4j 1006 | ======== 1007 | APP 46:/Volumes/banana/apks/com.kkkeyboard.emoji.keyboard.theme.ParisNight.apk 1008 | LibRadar: 1009 | Lretrofit 1010 | Landroid/support 1011 | Landroid/support/design 1012 | Landroid/support/v7 1013 | Landroid/support/v4 1014 | Landroid/support/a 1015 | Lcom/squareup/okhttp 1016 | Lcom/google/android/gms 1017 | Lcom/google/gson 1018 | Lcom/mopub 1019 | Lcom/nineoldandroids 1020 | Lcom/a 1021 | Lcom/facebook 1022 | Lcom/flurry/android 1023 | Lcom/flurry/sdk 1024 | Lcom/android/volley 1025 | Lokio 1026 | Lcom/appjolt 1027 | -------- 1028 | LibD: 1029 | retrofit/mime 1030 | retrofit/converter 1031 | com/nineoldandroids/view 1032 | retrofit/appengine 1033 | retrofit/android 1034 | retrofit/http 1035 | retrofit/client 1036 | ======== 1037 | APP 47:/Volumes/banana/apks/com.com2us.derbyday.normal.freefull.google.global.android.common.apk 1038 | LibRadar: 1039 | Landroid/support 1040 | Landroid/support/v4 1041 | Lbolts 1042 | Lcom/tapjoy 1043 | Lcom/chartboost/sdk 1044 | Lcom/google/android/gms 1045 | Lcom/google/android/vending 1046 | Lcom/google/android/gcm 1047 | Lcom/android/vending 1048 | Lcom/facebook 1049 | Lcom/com2us/wrapper/module 1050 | Lcom/com2us/peppermint/util 1051 | Lcom/com2us/module/inapp/lebi 1052 | Lcom/com2us/module/inapp/googleplay 1053 | Lcom/com2us/module/util 1054 | Lcom/com2us/module/activeuser/useragree 1055 | Lcom/com2us/module/activeuser/checkpermission 1056 | Lcom/com2us/module/manager 1057 | Lcom/com2us/module/push 1058 | -------- 1059 | LibD: 1060 | com/gamelion/Dradfgedhyew 1061 | ======== 1062 | APP 48:/Volumes/banana/apks/com.digitalpalette.pizap.apk 1063 | LibRadar: 1064 | Lcom/newrelic 1065 | Lcom/nostra13 1066 | Lcom/github 1067 | Lcom/google/android/gms 1068 | Lcom/google/ads 1069 | Lcom/sec/android/iap 1070 | Lcom/crashlytics 1071 | Lcom/android/vending 1072 | Lcom/nokia/payment/iap 1073 | Lcom/facebook 1074 | Landroid/support 1075 | Landroid/support/v4 1076 | Landroid/support/v7 1077 | Lbolts 1078 | Lde 1079 | Lcom/afollestad 1080 | Lcom/skubit 1081 | Lcom/androidquery/util 1082 | Lcom/androidquery/auth 1083 | Lcom/larvalabs 1084 | Lafzkl 1085 | Lorg/onepf/oms/appstore/fortumoUtils 1086 | Lorg/onepf/oms/appstore/skubitUtils 1087 | -------- 1088 | LibD: 1089 | com/android/vending 1090 | com/nokia/payment 1091 | a/b 1092 | com/afollestad/materialdialogs 1093 | javax/inject 1094 | com/sec/android 1095 | filter/pack 1096 | com/skubit/android 1097 | ======== 1098 | APP 49:/Volumes/banana/apks/com.gameinsight.bbdeluxe.apk 1099 | LibRadar: 1100 | Landroid/support 1101 | Landroid/support/v4 1102 | Lbolts 1103 | Lde/greenrobot/event 1104 | Lcom/chartboost/sdk 1105 | Lcom/google/android/gms 1106 | Lcom/google/android/gcm 1107 | Lcom/google/ads 1108 | Lcom/appsflyer 1109 | Lcom/android/vending 1110 | Lcom/flurry/android 1111 | Lcom/flurry/sdk 1112 | Lcom/facebook 1113 | Lde 1114 | Lcom/vk/sdk/dialogs 1115 | Lcom/vk/sdk/payments 1116 | Lcom/vk/sdk/api/httpClient 1117 | Lcom/vk/sdk/util 1118 | Lcom/tune/ma 1119 | Lcom/tune/http 1120 | Lcom/google/example 1121 | Lcom/devtodev/core/data/metrics/simple 1122 | Lcom/gameinsight/ic 1123 | Lcom/gameinsight/fzmobile/service 1124 | Lcom/gameinsight/fzmobile/fzudid 1125 | Lcom/gameinsight/fzmobile/facebook 1126 | -------- 1127 | LibD: 1128 | com/android/vending 1129 | ======== 1130 | APP 50:/Volumes/banana/apks/nexbit.icons.moonshine.apk 1131 | LibRadar: 1132 | Landroid/support 1133 | Landroid/support/design 1134 | Landroid/support/v7 1135 | Landroid/support/v4 1136 | Lcom/bumptech 1137 | Lcom/google/android/apps 1138 | Lcom/makeramen 1139 | Lcom/afollestad 1140 | -------- 1141 | LibD: 1142 | com/makeramen/roundedimageview 1143 | com/afollestad/materialdialogs 1144 | com/bumptech/glide 1145 | ======== 1146 | 1147 | ``` -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | redis>=2.10.5 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | from setuptools import setup 4 | 5 | # Utility function to read the README file. 6 | # Used for the long_description. It's nice, because now 1) we have a top level 7 | # README file and 2) it's easier to type in the README file than to put a raw 8 | # string in below ... 9 | 10 | import LibRadar 11 | 12 | 13 | def read(f_name): 14 | return open(os.path.join(os.path.dirname(__file__), f_name)).read() 15 | 16 | setup( 17 | name=LibRadar.__title__, 18 | version=LibRadar.__version__, 19 | author=LibRadar.__author__, 20 | author_email=LibRadar.__email__, 21 | description=LibRadar.__summary__, 22 | license=LibRadar.__license__, 23 | keywords="Android Third-party Library", 24 | url=LibRadar.__uri__, 25 | packages=['LibRadar'], 26 | long_description=read('docs/PyPI_Index.rst'), 27 | requires=['redis'], 28 | classifiers=[ 29 | "Development Status :: 4 - Beta", 30 | "Programming Language :: Python :: 2.7", 31 | "Programming Language :: Python :: 2 :: Only", 32 | "Operating System :: MacOS :: MacOS X", 33 | "Operating System :: POSIX :: Linux", 34 | "Topic :: Security", 35 | "Topic :: Utilities", 36 | "License :: OSI Approved :: Apache Software License" 37 | ], 38 | ) 39 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | print "Err :old\n" -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import commands 5 | 6 | 7 | """ 8 | 9 | Test 2016-12-01 1 10 | 11 | usage of jad and commands 12 | 13 | 14 | cmd = "jadc" 15 | status, out = commands.getstatusoutput('jasdcs') 16 | print "OUTPUT is " 17 | print out 18 | print status 19 | """ 20 | 21 | """ 22 | 23 | Test 2016-12-02 1 24 | 25 | 26 | 27 | """ 28 | 29 | import redis 30 | 31 | r = redis.StrictRedis() 32 | r.set('name', 'zachary marv') 33 | print r.get('man') -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Test Database 4 | 5 | I have analysed 3048 APKs in 2017-1-7. 6 | Some features are already put into database. 7 | 8 | This script is used to exam the information in database. 9 | """ 10 | 11 | """ 12 | Test db case 1 13 | """ 14 | 15 | def case1(): 16 | import redis 17 | 18 | db3 = redis.StrictRedis(db=3) 19 | db4 = redis.StrictRedis(db=4) 20 | db5 = redis.StrictRedis(db=5) 21 | 22 | for key in db3.keys(): 23 | if int(db3.get(key)) > 100: 24 | if int(db4.get(key)) > 10: 25 | print "PackageName :", db5.get(key) 26 | print "Count :", db3.get(key) 27 | print "Weight :", db4.get(key) 28 | print "-------" 29 | 30 | def case2(): 31 | import redis 32 | 33 | db3 = redis.StrictRedis(db=3) 34 | db4 = redis.StrictRedis(db=4) 35 | db5 = redis.StrictRedis(db=5) 36 | 37 | for key in db3.keys(): 38 | if int(db3.get(key)) > 10: 39 | if int(db4.get(key)) > 100: 40 | print "PackageName :", db5.get(key) 41 | print "Count :", db3.get(key) 42 | print "Weight :", db4.get(key) 43 | print "-------" 44 | 45 | if __name__ == "__main__": 46 | case2() -------------------------------------------------------------------------------- /tests/test_logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | 5 | logging.warning("hongda!") 6 | logging.error("Info") 7 | logging.critical('critical message') -------------------------------------------------------------------------------- /tests/test_multiprocessing.py: -------------------------------------------------------------------------------- 1 | #coding:utf-8 2 | 3 | from multiprocessing import Process, Queue, Pool 4 | import multiprocessing 5 | import os, time, random 6 | 7 | # 写数据进程执行的代码: 8 | def write(q): 9 | for value in ['A', 'B', 'C']: 10 | print 'Put %s to queue...' % value 11 | q.put(value) 12 | time.sleep(random.random()) 13 | 14 | # 读数据进程执行的代码: 15 | def read(q): 16 | while True: 17 | if not q.empty(): 18 | value = q.get(True) 19 | print 'Get %s from queue.' % value 20 | time.sleep(random.random()) 21 | else: 22 | break 23 | 24 | 25 | if __name__ == '__main__': 26 | manager = multiprocessing.Manager() 27 | # 父进程创建Queue，并传给各个子进程： 28 | q = manager.Queue() 29 | p = Pool() 30 | pw = p.apply_async(write, args=(q,)) 31 | time.sleep(0.5) 32 | pr = p.apply_async(read, args=(q,)) 33 | p.close() 34 | p.join() 35 | 36 | print 37 | print '所有数据都写入并且读完' -------------------------------------------------------------------------------- /tests/test_queue.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf8 3 | import random, threading, time 4 | from Queue import Queue 5 | 6 | 7 | # Producer thread 8 | class Producer(threading.Thread): 9 | def __init__(self, t_name, queue): 10 | threading.Thread.__init__(self, name=t_name) 11 | self.data = queue 12 | 13 | def run(self): 14 | for i in range(10): # 随机产生10个数字 ，可以修改为任意大小 15 | randomnum = random.randint(1, 99) 16 | print "%s: %s is producing %d to the queue!" % (time.ctime(), self.getName(), randomnum) 17 | self.data.put(randomnum) # 将数据依次存入队列 18 | time.sleep(1) 19 | print "%s: %s finished!" % (time.ctime(), self.getName()) 20 | 21 | 22 | # Consumer thread 23 | class Consumer_even(threading.Thread): 24 | def __init__(self, t_name, queue): 25 | threading.Thread.__init__(self, name=t_name) 26 | self.data = queue 27 | 28 | def run(self): 29 | while 1: 30 | try: 31 | val_even = self.data.get(1, 5) # get(self, block=True, timeout=None) ,1就是阻塞等待,5是超时5秒 32 | if val_even % 2 == 0: 33 | print "%s: %s is consuming. %d in the queue is consumed!" % (time.ctime(), self.getName(), val_even) 34 | time.sleep(2) 35 | else: 36 | self.data.put(val_even) 37 | time.sleep(2) 38 | except: # 等待输入，超过5秒 就报异常 39 | print "%s: %s finished!" % (time.ctime(), self.getName()) 40 | break 41 | 42 | 43 | class Consumer_odd(threading.Thread): 44 | def __init__(self, t_name, queue): 45 | threading.Thread.__init__(self, name=t_name) 46 | self.data = queue 47 | 48 | def run(self): 49 | while 1: 50 | try: 51 | val_odd = self.data.get(1, 5) 52 | if val_odd % 2 != 0: 53 | print "%s: %s is consuming. %d in the queue is consumed!" % (time.ctime(), self.getName(), val_odd) 54 | time.sleep(2) 55 | else: 56 | self.data.put(val_odd) 57 | time.sleep(2) 58 | except: 59 | print "%s: %s finished!" % (time.ctime(), self.getName()) 60 | break 61 | 62 | 63 | # Main thread 64 | def main(): 65 | queue = Queue() 66 | producer = Producer('Pro.', queue) 67 | consumer_even = Consumer_even('Con_even.', queue) 68 | consumer_odd = Consumer_odd('Con_odd.', queue) 69 | producer.start() 70 | consumer_even.start() 71 | consumer_odd.start() 72 | producer.join() 73 | consumer_even.join() 74 | consumer_odd.join() 75 | print 'All threads terminate!' 76 | 77 | 78 | if __name__ == '__main__': 79 | main() -------------------------------------------------------------------------------- /tests/test_tk.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from Tkinter import * 3 | 4 | class Application(Frame): 5 | def say_hi(self): 6 | print "hi there, everyone!" 7 | 8 | def createWidgets(self): 9 | self.Label = Label(self, text='|---- LibRadar Tagging System -----|\n| Version: 2.0.1.dev1 |') 10 | self.Label.pack() 11 | 12 | self.QUIT = Button(self) 13 | self.QUIT["text"] = "QUIT" 14 | self.QUIT["fg"] = "red" 15 | self.QUIT["command"] = self.quit 16 | 17 | self.QUIT.pack({"side": "left"}) 18 | 19 | self.hi_there = Button(self) 20 | self.hi_there["text"] = "Hello", 21 | self.hi_there["command"] = self.say_hi 22 | 23 | self.hi_there.pack({"side": "left"}) 24 | 25 | def __init__(self, master=None): 26 | Frame.__init__(self, master) 27 | self.pack() 28 | self.createWidgets() 29 | 30 | root = Tk() 31 | app = Application(master=root) 32 | app.mainloop() 33 | # root.destroy() -------------------------------------------------------------------------------- /tool/.redis.conf.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pkumza/LibRadar/2fa3891123e5fd97d631fbe14bf029714c328ca3/tool/.redis.conf.swp -------------------------------------------------------------------------------- /tool/AXMLPrinter2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pkumza/LibRadar/2fa3891123e5fd97d631fbe14bf029714c328ca3/tool/AXMLPrinter2.jar -------------------------------------------------------------------------------- /tool/jad: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pkumza/LibRadar/2fa3891123e5fd97d631fbe14bf029714c328ca3/tool/jad -------------------------------------------------------------------------------- /tool/jar_decompiler.sh: -------------------------------------------------------------------------------- 1 | # use this shell script to decompile android.jar 2 | # thanks to http://stackoverflow.com/questions/647116/how-to-decompile-a-whole-jar-file 3 | # you have to put `jad` into your PATH environment. 4 | # zachary created at 2016-12 5 | JAR=$1 6 | unzip -d $JAR.dir $JAR 7 | pushd $JAR.dir 8 | for f in `find . -name '*.class'`; do 9 | jad -d $(dirname $f) -s java -lnc $f 10 | done 11 | popd 12 | --------------------------------------------------------------------------------